class ConnectionsResource(LoggingMixin, Resource):
id = fields.CharField(attribute='id')
facebook_id = fields.CharField(attribute='facebook_id', null=True)
username = fields.CharField(attribute='username', null=True)
first_name = fields.CharField(attribute='first_name')
last_name = fields.CharField(attribute='last_name')
user_id = fields.CharField(attribute='user_id')
age = fields.IntegerField(attribute='age')
distance = fields.ListField(attribute='distance')
about = fields.CharField(attribute='about', null=True)
gender = fields.CharField(attribute='gender', default=u'all')
photos = fields.ListField(attribute='photos')
goals = fields.ListField(attribute='goals')
offers = fields.ListField(attribute='offers')
interests = fields.ListField(attribute='interests')
top_interests = fields.ListField(attribute='top_interests')
score = fields.IntegerField(attribute='score', null=True)
mutual_likes_count = fields.IntegerField(attribute='mutual_likes_count',
null=True)
total_likes_count = fields.IntegerField(attribute='total_likes_count',
null=True)
es_score = fields.FloatField(attribute='es_score', null=True)
friends_score = fields.IntegerField(attribute='friends_score', null=True)
friend_id = fields.CharField(attribute='friend_id', null=True)
seen = fields.BooleanField(attribute='seen', null=True)
last_login = fields.DateTimeField(attribute='last_login', null=True)
image = fields.FileField(attribute="image", null=True, blank=True)
position = fields.DictField(attribute="position", null=True, blank=True)
lives_in = fields.CharField(attribute="lives_in", null=True, blank=True)
twitter_provider = fields.CharField(attribute="twitter_provider",
null=True,
blank=True)
linkedin_provider = fields.CharField(attribute="linkedin_provider",
null=True,
blank=True)
twitter_username = fields.CharField(attribute="twitter_username",
null=True,
blank=True)
class Meta:
resource_name = 'connections'
authentication = JSONWebTokenAuthentication()
authorization = Authorization()
def detail_uri_kwargs(self, bundle_or_obj):
kwargs = {}
if isinstance(bundle_or_obj, Bundle):
kwargs['pk'] = bundle_or_obj.obj.id
else:
kwargs['pk'] = bundle_or_obj.id
return kwargs
def get_object_list(self, request):
logger.info('Get connections for user_id {}'.format(request.user.id))
fs = FilterState.objects.filter(user=request.user.id)
cache_match_users_with_filter = None
cache_match_users = None
filter_updated_sha = None
raw_filter = request.GET.get('filter')
is_filter = True if (raw_filter and raw_filter in ['true']) else False
if is_filter:
if fs:
try:
attrs = [
fs[0].gender, fs[0].min_age, fs[0].max_age,
fs[0].distance, fs[0].distance_unit,
fs[0].order_criteria, fs[0].keyword
]
filter_updated = '.'.join(map(str, attrs))
filter_updated_sha = hashlib.sha1(
filter_updated).hexdigest()
cache_match_users_with_filter = cache.get(
'c_%s_%s' % (request.user.id, filter_updated_sha))
except AttributeError as err:
logger.error(err)
if cache_match_users_with_filter:
match_users = cache_match_users_with_filter
else:
match_users = MatchQuerySet. \
all(request.user.id, is_filter=True, friends=True)
cache.set('c_%s_%s' % (request.user.id, filter_updated_sha),
match_users)
else:
if cache_match_users:
match_users = cache_match_users
else:
match_users = MatchQuerySet. \
all(request.user.id, friends=True)
match_users = sorted(match_users, key=lambda x: -x.score)
cache.set('c_%s' % (request.user.id, ), match_users)
return match_users
def obj_get_list(self, bundle, **kwargs):
# Filtering disabled for brevity...
return self.get_object_list(bundle.request)
def rollback(self, bundles):
pass
def obj_get(self, bundle, **kwargs):
pass
def dehydrate(self, bundle):
bundle.data['religious_views'] = get_religious_views(bundle.obj.id)
bundle.data['political_views'] = get_political_views(bundle.obj.id)
return bundle
class FoodTruckResource(Resource):
'''
Tastypie non-ORM resource to capture Yelp data and pass it back to the client.
'''
# Define all the relevant "model" fields here without actually defining a Django ORM model
id = fields.CharField(attribute='id')
name = fields.CharField(attribute='name')
categories = fields.CharField(attribute='categories')
display_address = fields.CharField(attribute='display_address')
url = fields.CharField(attribute='url')
review_count = fields.IntegerField(attribute='review_count')
rating_img_url = fields.CharField(attribute='rating_img_url')
rating = fields.FloatField(attribute='rating')
latitude = fields.FloatField(attribute='latitude')
longitude = fields.FloatField(attribute='longitude')
class Meta:
object_class = FoodTruck
def obj_get_list(self, bundle, **kwargs):
'''
Search's Yelp's API for relevant food trucks defined in the search parameters given in bundle.
:param bundle: A Tastypie bundle containing, among other things, the HTTP request containing search parameters.
Mandatory parameters are:
-ne: coordinates (in latitude, longitude form) representing the north-eastern point of the search area.
-sw: coordinates (in latitude, longitude form) representing the south-wetern point of the search area.
Optional parameters are:
-name: a search term relevant to the food truck you are looking for, i.e. waffles, HappyIceCreamTruck, etc.
:type bundle: Bundle.
'''
foodTrucks = []
if 'ne' in bundle.request.GET and bundle.request.GET[
'ne'] and 'sw' in bundle.request.GET and bundle.request.GET[
'sw']:
[neLat, neLng] = [
float(c)
for c in re.findall('[0-9.-]+', bundle.request.GET['ne'])
]
[swLat, swLng] = [
float(c)
for c in re.findall('[0-9.-]+', bundle.request.GET['sw'])
]
# Set up a session to the Yelp API
session = OAuth1Session(
consumer_key=os.environ['YELP_CONSUMER_KEY'],
consumer_secret=os.environ['YELP_CONSUMER_SECRET'],
access_token=os.environ['YELP_TOKEN'],
access_token_secret=os.environ['YELP_TOKEN_SECRET'])
searchParams = {
'category_filter': 'foodtrucks',
'limit': 20,
'bounds': '{0},{1}|{2},{3}'.format(swLat, swLng, neLat, neLng)
}
# Optionally pass in the name of the food truck the user is looking for, if at all
if 'name' in bundle.request.GET and bundle.request.GET['name']:
searchParams['term'] = bundle.request.GET['name']
response = session.get('http://api.yelp.com/v2/search',
params=searchParams)
parsed_response = response.json()
for foodTruck in parsed_response['businesses']:
if foodTruck['is_closed'] == False:
display_address = "\n".join(
foodTruck['location']['display_address'])
# Get rid of the Yelp category identifiers
categories = ", ".join(
[x[0] for x in foodTruck['categories']])
ftModel = {
'id':
foodTruck['id'],
'name':
foodTruck['name'],
'display_address':
display_address,
'url':
foodTruck['url'],
'categories':
categories,
'review_count':
foodTruck['review_count'],
'rating_img_url':
foodTruck['rating_img_url'],
'rating':
foodTruck['rating'],
'latitude':
foodTruck['location']['coordinate']['latitude'],
'longitude':
foodTruck['location']['coordinate']['longitude']
}
foodTrucks.append(FoodTruck(**ftModel))
return foodTrucks
class ProjectsResource(ModelResource):
"""Projects resource"""
id = fields.CharField(attribute='id', readonly=True)
name = fields.CharField(attribute='name', readonly=True)
url = fields.CharField(attribute='url', readonly=True)
is_private = fields.BooleanField(attribute='is_private', readonly=True)
branches = fields.ListField(attribute='branches', readonly=True)
icon = fields.CharField(attribute='icon', readonly=True, null=True)
badge_url = fields.CharField(attribute='get_badge_url', readonly=True)
owner_id = fields.CharField(attribute='owner_id', readonly=True)
token = fields.CharField(attribute='token', blank=True, null=True)
default_branch = fields.CharField(
attribute='default_branch',
blank=True,
null=True,
)
dashboard_branch = fields.CharField(
attribute='dashboard_branch',
blank=True,
null=True,
)
can_change = fields.BooleanField(default=False)
success_percents = fields.ListField(blank=True, null=True)
last_task = fields.DictField(blank=True, null=True)
trend = fields.FloatField(blank=True, null=True)
week_statistic = fields.DictField(
attribute='week_statistic',
blank=True,
null=True,
readonly=True,
)
day_time_statistic = fields.DictField(
attribute='day_time_statistic',
blank=True,
null=True,
readonly=True,
)
quality_game = fields.DictField(
attribute='quality_game',
blank=True,
null=True,
readonly=True,
)
class Meta:
queryset = Project.objects.all()
authentication = Authentication()
authorization = ProjectsAuthorization()
resource_name = 'projects/project'
detail_uri_name = 'name'
fields = (
'name',
'is_enabled',
'id',
'is_private',
'icon',
'comment_from_owner_account',
'run_here',
)
def dehydrate(self, bundle):
"""Attach token to bundle if owner"""
if (bundle.request.user.is_authenticated()
and bundle.obj.can_change(bundle.request.user)):
bundle.data['can_change'] = True
else:
bundle.data['token'] = ''
self._attach_success_percent(bundle)
self._attach_last_task(bundle)
self._attach_trend(bundle)
return bundle
@attach_field('with_success_percent', 'success_percents')
def _attach_success_percent(self, bundle):
"""Attach success percent"""
return bundle.obj.get_success_percents(
100,
bundle.request.GET.get('branch'),
)
@attach_field('with_last_task', 'last_task')
def _attach_last_task(self, bundle):
"""Attach last task to project"""
try:
return bundle.obj.get_last_task()
except TaskDoesNotExists:
return None
@attach_field('with_trend', 'trend')
def _attach_trend(self, bundle):
"""Attach trend to project"""
return bundle.obj.get_trend(bundle.request.GET.get('branch'))
class TargetResource(ChemblModelResource):
target_chembl_id = fields.CharField('chembl_id')
target_type = fields.CharField('target_type_id')
target_components = fields.ToManyField(
'chembl_webservices.resources.target.TargetComponentsResource',
'targetcomponents_set',
full=True,
null=True,
blank=True)
cross_references = fields.ToManyField(
'chembl_webservices.resources.target.TargetXRefResource',
'targetxref_set',
full=True,
null=True,
blank=True)
score = fields.FloatField('score', use_in='search', null=True, blank=True)
class Meta(ChemblResourceMeta):
queryset = TargetDictionary.objects.all() if 'downgraded' not in available_fields else \
TargetDictionary.objects.filter(downgraded=False)
es_join_column = 'chembl_id'
excludes = ['tid']
resource_name = 'target'
collection_name = 'targets'
serializer = ChEMBLApiSerializer(
resource_name, {
collection_name: resource_name,
'target_components': 'target_component',
'target_component_synonyms': 'target_component_synonym'
})
detail_uri_name = 'chembl_id'
prefetch_related = [
Prefetch('targetcomponents_set',
queryset=TargetComponents.objects.only(
'pk', 'relationship', 'target', 'component')),
Prefetch('targetcomponents_set__component',
queryset=ComponentSequences.objects.only(
'accession', 'pk', 'component_type', 'description')),
Prefetch('targetxref_set',
queryset=TargetXref.objects.only('pk', 'target',
'xref_name', 'xref_id',
'xref_src_db')),
Prefetch('targetxref_set__xref_src_db',
queryset=XrefSource.objects.only('pk')),
Prefetch('targetcomponents_set__component__componentsynonyms_set'),
Prefetch('targetcomponents_set__component__componentxref_set'),
]
fields = (
'organism',
'tax_id',
'pref_name',
'species_group_flag',
'target_chembl_id',
)
filtering = {
'organism': CHAR_FILTERS,
'tax_id': NUMBER_FILTERS,
'pref_name': CHAR_FILTERS,
'target_type': CHAR_FILTERS,
'species_group_flag': FLAG_FILTERS,
'target_chembl_id': ALL,
'target_components': ALL_WITH_RELATIONS,
}
ordering = [
'organism',
'tax_id',
'pref_name',
'target_type',
'species_group_flag',
'target_chembl_id',
]
# ----------------------------------------------------------------------------------------------------------------------
def prepend_urls(self):
"""
Returns a URL scheme based on the default scheme to specify
the response format as a file extension, e.g. /api/v1/users.json
"""
return [
url(r"^(?P<resource_name>%s)/search%s$" %
(self._meta.resource_name, trailing_slash()),
self.wrap_view('get_search'),
name="api_get_search"),
url(r"^(?P<resource_name>%s)/search\.(?P<format>xml|json|jsonp|yaml)$"
% self._meta.resource_name,
self.wrap_view('get_search'),
name="api_get_search"),
url(r"^(?P<resource_name>%s)/datatables\.(?P<format>\w+)$" %
self._meta.resource_name,
self.wrap_view('get_datatables'),
name="api_get_datatables"),
url(r"^(?P<resource_name>%s)\.(?P<format>\w+)$" %
self._meta.resource_name,
self.wrap_view('dispatch_list'),
name="api_dispatch_list"),
url(r"^(?P<resource_name>%s)/schema\.(?P<format>\w+)$" %
self._meta.resource_name,
self.wrap_view('get_schema'),
name="api_get_schema"),
url(r"^(?P<resource_name>%s)/set/(?P<%s_list>\w[\w/;-]*)\.(?P<format>\w+)$"
% (self._meta.resource_name, self._meta.detail_uri_name),
self.wrap_view('get_multiple'),
name="api_get_multiple"),
url(r"^(?P<resource_name>%s)/(?P<%s>\w[\w/-]*)\.(?P<format>\w+)$" %
(self._meta.resource_name, self._meta.detail_uri_name),
self.wrap_view('dispatch_detail'),
name="api_dispatch_detail"),
]
# ----------------------------------------------------------------------------------------------------------------------
def preprocess_filters(self, filters, for_cache_key=False):
ret = {}
for filter_expr, value in list(filters.items()):
filter_bits = filter_expr.split(LOOKUP_SEP)
field_name = filter_bits.pop(0)
if field_name == 'target_synonym':
field_name = 'target_components'
filter_bits = [
'target_component_synonyms', 'component_synonym'
] + filter_bits
ret[LOOKUP_SEP.join([field_name] + filter_bits)] = value
else:
ret[filter_expr] = value
return ret
class SearchResource(ModelResourceWithFieldsFilter):
# Roses to the clever person that makes this introspect the model and
# removes all this code.
absolute_url = fields.CharField(
attribute='absolute_url',
help_text="The URL on CourtListener for the item.",
null=True,
)
case_name = fields.CharField(
attribute='caseName',
help_text="The full name of the case",
null=True,
)
case_number = fields.CharField(
attribute='caseNumber',
help_text="The combination of the citation and the docket number. "
"Only applies to opinion results.",
null=True,
)
citation = fields.CharField(
attribute='citation',
help_text="A concatenated list of all the citations for an opinion. "
"Only applies to opinion results.",
null=True,
)
cite_count = fields.IntegerField(
attribute='citeCount',
help_text="The number of times this document is cited by other cases. "
"Only applies to opinion results.",
null=True,
)
court = fields.CharField(
attribute='court',
help_text="The name of the court where the document was filed.",
null=True,
)
court_id = fields.CharField(
attribute='court_id',
help_text='The id of the court where the document was filed.',
null=True,
)
date_argued = fields.DateField(
attribute='dateArgued',
help_text='The date argued in the court. Only applies to oral '
'argument results.',
null=True,
)
date_filed = fields.DateField(
attribute='dateFiled',
help_text='The date filed by the court. Only applies to opinion '
'results.',
null=True,
)
docket_number = fields.CharField(
attribute='docketNumber',
help_text='The docket numbers of a case, can be consolidated and '
'quite long',
null=True,
)
download_url = fields.CharField(
attribute='download_url',
help_text='The URL on the court website where the item was '
'originally scraped',
null=True,
)
duration = fields.IntegerField(
attribute='duration',
help_text='The length, in seconds, of the mp3 file. Only applies to '
'oral arguments.',
null=True,
)
id = fields.CharField(
attribute='id',
help_text='The primary key for an item.',
)
judge = fields.CharField(
attribute='judge',
help_text='The judges that brought the opinion as a simple text '
'string',
null=True,
)
local_path = fields.CharField(
attribute='local_path',
help_text='The location, relative to MEDIA_ROOT on the CourtListener '
'server, where files are stored.',
null=True,
)
score = fields.FloatField(
attribute='score',
help_text='The relevance of the result. Will vary from query to '
'query.',
)
source = fields.CharField(
attribute='source',
help_text='the source of the document, one of: %s' %
', '.join(['%s (%s)' % (t[0], t[1]) for t in SOURCES]),
null=True,
)
snippet = fields.CharField(
attribute='snippet',
help_text='a snippet as found in search results, utilizing <mark> for '
'highlighting and … for ellipses.',
null=True,
)
status = fields.CharField(
attribute='status',
help_text='The precedential status of document, one of: %s. Only '
'applies to opinion results.' %
', '.join([('stat_%s' % t[1]).replace(' ', '+')
for t in DOCUMENT_STATUSES]),
null=True,
)
suit_nature = fields.CharField(
attribute='suitNature',
help_text="The nature of the suit. For the moment can be codes or "
"laws or whatever. Only applies to opinion results.",
null=True,
)
text = fields.CharField(
attribute='text',
use_in='detail', # Only shows on the detail page.
help_text="A concatenated copy of most fields in the item so those "
"fields are available for search. Only applies to opinion "
"results.")
timestamp = fields.DateField(
attribute='timestamp',
help_text='The moment when an item was indexed by Solr.')
class Meta:
authentication = authentication.MultiAuthentication(
BasicAuthenticationWithUser(realm="courtlistener.com"),
authentication.SessionAuthentication())
throttle = PerUserCacheThrottle(throttle_at=1000)
resource_name = 'search'
max_limit = 20
include_absolute_url = True
allowed_methods = ['get']
filtering = {
'q': ('search', ),
'case_name': ('search', ),
'judge': ('search', ),
'stat_': ('boolean', ),
'filed_after': ('date', ),
'filed_before': ('date', ),
'argued_after': ('date', ),
'argued_before': ('date', ),
'citation': ('search', ),
'neutral_cite': ('search', ),
'docket_number': ('search', ),
'cited_gt': ('int', ),
'cited_lt': ('int', ),
'court': ('csv', ),
'type': ('search', ),
}
ordering = [
'dateFiled+desc',
'dateFiled+asc',
'dateArgued+desc',
'dateArgued+asc',
'citeCount+desc',
'citeCount+asc',
'score+desc',
]
def get_resource_uri(self,
bundle_or_obj=None,
url_name='api_dispatch_list'):
"""Creates a URI like /api/v1/search/$id/
"""
url_str = '/api/rest/%s/%s/%s/'
if bundle_or_obj:
return url_str % (
self.api_name,
'document',
bundle_or_obj.obj.id,
)
else:
return ''
def get_object_list(self, request=None, **kwargs):
"""Performs the Solr work."""
main_query = {'caller': 'api_search'}
try:
main_query.update(build_main_query(kwargs['cd'], highlight='text'))
sl = SolrList(main_query=main_query,
offset=request.GET.get('offset', 0),
limit=request.GET.get('limit', 20),
type=kwargs['cd']['type'])
except KeyError:
sf = forms.SearchForm({'q': "*:*"})
if sf.is_valid():
main_query.update(
build_main_query(sf.cleaned_data, highlight='text'))
sl = SolrList(
main_query=main_query,
offset=request.GET.get('offset', 0),
limit=request.GET.get('limit', 20),
)
return sl
def obj_get_list(self, bundle, **kwargs):
search_form = forms.SearchForm(bundle.request.GET)
if search_form.is_valid():
cd = search_form.cleaned_data
if cd['q'] == '':
cd['q'] = '*:*' # Get everything.
return self.get_object_list(bundle.request, cd=cd)
else:
BadRequest("Invalid resource lookup data provided. Unable to "
"complete your query.")
def obj_get(self, bundle, **kwargs):
search_form = forms.SearchForm(bundle.request.GET)
if search_form.is_valid():
cd = search_form.cleaned_data
cd['q'] = 'id:%s' % kwargs['pk']
return self.get_object_list(bundle.request, cd=cd)[0]
else:
BadRequest("Invalid resource lookup data provided. Unable to "
"complete your request.")
def apply_sorting(self, obj_list, options=None):
"""Since we're not using Django Model sorting, we just want to use
our own, which is already passed into the search form anyway.
Thus: Do nothing here.
"""
return obj_list
class ChemblIdLookupResource(ChemblModelResource):
resource_url = fields.CharField()
score = fields.FloatField('score', use_in='search', null=True, blank=True)
class Meta(ChemblResourceMeta):
queryset = ChemblIdLookup.objects.all()
es_join_column = 'chembl_id'
es_multi_index_search_resources = [
'molecule', 'document', 'assay', 'target'
]
resource_name = 'chembl_id_lookup'
collection_name = 'chembl_id_lookups'
serializer = ChEMBLApiSerializer(resource_name,
{collection_name: resource_name})
fields = (
'chembl_id',
'entity_type',
'status',
)
filtering = {
'chembl_id': CHAR_FILTERS,
'entity_type': CHAR_FILTERS,
'status': CHAR_FILTERS,
}
ordering = [
field for field in list(filtering.keys())
if not ('comment' in field or 'description' in field)
]
# ----------------------------------------------------------------------------------------------------------------------
def alter_list_data_to_serialize(self, request, data):
"""
A hook to alter list data just before it gets serialized & sent to the user.
Useful for restructuring/renaming aspects of the what's going to be
sent.
Should accommodate for a list of objects, generally also including
meta data.
"""
bundles = data['chembl_id_lookups']
for idx, bundle in enumerate(bundles):
bundles[idx] = self.alter_detail_data_to_serialize(request, bundle)
return data
# ----------------------------------------------------------------------------------------------------------------------
def alter_detail_data_to_serialize(self, request, bundle):
"""
A hook to alter detail data just before it gets serialized & sent to the user.
Useful for restructuring/renaming aspects of the what's going to be
sent.
Should accommodate for receiving a single bundle of data.
"""
datas = bundle.data
detail_name = 'api_dispatch_detail'
if datas.get('entity_type'):
if datas['entity_type'] == 'COMPOUND':
datas['resource_url'] = molecule._build_reverse_url(
detail_name, kwargs=molecule.resource_uri_kwargs(bundle))
elif datas['entity_type'] == 'ASSAY':
datas['resource_url'] = assay._build_reverse_url(
detail_name, kwargs=assay.resource_uri_kwargs(bundle))
elif datas['entity_type'] == 'TARGET':
datas['resource_url'] = target._build_reverse_url(
detail_name, kwargs=target.resource_uri_kwargs(bundle))
elif datas['entity_type'] == 'DOCUMENT':
datas['resource_url'] = document._build_reverse_url(
detail_name, kwargs=document.resource_uri_kwargs(bundle))
elif datas['entity_type'] == 'CELL':
kw = cell.resource_uri_kwargs(bundle)
kw['cell_chembl_id'] = kw['pk']
del kw['pk']
datas['resource_url'] = cell._build_reverse_url(detail_name,
kwargs=kw)
return bundle
# ----------------------------------------------------------------------------------------------------------------------
def prepend_urls(self):
"""
Returns a URL scheme based on the default scheme to specify
the response format as a file extension, e.g. /api/v1/users.json
"""
return [
url(r"^(?P<resource_name>%s)/search%s$" %
(self._meta.resource_name, trailing_slash()),
self.wrap_view('get_search'),
name="api_get_search"),
url(r"^(?P<resource_name>%s)/search\.(?P<format>xml|json|jsonp|yaml)$"
% self._meta.resource_name,
self.wrap_view('get_search'),
name="api_get_search"),
url(r"^(?P<resource_name>%s)\.(?P<format>\w+)$" %
self._meta.resource_name,
self.wrap_view('dispatch_list'),
name="api_dispatch_list"),
url(r"^(?P<resource_name>%s)/schema\.(?P<format>\w+)$" %
self._meta.resource_name,
self.wrap_view('get_schema'),
name="api_get_schema"),
url(r"^(?P<resource_name>%s)/datatables\.(?P<format>\w+)$" %
self._meta.resource_name,
self.wrap_view('get_datatables'),
name="api_get_datatables"),
url(r"^(?P<resource_name>%s)/set/(?P<%s_list>\w[\w/;-]*)\.(?P<format>\w+)$"
% (self._meta.resource_name, self._meta.detail_uri_name),
self.wrap_view('get_multiple'),
name="api_get_multiple"),
url(r"^(?P<resource_name>%s)/(?P<%s>\w[\w/-]*)\.(?P<format>\w+)$" %
(self._meta.resource_name, self._meta.detail_uri_name),
self.wrap_view('dispatch_detail'),
name="api_dispatch_detail"),
]
class JobDuration(StatisticsResource):
"""
Histograms of total job duration (from submission to completion)
for completed jobs and for error jobs
"""
status = fields.CharField(attribute="status")
platform = fields.CharField(attribute="platform")
values = fields.ListField(attribute="values")
bins = fields.ListField(attribute="bins")
max = fields.FloatField(attribute="max")
class Meta:
resource_name = "statistics/job-duration"
list_allowed_methods = ['get']
detail_allowed_methods = []
def get_object_list(self, request):
n_bins = int(request.GET.get("bins", 50))
scale = request.GET.get("scale", "linear")
requested_max = request.GET.get("max", None)
all_jobs = Job.objects.annotate(
duration=ExpressionWrapper(F('timestamp_completion') -
F('timestamp_submission'),
output_field=DurationField()))
job_durations = []
for status in ("finished", "error"):
for platform in STANDARD_QUEUES:
durations = [
x['duration'].total_seconds() for x in all_jobs.filter(
status=status, hardware_platform=platform).values(
'duration') if x['duration'] is not None
]
durations = np.array(durations)
negative_durations = (durations < 0)
if negative_durations.any():
n_neg = negative_durations.sum()
logger.warning(
"There were {} negative durations ({}%) for status={} and platform={}"
.format(n_neg, 100 * n_neg / durations.size, status,
platform))
durations = durations[~negative_durations]
if durations.size > 0:
if requested_max is None:
max = (durations.max() // n_bins + 1) * n_bins
else:
max = float(requested_max)
if scale == "log":
log_bins = np.linspace(0, np.ceil(np.log10(max)),
n_bins)
values = np.histogram(np.log10(durations),
bins=log_bins)[0]
#bins = np.power(10, log_bins)
bins = log_bins
else: # linear, whatever the value of `scale`
values, bins = np.histogram(durations,
bins=n_bins,
range=(0, max))
job_durations.append(
Histogram(platform=platform,
status=status,
values=values.tolist(),
bins=bins,
scale=scale,
max=max))
return job_durations
class MoleculeResource(ChemblModelResource):
molecule_chembl_id = fields.CharField('chembl_id')
molecule_properties = fields.ForeignKey(
'chembl_webservices.resources.molecule.MoleculePropertiesResource',
'compoundproperties',
full=True,
null=True,
blank=True)
molecule_hierarchy = fields.ForeignKey(
'chembl_webservices.resources.molecule.MoleculeHierarchyResource',
'moleculehierarchy',
full=True,
null=True,
blank=True)
molecule_structures = fields.ForeignKey(
'chembl_webservices.resources.molecule.MoleculeStructuresResource',
'compoundstructures',
full=True,
null=True,
blank=True)
molecule_synonyms = fields.ToManyField(
'chembl_webservices.resources.molecule.MoleculeSynonymsResource',
'moleculesynonyms_set',
full=True,
null=True,
blank=True)
helm_notation = fields.CharField('biotherapeutics__helm_notation',
null=True,
blank=True)
biotherapeutic = fields.ForeignKey(
'chembl_webservices.resources.bio_component.BiotherapeuticComponentsResource',
'biotherapeutics',
full=True,
null=True,
blank=True)
atc_classifications = fields.ToManyField(
'chembl_webservices.resources.atc.AtcResource',
'atcclassification_set',
full=False,
null=True,
blank=True)
cross_references = fields.ToManyField(
'chembl_webservices.resources.molecule.MoleculeXRefResource',
'compoundxref_set',
full=True,
null=True,
blank=True)
score = fields.FloatField('score', use_in='search', null=True, blank=True)
class Meta(ChemblResourceMeta):
queryset = MoleculeDictionary.objects.all() if 'downgraded' not in available_fields else \
MoleculeDictionary.objects.exclude(downgraded=True)
es_join_column = 'chembl_id'
excludes = ['molregno']
resource_name = 'molecule'
collection_name = 'molecules'
description = {
'api_dispatch_list':
'''
Retrieve list of molecules. Apart from the standard set of relation types, there is one specific operator:
* __flexmatch__ \- matches _SMILES_ with the same structure, as opposed to exact match, for example:
[`COc1ccc2[C@@H]3[C@H](COc2c1)C(C)(C)OC4=C3C(=O)C(=O)C5=C4OC`
`(C)(C)[C@H]6COc7cc(OC)ccc7[C@@H]56`](https://www.ebi.ac.uk/chembl/api/data/molecule?molecule_structures__canonical_smiles__flexmatch=COc1ccc2[C@@H]3[C@H]%28COc2c1%29C%28C%29%28C%29OC4=C3C%28=O%29C%28=O%29C5=C4OC%28C%29%28C%29[C@H]6COc7cc%28OC%29ccc7[C@@H]56 "Example")
will match two molecules with:
* `COc1ccc2[C@@H]3[C@H](COc2c1)C(C)(C)OC4=C3C(=O)C(=O)C5=C4OC`
`(C)(C)[C@H]6COc7cc(OC)ccc7[C@@H]56` and
* `COc1ccc2[C@@H]3[C@H](COc2c1)C(C)(C)OC4=C3C(=O)C(=O)C5=C4OC`
`(C)(C)[C@@H]6COc7cc(OC)ccc7[C@H]56`
_SMILES_.
'''
}
serializer = MoleculeSerializer(
resource_name, {
collection_name: resource_name,
'biocomponents': 'biocomponent',
'molecule_synonyms': 'synonym',
'atc_classifications': 'level5'
})
detail_uri_name = 'chembl_id'
prefetch_related = [
Prefetch('moleculesynonyms_set'),
Prefetch('atcclassification_set'),
Prefetch('biotherapeutics__bio_component_sequences'),
Prefetch('compoundproperties'),
Prefetch('moleculehierarchy'),
Prefetch('compoundstructures'),
Prefetch('compoundxref_set',
queryset=CompoundXref.objects.only(
'pk', 'xref_name', 'xref_id', 'xref_src_db',
'molecule')),
Prefetch('compoundxref_set__xref_src_db',
queryset=XrefSource.objects.only('pk')),
Prefetch('moleculehierarchy__parent_molecule',
queryset=MoleculeDictionary.objects.only('chembl')),
]
fields = (
'atc_classifications',
'availability_type',
'biotherapeutic',
'black_box_warning',
'chebi_par_id',
'chirality',
'cross_references',
'dosed_ingredient',
'first_approval',
'first_in_class',
'helm_notation',
'indication_class',
'inorganic_flag',
'max_phase',
'molecule_chembl_id',
'molecule_hierarchy',
'molecule_properties',
'molecule_structures',
'molecule_type',
'natural_product',
'oral',
'parenteral',
'polymer_flag',
'pref_name',
'prodrug',
'structure_type',
'therapeutic_flag',
'topical',
'usan_stem',
'usan_stem_definition',
'usan_substem',
'usan_year',
'withdrawn_flag',
'withdrawn_year',
'withdrawn_country',
'withdrawn_reason',
'withdrawn_class',
)
filtering = {
'availability_type': CHAR_FILTERS,
'biotherapeutic': ALL_WITH_RELATIONS,
'molecule_synonyms': ALL_WITH_RELATIONS,
'black_box_warning': FLAG_FILTERS,
'chebi_par_id': NUMBER_FILTERS,
'chirality': NUMBER_FILTERS,
'dosed_ingredient': FLAG_FILTERS,
'first_approval': NUMBER_FILTERS,
'first_in_class': FLAG_FILTERS,
'indication_class': CHAR_FILTERS,
'inorganic_flag': FLAG_FILTERS,
'helm_notation': CHAR_FILTERS,
'max_phase': NUMBER_FILTERS,
'molecule_chembl_id': ALL,
'atc_classifications': ALL_WITH_RELATIONS,
'cross_references': ALL_WITH_RELATIONS,
'molecule_hierarchy': ALL_WITH_RELATIONS,
'molecule_properties': ALL_WITH_RELATIONS,
'molecule_structures': ALL_WITH_RELATIONS,
'molecule_type': CHAR_FILTERS,
'natural_product': FLAG_FILTERS,
'oral': FLAG_FILTERS,
'parenteral': FLAG_FILTERS,
'polymer_flag': FLAG_FILTERS,
'pref_name': CHAR_FILTERS,
'prodrug': FLAG_FILTERS,
'structure_type': CHAR_FILTERS,
'therapeutic_flag': FLAG_FILTERS,
'topical': FLAG_FILTERS,
'usan_stem': CHAR_FILTERS,
'usan_stem_definition': CHAR_FILTERS,
'usan_substem': CHAR_FILTERS,
'usan_year': NUMBER_FILTERS,
'withdrawn_flag': FLAG_FILTERS,
'withdrawn_year': NUMBER_FILTERS,
'withdrawn_country': CHAR_FILTERS,
'withdrawn_reason': CHAR_FILTERS,
'withdrawn_class': CHAR_FILTERS,
}
ordering = [
'availability_type',
'biotherapeutic',
'black_box_warning',
'chebi_par_id',
'chirality',
'dosed_ingredient',
'first_approval',
'first_in_class',
'indication_class',
'inorganic_flag',
'helm_notation',
'max_phase',
'molecule_chembl_id',
'molecule_hierarchy',
'molecule_properties',
'molecule_structures',
'molecule_type',
'natural_product',
'oral',
'parenteral',
'polymer_flag',
'pref_name',
'prodrug',
'structure_type',
'therapeutic_flag',
'topical',
'usan_stem',
'usan_stem_definition',
'usan_substem',
'usan_year',
'withdrawn_flag',
'withdrawn_year',
'withdrawn_country',
'withdrawn_reason',
'withdrawn_class',
]
# ----------------------------------------------------------------------------------------------------------------------
def base_urls(self):
return [
url(r"^(?P<resource_name>%s)/search%s$" %
(self._meta.resource_name, trailing_slash()),
self.wrap_view('get_search'),
name="api_get_search"),
url(r"^(?P<resource_name>%s)/search\.(?P<format>xml|json|jsonp|yaml|sdf|mol)$"
% self._meta.resource_name,
self.wrap_view('get_search'),
name="api_get_search"),
url(r"^(?P<resource_name>%s)%s$" %
(self._meta.resource_name, trailing_slash()),
self.wrap_view('dispatch_list'),
name="api_dispatch_list"),
url(r"^(?P<resource_name>%s)\.(?P<format>xml|json|jsonp|yaml|sdf|mol)$"
% self._meta.resource_name,
self.wrap_view('dispatch_list'),
name="api_dispatch_list"),
url(r"^(?P<resource_name>%s)/schema%s$" %
(self._meta.resource_name, trailing_slash()),
self.wrap_view('get_schema'),
name="api_get_schema"),
url(r"^(?P<resource_name>%s)/schema\.(?P<format>xml|json|jsonp|yaml|sdf|mol)$"
% self._meta.resource_name,
self.wrap_view('get_schema'),
name="api_get_schema"),
url(r"^(?P<resource_name>%s)/datatables\.(?P<format>\w+)$" %
self._meta.resource_name,
self.wrap_view('get_datatables'),
name="api_get_datatables"),
url(r"^(?P<resource_name>%s)/set/(?P<chembl_id_list>[Cc][Hh][Ee][Mm][Bb][Ll]\d[\d]*(;[Cc][Hh][Ee][Mm][Bb][Ll]\d[\d]*)*)%s$"
% (self._meta.resource_name, trailing_slash()),
self.wrap_view('get_multiple'),
name="api_get_multiple"),
url(r"^(?P<resource_name>%s)/set/(?P<chembl_id_list>[Cc][Hh][Ee][Mm][Bb][Ll]\d[\d]*(;[Cc][Hh][Ee][Mm][Bb][Ll]\d[\d]*)*)\.(?P<format>xml|json|jsonp|yaml|sdf|mol)$"
% self._meta.resource_name,
self.wrap_view('get_multiple'),
name="api_get_multiple"),
url(r"^(?P<resource_name>%s)/set/(?P<molecule_structures__standard_inchi_key_list>[A-Z]{14}-[A-Z]{10}-[A-Z](;[A-Z]{14}-[A-Z]{10}-[A-Z])*)%s$"
% (self._meta.resource_name, trailing_slash()),
self.wrap_view('get_multiple'),
name="api_get_multiple"),
url(r"^(?P<resource_name>%s)/set/(?P<molecule_structures__standard_inchi_key_list>[A-Z]{14}-[A-Z]{10}-[A-Z](;[A-Z]{14}-[A-Z]{10}-[A-Z])*)\.(?P<format>xml|json|jsonp|yaml|sdf|mol)$"
% self._meta.resource_name,
self.wrap_view('get_multiple'),
name="api_get_multiple"),
url(r"^(?P<resource_name>%s)/set/(?P<molecule_structures__canonical_smiles_list>[^jx]+(;[^jx]+)*)%s$"
% (self._meta.resource_name, trailing_slash()),
self.wrap_view('get_multiple'),
name="api_get_multiple"),
url(r"^(?P<resource_name>%s)/set/(?P<molecule_structures__canonical_smiles_list>[^jx]+(;[^jx]+)*)\.(?P<format>xml|json|jsonp|yaml|sdf|mol)$"
% self._meta.resource_name,
self.wrap_view('get_multiple'),
name="api_get_multiple"),
url(r"^(?P<resource_name>%s)/(?P<chembl_id>[Cc][Hh][Ee][Mm][Bb][Ll]\d[\d]*)\.(?P<format>xml|json|jsonp|yaml|sdf|mol)$"
% self._meta.resource_name,
self.wrap_view('dispatch_detail'),
name="api_dispatch_detail"),
url(r"^(?P<resource_name>%s)/(?P<chembl_id>[Cc][Hh][Ee][Mm][Bb][Ll]\d[\d]*)%s$"
% (self._meta.resource_name, trailing_slash()),
self.wrap_view('dispatch_detail'),
name="api_dispatch_detail"),
url(r"^(?P<resource_name>%s)/(?P<molecule_structures__standard_inchi_key>[A-Z]{14}-[A-Z]{10}-[A-Z])\.(?P<format>xml|json|jsonp|yaml|sdf|mol)$"
% self._meta.resource_name,
self.wrap_view('dispatch_detail'),
name="api_dispatch_detail"),
url(r"^(?P<resource_name>%s)/(?P<molecule_structures__standard_inchi_key>[A-Z]{14}-[A-Z]{10}-[A-Z])%s$"
% (self._meta.resource_name, trailing_slash()),
self.wrap_view('dispatch_detail'),
name="api_dispatch_detail"),
url(r"^(?P<resource_name>%s)/(?P<molecule_structures__canonical_smiles>[^jx]+)\.(?P<format>xml|json|jsonp|yaml|sdf|mol)$"
% self._meta.resource_name,
self.wrap_view('dispatch_detail'),
name="api_dispatch_detail"),
url(r"^(?P<resource_name>%s)/(?P<molecule_structures__canonical_smiles>[^jx]+)%s$"
% (self._meta.resource_name, trailing_slash()),
self.wrap_view('dispatch_detail'),
name="api_dispatch_detail"),
]
# ----------------------------------------------------------------------------------------------------------------------
def prepend_urls(self):
return []
# ----------------------------------------------------------------------------------------------------------------------
def get_multiple(self, request, **kwargs):
"""
Returns a serialized list of resources based on the identifiers
from the URL.
Calls ``obj_get`` to fetch only the objects requested. This method
only responds to HTTP GET.
Should return a HttpResponse (200 OK).
"""
self.method_check(request, allowed=['get'])
self.is_authenticated(request)
self.throttle_check(request)
detail_uri_name = None
obj_identifiers = None
for key, value in list(kwargs.items()):
if key.endswith('_list'):
detail_uri_name = key.split('_list')[0]
obj_identifiers = value.split(';')
break
objects = []
not_found = []
base_bundle = self.build_bundle(request=request)
for identifier in obj_identifiers:
try:
obj, _ = self.cached_obj_get(bundle=base_bundle,
**{detail_uri_name: identifier})
bundle = self.build_bundle(obj=obj, request=request)
bundle = self.full_dehydrate(bundle, for_list=True, **kwargs)
objects.append(bundle)
except (ObjectDoesNotExist, Unauthorized):
not_found.append(identifier)
object_list = {
self._meta.collection_name: objects,
}
if len(not_found):
object_list['not_found'] = not_found
self.log_throttled_access(request)
return self.create_response(request, object_list)
# ----------------------------------------------------------------------------------------------------------------------
def decode_plus(self, kwargs):
return {
k: v.replace(' ', '+') if
(isinstance(k, str)
and k.startswith('molecule_structures__canonical_smiles')) else v
for k, v in list(kwargs.items())
}
# ----------------------------------------------------------------------------------------------------------------------
def remove_api_resource_names(self, kwargs):
aliases = {'smiles': 'molecule_structures__canonical_smiles'}
for alias, name in list(aliases.items()):
if alias in kwargs:
kwargs[name] = kwargs.pop(alias)
decoded_kwargs = self.decode_plus(kwargs)
return super(MoleculeResource,
self).remove_api_resource_names(decoded_kwargs)
# ----------------------------------------------------------------------------------------------------------------------
def alter_list_data_to_serialize(self, request, data):
bundles = data['molecules']
for idx, bundle in enumerate(bundles):
bundles[idx] = self.alter_detail_data_to_serialize(request, bundle)
return data
# ----------------------------------------------------------------------------------------------------------------------
def alter_detail_data_to_serialize(self, request, data):
if 'molecule_structures' in data.data:
mol_struts = data.data['molecule_structures']
sdf_style = request.format != 'mol'
if mol_struts is not None and mol_struts.data is not None and sdf_style:
mol_struts.data[
'molfile'] += '\n\n> <chembl_id>\n{0}\n\n'.format(
data.data.get('molecule_chembl_id'))
mol_struts.data[
'molfile'] += '> <chembl_pref_name>\n{0}\n\n'.format(
data.data.get('pref_name', 'undefined'))
if 'atc_classifications' in data.data:
atc = data.data['atc_classifications']
for idx, item in enumerate(atc):
atc[idx] = item.split('/')[-1]
return data
# ----------------------------------------------------------------------------------------------------------------------
def elastic_search_flexmatch(self, smiles):
smiles_2_inchi_key_url = settings.BEAKER_URL + '/smiles2inchiKey'
inchikey_resp = requests.post(smiles_2_inchi_key_url, data=smiles)
if inchikey_resp.status_code != 200 or inchikey_resp.text.strip(
) == "":
raise ImmediateHttpResponse(response=http.HttpBadRequest(
'ERROR: could not generate an inchi key for smiles.\n{}'.
format(smiles)))
connectivity_layer = inchikey_resp.text.split('-')[0]
idx_name = (settings.ELASTICSEARCH_INDEXES_PREFIX +
'{0}').format('molecule')
try:
es_conn = get_es_connection()
es_query = \
{
'track_total_hits': True,
'_source': False,
'query': {
'query_string': {
'query': '_metadata.hierarchy.family_inchi_connectivity_layer:{}'.format(connectivity_layer)
}
}
}
search_results = elasticsearch.helpers.scan(es_conn,
query=es_query,
index=idx_name,
preserve_order=True,
size=1000)
mol_chembl_ids = []
for result_i in search_results:
chembl_id = result_i['_id']
mol_chembl_ids.append(chembl_id)
return mol_chembl_ids
except:
self.log.error('Searching exception',
exc_info=True,
extra={
'user_query': connectivity_layer,
})
raise RuntimeError(
'Could not execute query on elasticsearch engine at {0}'.
format(settings.ELASTICSEARCH_CONNECTION_URL))
# ----------------------------------------------------------------------------------------------------------------------
def preprocess_filters(self, filters, for_cache_key=False):
ret = {}
for filter_expr, value in list(filters.items()):
filter_bits = filter_expr.split(LOOKUP_SEP)
if filter_expr == 'similarity':
continue
if len(filter_bits) and filter_bits[-1] == 'flexmatch':
if not value.strip():
raise BadRequest("Input string is empty")
if value.upper().startswith('CHEMBL'):
try:
mol = MoleculeDictionary.objects.get(
chembl_id=value.upper())
smiles = mol.compoundstructures.canonical_smiles
value = smiles
except ObjectDoesNotExist:
raise ImmediateHttpResponse(
response=http.HttpNotFound())
if not for_cache_key:
molecule_chembl_ids_flexmatch = self.elastic_search_flexmatch(
value)
ret["molecule_chembl_id__in"] = molecule_chembl_ids_flexmatch
continue
ret[filter_expr] = value
return ret
# ----------------------------------------------------------------------------------------------------------------------
def _get_cache_args(self, *args, **kwargs):
cache_ordered_dict = super(MoleculeResource,
self)._get_cache_args(*args, **kwargs)
smooshed = []
filters, _ = self.build_filters(kwargs, for_cache_key=True)
for key, value in list(filters.items()):
smooshed.append("%s=%s" % (key, value))
cache_ordered_dict['filters'] = '|'.join(sorted(smooshed))
return cache_ordered_dict
class StatisticsResource(BaseModelResource):
account_id = fields.IntegerField('account_id')
handle = fields.CharField('account__key')
contest_id = fields.IntegerField('contest_id')
event = fields.CharField('contest__title')
date = fields.DateTimeField('contest__end_time')
coder_id = fields.IntegerField()
place = fields.IntegerField('place_as_int', null=True)
score = fields.FloatField('solving')
new_rating = fields.IntegerField('new_rating', null=True)
old_rating = fields.IntegerField('old_rating', null=True)
rating_change = fields.IntegerField('rating_change', null=True)
total_count = fields.BooleanField()
class Meta(BaseModelResource.Meta):
abstract = False
queryset = Statistics.objects.all()
resource_name = 'statistics'
excludes = ('total_count', )
filtering = {
'total_count': ['exact'],
'contest_id': ['exact'],
'account_id': ['exact'],
'coder_id': ['exact'],
'place': ['exact', 'isnull'],
'new_rating': ['isnull'],
'rating_change': ['isnull'],
}
ordering = ['score', 'place', 'new_rating', 'rating_change', 'date']
detail_allowed_methods = []
def build_filters(self, filters=None, *args, **kwargs):
filters = filters or {}
tmp = {}
for k in 'new_rating__isnull', 'rating_change__isnull', 'coder_id':
tmp[k] = filters.pop(k, None)
filters = super().build_filters(filters, *args, **kwargs)
filters.update(tmp)
return filters
def apply_filters(self, request, applicable_filters):
one_of = ['contest_id__exact', 'account_id__exact', 'coder_id']
for k in one_of:
if applicable_filters.get(f'{k}'):
break
else:
raise BadRequest(
f'One of {[k.split("__")[0] for k in one_of]} is required')
rating_change_isnull = applicable_filters.pop('rating_change__isnull',
None)
new_rating_isnull = applicable_filters.pop('new_rating__isnull', None)
coder_id = applicable_filters.pop('coder_id', None)
qs = super().apply_filters(request, applicable_filters)
qs = qs.select_related('account', 'contest')
if rating_change_isnull:
qs = qs.filter(
addition__rating_change__isnull=rating_change_isnull[0] in
['true', '1', 'yes'])
if new_rating_isnull:
qs = qs.filter(addition__new_rating__isnull=new_rating_isnull[0] in
['true', '1', 'yes'])
if coder_id:
qs = qs.filter(account__coders=coder_id[0])
qs = qs \
.annotate(new_rating=Cast(KeyTextTransform('new_rating', 'addition'), IntegerField())) \
.annotate(old_rating=Cast(KeyTextTransform('old_rating', 'addition'), IntegerField())) \
.annotate(rating_change=Cast(KeyTextTransform('rating_change', 'addition'), IntegerField()))
return qs
def dehydrate(self, *args, **kwargs):
bundle = super().dehydrate(*args, **kwargs)
bundle.data.pop('coder_id', None)
return bundle
class AssayResource(ChemblModelResource):
assay_chembl_id = fields.CharField('chembl_id', null=True, blank=True)
document_chembl_id = fields.CharField('doc__chembl_id',
null=True,
blank=True)
target_chembl_id = fields.CharField('target__chembl_id',
null=True,
blank=True)
tissue_chembl_id = fields.CharField('tissue__chembl_id',
null=True,
blank=True)
cell_chembl_id = fields.CharField('cell__chembl_id', null=True, blank=True)
assay_type = fields.CharField('assay_type__assay_type',
null=True,
blank=True)
assay_type_description = fields.CharField('assay_type__assay_desc',
null=True,
blank=True)
relationship_type = fields.CharField(
'relationship_type__relationship_type', null=True, blank=True)
relationship_description = fields.CharField(
'relationship_type__relationship_desc', null=True, blank=True)
confidence_score = fields.IntegerField(
'confidence_score__confidence_score', null=True, blank=True)
confidence_description = fields.CharField('confidence_score__description',
null=True,
blank=True)
src_id = fields.IntegerField('src_id', null=True, blank=True)
bao_format = fields.CharField('bao_format_id', null=True, blank=True)
bao_label = fields.CharField('bao_format__label', null=True, blank=True)
score = fields.FloatField('score', use_in='search', null=True, blank=True)
assay_classifications = fields.ToManyField(
'chembl_webservices.resources.assays.AssayClassResource',
'assayclassification_set',
full=True,
null=True,
blank=True)
assay_parameters = fields.ToManyField(
'chembl_webservices.resources.assays.AssayParametersResource',
'assayparameters_set',
full=True,
null=True,
blank=True)
variant_sequence = fields.ForeignKey(
'chembl_webservices.resources.assays.VariantSequenceResource',
'variant',
full=True,
null=True,
blank=True)
class Meta(ChemblResourceMeta):
queryset = Assays.objects.all()
es_join_column = 'chembl_id'
excludes = ['assay_id']
resource_name = 'assay'
collection_name = 'assays'
detail_uri_name = 'chembl_id'
serializer = ChEMBLApiSerializer(
resource_name, {
collection_name: resource_name,
'assay_classifications': 'assay_class',
'assay_parameters': 'assay_parameters'
})
prefetch_related = [
Prefetch('assay_type',
queryset=AssayType.objects.only('assay_type',
'assay_desc')),
Prefetch('cell',
queryset=CellDictionary.objects.only('chembl_id')),
Prefetch('confidence_score',
queryset=ConfidenceScoreLookup.objects.only(
'confidence_score', 'description')),
Prefetch('doc', queryset=Docs.objects.only('chembl_id')),
Prefetch('relationship_type',
queryset=RelationshipType.objects.only(
'relationship_type', 'relationship_desc')),
Prefetch('src', queryset=Source.objects.only('src_id')),
Prefetch('target',
queryset=TargetDictionary.objects.only('chembl_id')),
Prefetch('tissue',
queryset=TissueDictionary.objects.only('chembl_id')),
Prefetch('bao_format',
queryset=BioassayOntology.objects.only('bao_id',
'label')),
Prefetch('assayclassification_set'),
Prefetch('assayparameters_set'),
Prefetch('variant'),
]
fields = (
'assay_classifications',
'assay_category',
'assay_cell_type',
'assay_chembl_id',
'assay_organism',
'assay_strain',
'assay_subcellular_fraction',
'assay_tax_id',
'assay_test_type',
'assay_tissue',
'assay_type',
'assay_type_description',
'bao_format',
'cell_chembl_id',
'confidence_description',
'confidence_score',
'description',
'document_chembl_id',
'relationship_description',
'relationship_type',
'src_assay_id',
'src_id',
'target_chembl_id',
'variant_sequence',
)
filtering = {
'assay_parameters': ALL_WITH_RELATIONS,
'assay_classifications': ALL_WITH_RELATIONS,
'assay_category': CHAR_FILTERS,
'assay_cell_type': CHAR_FILTERS,
'assay_chembl_id': ALL,
'assay_organism': CHAR_FILTERS,
'assay_strain': CHAR_FILTERS,
'assay_subcellular_fraction': CHAR_FILTERS,
'assay_tax_id': NUMBER_FILTERS,
'assay_test_type': CHAR_FILTERS,
'assay_tissue': CHAR_FILTERS,
'assay_type': CHAR_FILTERS,
# 'assay_type_description': ALL,
'bao_format': ALL,
'cell_chembl_id': CHAR_FILTERS,
# 'confidence_description': ALL,
'confidence_score': NUMBER_FILTERS,
'description': CHAR_FILTERS, #TODO: remove from ordering
'document_chembl_id': ALL,
# 'relationship_description': ALL,
'relationship_type': CHAR_FILTERS,
'src_assay_id': NUMBER_FILTERS,
'src_id': NUMBER_FILTERS,
'target_chembl_id': ALL,
'variant_sequence': ALL_WITH_RELATIONS
}
ordering = [
field for field in list(filtering.keys())
if not ('comment' in field or 'description' in field)
]
# ----------------------------------------------------------------------------------------------------------------------
def prepend_urls(self):
"""
Returns a URL scheme based on the default scheme to specify
the response format as a file extension, e.g. /api/v1/users.json
"""
return [
url(r"^(?P<resource_name>%s)/search%s$" %
(self._meta.resource_name, trailing_slash()),
self.wrap_view('get_search'),
name="api_get_search"),
url(r"^(?P<resource_name>%s)/search\.(?P<format>xml|json|jsonp|yaml)$"
% self._meta.resource_name,
self.wrap_view('get_search'),
name="api_get_search"),
url(r"^(?P<resource_name>%s)\.(?P<format>\w+)$" %
self._meta.resource_name,
self.wrap_view('dispatch_list'),
name="api_dispatch_list"),
url(r"^(?P<resource_name>%s)/schema\.(?P<format>\w+)$" %
self._meta.resource_name,
self.wrap_view('get_schema'),
name="api_get_schema"),
url(r"^(?P<resource_name>%s)/datatables\.(?P<format>\w+)$" %
self._meta.resource_name,
self.wrap_view('get_datatables'),
name="api_get_datatables"),
url(r"^(?P<resource_name>%s)/set/(?P<%s_list>\w[\w/;-]*)\.(?P<format>\w+)$"
% (self._meta.resource_name, self._meta.detail_uri_name),
self.wrap_view('get_multiple'),
name="api_get_multiple"),
url(r"^(?P<resource_name>%s)/(?P<%s>\w[\w/-]*)\.(?P<format>\w+)$" %
(self._meta.resource_name, self._meta.detail_uri_name),
self.wrap_view('dispatch_detail'),
name="api_dispatch_detail"),
]
class TaskResource(LockTimeoutMixin, resources.Resource):
TASK_TYPE = 'pending'
SYNTHETIC_FIELDS = [
'blocks',
]
id = UUIDField(attribute='uuid', null=True)
uuid = UUIDField(attribute='uuid')
short_id = fields.IntegerField(attribute='id', null=True)
status = fields.CharField(attribute='status')
urgency = fields.FloatField(attribute='urgency')
description = fields.CharField(attribute='description')
priority = fields.CharField(attribute='priority', null=True)
project = fields.CharField(attribute='project', null=True)
due = fields.DateTimeField(attribute='due', null=True)
entry = fields.DateTimeField(attribute='entry', null=True)
modified = fields.DateTimeField(attribute='modified', null=True)
start = fields.DateTimeField(attribute='start', null=True)
wait = fields.DateTimeField(attribute='wait', null=True)
scheduled = fields.DateTimeField(attribute='scheduled', null=True)
depends = fields.ListField(attribute='depends', null=True)
blocks = fields.ListField(attribute='blocks', null=True)
annotations = fields.ListField(attribute='annotations', null=True)
tags = fields.ListField(attribute='tags', null=True)
imask = fields.IntegerField(attribute='imask', null=True)
def prepend_urls(self):
return [
url(r"^(?P<resource_name>%s)/(?P<username>[\w\d_.-]+)/sms/?$" %
(self._meta.resource_name),
self.wrap_view('incoming_sms'),
name="incoming_sms"),
url(
r"^(?P<resource_name>%s)/(?P<uuid>[\w\d_.-]+)/start/?$" %
(self._meta.resource_name), self.wrap_view('start_task')),
url(
r"^(?P<resource_name>%s)/(?P<uuid>[\w\d_.-]+)/stop/?$" %
(self._meta.resource_name), self.wrap_view('stop_task')),
url(
r"^(?P<resource_name>%s)/(?P<uuid>[\w\d_.-]+)/delete/?$" %
(self._meta.resource_name), self.wrap_view('delete')),
url(r"^(?P<resource_name>%s)/lock/?$" % (self._meta.resource_name),
self.wrap_view('manage_lock')),
url(
r"^(?P<resource_name>%s)/pebble-card/"
r"(?P<secret_id>[\w\d_.-]+)/?$" % (self._meta.resource_name),
self.wrap_view('pebble_card'),
name='pebble_card_url',
),
url(
r"^(?P<resource_name>%s)/ical/(?P<variant>\w+)/"
r"(?P<secret_id>[\w\d_.-]+)/?$" % (self._meta.resource_name),
self.wrap_view('ical_feed'),
name='ical_feed',
),
url(
r"^(?P<resource_name>%s)/refresh/?$" %
(self._meta.resource_name),
self.wrap_view('refresh_tasks'),
name='refresh_tasks',
),
url(
r"^(?P<resource_name>%s)/revert/?$" %
(self._meta.resource_name),
self.wrap_view('revert_to_last_commit'),
name='revert_to_last_commit',
),
url(
r"^(?P<resource_name>%s)/sync/?$" % (self._meta.resource_name),
self.wrap_view('sync_immediately'),
name='sync_immediately',
),
url(
r"^(?P<resource_name>%s)/sync-init/?$" %
(self._meta.resource_name),
self.wrap_view('sync_init'),
name='sync_init',
),
url(
r"^(?P<resource_name>%s)/trello/?$" %
(self._meta.resource_name),
self.wrap_view('trello_authorize'),
name='trello_authorize',
),
url(
r"^(?P<resource_name>%s)/trello/callback/?$" %
(self._meta.resource_name),
self.wrap_view('trello_callback'),
name='trello_callback',
),
url(
r"^(?P<resource_name>%s)/bugwarrior/?$" %
(self._meta.resource_name),
self.wrap_view('bugwarrior_config'),
name='bugwarrior_config',
),
url(
r"^(?P<resource_name>%s)/bugwarrior/sync/?$" %
(self._meta.resource_name),
self.wrap_view('bugwarrior_sync'),
name='bugwarrior_sync',
),
url(
r"^(?P<resource_name>%s)/trello/incoming/"
r"(?P<secret_id>[\w\d_.-]+)/?$" % (self._meta.resource_name),
self.wrap_view('trello_incoming'),
name='trello_incoming',
),
url(
r"^(?P<resource_name>%s)/trello/resynchronize/?$" %
(self._meta.resource_name),
self.wrap_view('trello_resynchronize'),
name='trello_resynchronize',
),
url(
r"^(?P<resource_name>%s)/trello/reset/?$" %
(self._meta.resource_name),
self.wrap_view('trello_reset'),
name='trello_reset',
)
]
def get_task_store(self, request):
return models.TaskStore.get_for_user(request.user)
def dehydrate(self, bundle):
store = self.get_task_store(bundle.request)
for key, data in store.client.config.get_udas().items():
value = getattr(bundle.obj, key.encode('utf8'), None)
bundle.data[key] = value
return bundle
def hydrate(self, bundle):
store = self.get_task_store(bundle.request)
for key, field_instance in store.client.config.get_udas().items():
value = bundle.data.get(key, None)
if value and isinstance(field_instance, DateField):
try:
setattr(bundle.obj, key, parse(value))
except (TypeError, ValueError):
raise exceptions.BadRequest(
"Invalid date provided for field %s: " % (
key,
value,
))
elif value:
setattr(bundle.obj, key, value)
try:
depends = []
for task_id in bundle.data.get('depends', []):
depends.append(uuid.UUID(task_id))
bundle.data['depends'] = depends
except (TypeError, ValueError):
raise exceptions.BadRequest(
"Invalid task IDs provided for field depends: %s" %
(bundle.data.get('depends')))
return bundle
@process_authentication()
def manage_lock(self, request, **kwargs):
store = self.get_task_store(request)
lock_name = get_lock_name_for_store(store)
client = get_lock_redis()
if request.method == 'DELETE':
value = client.delete(lock_name)
if value:
store.log_message("Lockfile deleted.")
return HttpResponse(json.dumps({
'lock_status': value,
}),
status=200)
store.log_error(
"Attempted to delete lockfile, but repository was not locked.")
return HttpResponse(json.dumps(
{'error_message': 'Repository is not locked'}),
status=404)
elif request.method == 'GET':
value = client.get(lock_name)
if value:
return HttpResponse(json.dumps({
'lock_status': value,
}),
status=200)
return HttpResponse(json.dumps(
{'error_message': 'Repository is not locked'}),
status=404)
return HttpResponseNotAllowed(request.method)
@requires_task_store
def revert_to_last_commit(self, request, store, **kwargs):
if request.method != 'POST':
return HttpResponseBadRequest(request.method)
old_head = store.repository.head()
new_head = store.repository.get_object(old_head).parents[0]
with git_checkpoint(store, "Reverting to previous commit", sync=True):
store.git_reset(new_head)
store.log_message(
"Taskstore was reverted from %s to %s via user-initiated "
"revert operation.",
old_head,
new_head,
)
return HttpResponse(
json.dumps({
'message': 'OK',
'old_head': old_head,
'new_head': new_head
}),
content_type='application/json',
)
@requires_task_store
def sync_immediately(self, request, store, **kwargs):
if request.method != 'POST':
return HttpResponseNotAllowed(request.method)
result = store.sync(async=False)
if not result:
return HttpResponse(
json.dumps({
'error_message': ('Synchronization is currently disabled '
'for your account.')
}),
content_type='application/json',
status=406,
)
return HttpResponse(
json.dumps({
'message': 'OK',
}),
content_type='application/json',
)
@requires_task_store
@git_managed("Sync init", sync=False)
def sync_init(self, request, store, **kwargs):
if request.method != 'POST':
return HttpResponseNotAllowed(request.method)
store.client.sync(init=True)
return HttpResponse(
json.dumps({
'message': 'OK',
}),
content_type='application/json',
)
@requires_task_store
def bugwarrior_config(self, request, store, **kwargs):
if request.method == 'DELETE':
config = store.bugwarrior_config
if not config:
return HttpResponseNotFound()
return HttpResponse(
json.dumps({
'message': 'OK',
}),
content_type='application/json',
)
elif request.method == 'PUT':
config = store.bugwarrior_config
if not config:
config = models.BugwarriorConfig.objects.create(
store=store,
enabled=True,
)
config.serialized_config = request.body
try:
config.validate_configuration()
config.save()
except Exception as e:
return HttpResponseBadRequest(
json.dumps({
'error_message': unicode(e),
}),
content_type='application/json',
)
return HttpResponse(
json.dumps({
'message': 'OK',
}),
content_type='application/json',
)
else:
return HttpResponseNotAllowed(request.method)
@requires_task_store
def bugwarrior_sync(self, request, store, **kwargs):
if request.method != 'POST':
return HttpResponseNotAllowed(request.method)
config = store.bugwarrior_config
if not config:
return HttpResponseNotFound()
store.sync_bugwarrior()
return HttpResponse(
json.dumps({
'message': 'OK',
}),
content_type='application/json',
)
@requires_task_store
def trello_authorize(self, request, store, **kwargs):
if request.method != 'GET':
return HttpResponseNotAllowed(request.method)
return HttpResponseRedirect(get_authorize_url(request))
@requires_task_store
def trello_reset(self, request, store, **kwargs):
if request.method != 'POST':
return HttpResponseNotAllowed(request.method)
for obj in models.TrelloObject.objects.filter(store=store):
obj.delete()
with git_checkpoint(store,
"Reset trello IDs for pending/waiting tasks."):
for task in store.client.filter_tasks({
'intheamtrelloid.any':
None,
'or': [
('status', 'pending'),
('status', 'waiting'),
]
}):
task['intheamtrelloid'] = ''
task['intheamtrelloboardid'] = ''
task['intheamtrellolistid'] = ''
store.client.task_update(task)
store.trello_auth_token = ''
store.save()
return HttpResponse(
json.dumps({
'message': 'OK',
}),
content_type='application/json',
)
@requires_task_store
def trello_resynchronize(self, request, store, **kwargs):
if request.method != 'POST':
return HttpResponseNotAllowed(request.method)
sync_trello_tasks.apply_async(args=(store.pk, ))
return HttpResponse(
json.dumps({
'message': 'OK',
}),
content_type='application/json',
)
def trello_incoming(self, request, secret_id, **kwargs):
try:
store = models.TaskStore.objects.get(secret_id=secret_id)
except:
return HttpResponseNotFound()
if not message_signature_is_valid(request):
# @TODO: Once this is verified, return
# HTTPRESPONSEBADREQUEST here.
pass
if request.method == 'POST':
process_trello_action.apply_async(
args=(store.pk, json.loads(request.body.decode('utf-8'))))
return HttpResponse(
json.dumps({
'message': 'OK',
}),
content_type='application/json',
)
@requires_task_store
def trello_callback(self, request, store, **kwargs):
if request.method != 'GET':
return HttpResponseNotAllowed(request.method)
if 'trello_oauth_token' not in request.session:
return HttpResponseBadRequest(
'Arrived at Trello authorization URL without having '
'initiated a Trello authorization!')
token = get_access_token(request)
store.trello_auth_token = token[0]
if not store.trello_board:
board = models.TrelloObject.create(
store=store,
type=models.TrelloObject.BOARD,
name='Inthe.AM Tasks',
desc='Tasks listed on your Inthe.AM account')
for list_data in board.client.get_list(board.id):
obj = models.TrelloObject.objects.create(
id=list_data.get('id'),
store=store,
type=models.TrelloObject.LIST,
parent=board,
meta=list_data)
obj.subscribe()
store.save()
store.sync_trello()
return HttpResponseRedirect('/')
@requires_task_store
def refresh_tasks(self, request, store, **kwargs):
""" Refreshes the task list manually.
.. warning::
This method is DEPRECATED, and will be removed
in a future version.
"""
if request.method != 'GET':
return HttpResponseNotAllowed(request.method)
try:
head = request.GET['head']
except KeyError:
return HttpResponseBadRequest()
store.sync(async=False)
changes = []
new_head = store.repository.head()
for id in store.get_changed_task_ids(head, new_head):
changes.append({
'action': 'task_changed',
'body': id,
})
if new_head != head:
changes.append({'action': 'head_changed', 'body': new_head})
response = {
'messages': changes,
}
return HttpResponse(
json.dumps(response),
content_type='application/json',
)
def ical_feed(self, request, variant, secret_id, **kwargs):
if variant not in ('due', 'waiting'):
return HttpResponseNotFound()
try:
store = models.TaskStore.objects.get(secret_id=secret_id)
except:
return HttpResponseNotFound()
if not store.ical_enabled:
return HttpResponseNotFound()
if variant == 'due':
calendar_title = "Tasks Due"
task_filter = {
'due.any': None,
'status': 'pending',
}
field = 'due'
elif variant == 'waiting':
calendar_title = "Tasks Waiting"
task_filter = {
'status': 'waiting',
}
field = 'wait'
tasks = store.client.filter_tasks(task_filter)
calendar = Calendar()
calendar.add('version', '2.0')
calendar.add('prodid', '-//inthe.am//ical.%s//' % variant)
calendar.add('X-WR-CALNAME', calendar_title)
for task in tasks:
event = Event()
event.add('uid', task['uuid'])
event.add('dtstart', task[field].date())
event.add('dtend', task[field].date() + datetime.timedelta(days=1))
event.add('dtstamp', task.get('modified', task['entry']))
event.add('summary', task['description'])
calendar.add_component(event)
return HttpResponse(
calendar.to_ical(),
content_type='text/calendar',
)
def pebble_card(self, request, secret_id, **kwargs):
if request.method != 'GET':
return HttpResponseNotAllowed(request.method)
try:
store = models.TaskStore.objects.get(secret_id=secret_id)
except:
return HttpResponseNotFound()
if not store.pebble_cards_enabled:
return HttpResponseNotFound()
pending_tasks = store.client.filter_tasks({'status': self.TASK_TYPE})
pending_tasks = sorted(pending_tasks,
key=lambda d: float(d['urgency']),
reverse=True)
response = {'content': None, 'refresh_frequency': 15}
try:
response['content'] = pending_tasks[0]['description']
except IndexError:
response['content'] = 'No tasks exist'
return HttpResponse(
json.dumps(response),
content_type='application/json',
)
@requires_task_store
@git_managed("Start task", sync=True)
def start_task(self, request, uuid, store, **kwargs):
if not request.method == 'POST':
return HttpResponseNotAllowed(request.method)
try:
store.client.task_start(uuid=uuid)
except ValueError:
raise exceptions.NotFound()
store.log_message("Task %s started.", uuid)
return HttpResponse(
json.dumps({
'message': 'OK',
}),
content_type='application/json',
)
@requires_task_store
@git_managed("Stop task", sync=True)
def stop_task(self, request, uuid, store, **kwargs):
if not request.method == 'POST':
return HttpResponseNotAllowed(request.method)
try:
store.client.task_stop(uuid=uuid)
except ValueError:
raise exceptions.NotFound()
store.log_message("Task %s stopped.", uuid)
return HttpResponse(
json.dumps({
'message': 'OK',
}),
content_type='application/json',
)
@requires_task_store
@git_managed("Delete task", sync=True)
def delete(self, request, uuid, store, **kwargs):
if not request.method == 'POST':
return HttpResponseNotAllowed(request.method)
try:
store.client.task_delete(uuid=uuid)
except ValueError:
raise exceptions.NotFound()
store.log_message("Task %s deleted.", uuid)
return HttpResponse(
json.dumps({
'message': 'OK',
}),
content_type='application/json',
)
def incoming_sms(self, request, username, **kwargs):
try:
user = User.objects.get(username=username)
except User.DoesNotExist:
return HttpResponse(status=404)
r = Response()
# This request is unauthenticated; we'll need to fetch the user's
# store directly rather than looking it up via the auth cookie.
store = models.TaskStore.get_for_user(user)
if not store.twilio_auth_token:
log_args = (
"Incoming SMS for %s, but no auth token specified.",
user,
user,
)
logger.warning(*log_args)
store.log_error(*log_args)
return HttpResponse(status=404)
if store.sms_whitelist:
incoming_number = re.sub('[^0-9]', '', request.POST['From'])
valid_numbers = [
re.sub('[^0-9]', '', n)
for n in store.sms_whitelist.split('\n')
]
if incoming_number not in valid_numbers:
log_args = (
"Incoming SMS for %s, but phone number %s is not "
"in the whitelist.",
user,
incoming_number,
)
store.log_error(*log_args)
logger.warning(*log_args,
extra={
'data': {
'incoming_number': incoming_number,
'whitelist': valid_numbers,
}
})
return HttpResponseForbidden()
try:
validator = RequestValidator(store.twilio_auth_token)
url = request.build_absolute_uri()
signature = request.META['HTTP_X_TWILIO_SIGNATURE']
except (AttributeError, KeyError) as e:
log_args = (
"Incoming SMS for %s, but error encountered while "
"attempting to build request validator: %s.",
user,
e,
)
logger.exception(*log_args)
store.log_error(*log_args)
return HttpResponseForbidden()
if not validator.validate(url, request.POST, signature):
log_args = (
"Incoming SMS for %s, but validator rejected message.",
user,
)
logger.warning(*log_args)
store.log_error(*log_args)
return HttpResponseForbidden()
with git_checkpoint(store, "Incoming SMS", sync=True):
from_ = request.POST['From']
body = request.POST['Body']
task_info = body[4:]
if not body.lower().startswith('add'):
if store.sms_replies >= store.REPLY_ERROR:
r.sms("Bad Request: Unknown command.")
log_args = (
"Incoming SMS from %s had no recognized command: '%s'." % (
from_,
body,
), )
logger.warning(*log_args)
store.log_error(*log_args)
elif not task_info:
log_args = ("Incoming SMS from %s had no content." % (
from_,
body,
), )
logger.warning(*log_args)
store.log_error(*log_args)
if store.sms_replies >= store.REPLY_ERROR:
r.sms("Bad Request: Empty task.")
else:
task_uuid = str(uuid.uuid4())
task_args = (['add'] + shlex.split(store.sms_arguments) +
shlex.split(task_info))
task_args.append('uuid:%s' % task_uuid)
result = store.client._execute_safe(*task_args)
stdout, stderr = result
if store.sms_replies >= store.REPLY_ALL:
r.sms("Added.")
log_args = ("Added task %s via SMS from %s; message '%s'; "
"automatic args: '%s';"
"response: '%s'." % (
task_uuid,
from_,
body,
store.sms_arguments,
stdout,
), )
logger.info(*log_args)
store.log_message(*log_args)
return HttpResponse(str(r), content_type='application/xml')
def _get_store(self, user):
return user.task_stores.get()
def detail_uri_kwargs(self, bundle_or_obj):
kwargs = {}
if isinstance(bundle_or_obj, bundle.Bundle):
kwargs['pk'] = bundle_or_obj.obj.uuid
else:
kwargs['pk'] = bundle_or_obj.uuid
return kwargs
def apply_sorting(self, obj_list, options=None):
if options is None:
options = {}
parameter_name = 'order_by'
if hasattr(options, 'getlist'):
order_bits = options.getlist(parameter_name)
else:
order_bits = options.get(parameter_name)
if not isinstance(order_bits, list, tuple):
order_bits = [order_bits]
if not order_bits:
order_bits = ['-urgency']
order_by_args = []
for order_by in order_bits:
order_by_bits = order_by.split(',')
field_name = order_by_bits[0]
reverse = False
if order_by_bits[0].startswith('-'):
field_name = order_by_bits[0][1:]
reverse = True
order_by_args.append((field_name, reverse))
order_by_args.reverse()
for arg, reverse in order_by_args:
obj_list = sorted(
obj_list,
key=operator.attrgetter(arg),
reverse=reverse,
)
return obj_list
def passes_filters(self, task, filters):
passes = True
for key, value in filters.items():
if key not in self.Meta.filter_fields:
continue
task_value = getattr(task, key, None)
if task_value != value:
passes = False
return passes
@requires_task_store
def obj_get_list(self, bundle, store, **kwargs):
if hasattr(bundle.request, 'GET'):
filters = bundle.request.GET.copy()
filters.update(kwargs)
objects = []
for task_json in store.client.filter_tasks({'status': self.TASK_TYPE}):
task = Task(task_json, store=store)
if self.passes_filters(task, filters):
objects.append(task)
return objects
@requires_task_store
def obj_get(self, bundle, store, **kwargs):
task = store.client.get_task(uuid=kwargs['pk'])[1]
if not task:
repository_head = store.repository.head()
logger.warning(
'Unable to find task with ID %s in repository %s at %s',
kwargs['pk'],
store.local_path,
repository_head,
extra={
'data': {
'store': store,
'local_path': store.local_path,
'pk': kwargs['pk'],
'head': repository_head
}
})
raise exceptions.NotFound()
return Task(task, store=store)
@requires_task_store
def obj_create(self, bundle, store, **kwargs):
with git_checkpoint(store, "Creating Task", sync=True):
if not bundle.data['description']:
raise exceptions.BadRequest(
"You must specify a description for each task.")
bundle.obj = self.get_empty_task(bundle.request)
bundle = self.full_hydrate(bundle)
data = bundle.obj.get_json()
for k in self.SYNTHETIC_FIELDS:
if k in data:
data.pop(k, None)
for k, v in TaskwTask.FIELDS.items():
if k in data and v.read_only:
data.pop(k, None)
for k, v in data.items():
if not v:
data.pop(k)
bundle.obj = Task(
store.client.task_add(**data),
store=store,
)
store.log_message(
"New task created: %s.",
bundle.obj.get_json(),
)
return bundle
@requires_task_store
def obj_update(self, bundle, store, **kwargs):
with git_checkpoint(store, "Updating Task", sync=True):
if bundle.data['uuid'] != kwargs['pk']:
raise exceptions.BadRequest(
"Changing the UUID of an existing task is not possible.")
elif not bundle.data['description']:
raise exceptions.BadRequest(
"You must specify a description for each task.")
original = store.client.get_task(uuid=kwargs['pk'])[1]
if not original:
raise exceptions.NotFound()
bundle.obj = self.get_empty_task(bundle.request)
bundle = self.full_hydrate(bundle)
data = bundle.obj.get_json()
for k in json.loads(bundle.request.body).keys():
v = data.get(k)
if ((k in original.FIELDS and original.FIELDS[k].read_only)
or k in self.SYNTHETIC_FIELDS):
continue
original[k] = v
changes = original.get_changes(keep=True)
store.client.task_update(original)
store.log_message(
"Task %s updated: %s.",
kwargs['pk'],
changes,
)
bundle.obj = Task(
store.client.get_task(uuid=kwargs['pk'])[1],
store=store,
)
return bundle
@requires_task_store
def obj_delete(self, bundle, store, **kwargs):
with git_checkpoint(store, "Completing Task", sync=True):
try:
store.log_message("Task %s completed.", kwargs['pk'])
store.client.task_done(uuid=kwargs['pk'])
return bundle
except ValueError:
raise exceptions.NotFound()
def obj_delete_list(self, bundle, store, **kwargs):
raise exceptions.BadRequest()
def dispatch(self, request_type, request, *args, **kwargs):
if request.user.is_authenticated():
metadata = models.UserMetadata.get_for_user(request.user)
else:
metadata = None
if metadata and not metadata.tos_up_to_date:
return HttpResponse(json.dumps({
'error_message': ('Please accept the terms of service at '
'https://inthe.am/terms-of-use.')
}),
status=403)
return super(TaskResource, self).dispatch(request_type, request, *args,
**kwargs)
def get_empty_task(self, request):
store = self.get_task_store(request)
return self._meta.object_class(store=store)
def build_bundle(self,
obj=None,
data=None,
request=None,
objects_saved=None):
""" Builds the bundle!
Overridden only to properly build the `Task` entry with its object.
"""
if obj is None and self._meta.object_class:
obj = self.get_empty_task(request)
return Bundle(obj=obj,
data=data,
request=request,
objects_saved=objects_saved)
class Meta:
always_return_data = True
authorization = authorization.Authorization()
authentication = authentication.MultiAuthentication(
authentication.SessionAuthentication(),
authentication.ApiKeyAuthentication(),
)
list_allowed_methods = ['get', 'put', 'post', 'delete']
detail_allowed_methods = ['get', 'put', 'post', 'delete']
filter_fields = [
'status',
'due',
'entry',
'id',
'imask',
'modified',
'parent',
'recur',
'status',
'urgency',
'uuid',
'wait',
]
limit = 100
max_limit = 400
object_class = Task
class DocsResource(ChemblModelResource):
document_chembl_id = fields.CharField('chembl_id', null=True, blank=True)
score = fields.FloatField('score', use_in='search', null=True, blank=True)
src_id = fields.IntegerField('src_id', null=True, blank=True)
journal_full_title = fields.CharField('journal_id__title',
null=True,
blank=True)
class Meta(ChemblResourceMeta):
queryset = Docs.objects.all()
haystack_queryset = Docs.objects.all().defer('abstract')
excludes = ['doc_id']
resource_name = 'document'
collection_name = 'documents'
detail_uri_name = 'chembl_id'
serializer = ChEMBLApiSerializer(resource_name,
{collection_name: resource_name})
prefetch_related = [
Prefetch('journal_id',
queryset=Journals.objects.only('pk', 'title'))
]
fields = (
'abstract',
'authors',
'doc_type',
'document_chembl_id',
'doi',
'doi_chembl',
'first_page',
'issue',
'journal',
'last_page',
'pubmed_id',
'src_id',
'title',
'volume',
'year',
'patent_id',
'journal_full_title',
)
filtering = {
'abstract': CHAR_FILTERS,
'authors': CHAR_FILTERS,
'doc_type': CHAR_FILTERS,
'document_chembl_id': NUMBER_FILTERS,
'doi': CHAR_FILTERS,
'doi_chembl': CHAR_FILTERS,
'first_page': CHAR_FILTERS,
'issue': CHAR_FILTERS,
'journal': CHAR_FILTERS,
'last_page': CHAR_FILTERS,
'pubmed_id': NUMBER_FILTERS,
'title': CHAR_FILTERS,
'volume': CHAR_FILTERS,
'year': NUMBER_FILTERS,
'patent_id': CHAR_FILTERS,
'journal_full_title': CHAR_FILTERS,
}
ordering = [
field for field in filtering.keys()
if not ('comment' in field or 'description' in field)
]
# ----------------------------------------------------------------------------------------------------------------------
def prepend_urls(self):
"""
Returns a URL scheme based on the default scheme to specify
the response format as a file extension, e.g. /api/v1/users.json
"""
return [
url(r"^(?P<resource_name>%s)/search%s$" %
(self._meta.resource_name, trailing_slash()),
self.wrap_view('get_search'),
name="api_get_search"),
url(r"^(?P<resource_name>%s)/search\.(?P<format>xml|json|jsonp|yaml)$"
% self._meta.resource_name,
self.wrap_view('get_search'),
name="api_get_search"),
url(r"^(?P<resource_name>%s)\.(?P<format>\w+)$" %
self._meta.resource_name,
self.wrap_view('dispatch_list'),
name="api_dispatch_list"),
url(r"^(?P<resource_name>%s)/schema\.(?P<format>\w+)$" %
self._meta.resource_name,
self.wrap_view('get_schema'),
name="api_get_schema"),
url(r"^(?P<resource_name>%s)/datatables\.(?P<format>\w+)$" %
self._meta.resource_name,
self.wrap_view('get_datatables'),
name="api_get_datatables"),
url(r"^(?P<resource_name>%s)/set/(?P<%s_list>\w[\w/;-]*)\.(?P<format>\w+)$"
% (self._meta.resource_name, self._meta.detail_uri_name),
self.wrap_view('get_multiple'),
name="api_get_multiple"),
url(r"^(?P<resource_name>%s)/(?P<%s>\w[\w/-]*)\.(?P<format>\w+)$" %
(self._meta.resource_name, self._meta.detail_uri_name),
self.wrap_view('dispatch_detail'),
name="api_dispatch_detail"),
]
class StatusResource(Resource):
detector = fields.CharField(attribute="detector")
node = fields.CharField(attribute="node")
datarate = fields.FloatField(attribute="datarate")
cpu = fields.FloatField(attribute="cpu")
ram = fields.FloatField(attribute="ram")
mode = fields.CharField(attribute="mode")
user = fields.CharField(attribute="user")
start = fields.DateTimeField()
bltrate = fields.IntegerField(attribute="bltrate")
state = fields.CharField(attribute="state")
alerts = fields.IntegerField(attribute="alerts")
class Meta:
resource_name = 'daq_status'
object_class = DAQStatus
authorization = ReadOnlyAuthorization()
authentication = ApiKeyAuthentication()
throttle = CacheThrottle(throttle_at=10, timeframe=60)
def _db(self):
try:
return (pymongo.MongoClient(
settings.MONITOR_DB_ADDR)[settings.MONITOR_DB_NAME])
except:
return None
# Needed by tastypie
def detail_uri_kwargs(self, bundle_or_obj):
kwargs = {}
if isinstance(bundle_or_obj, Bundle):
kwargs['pk'] = bundle_or_obj.obj.detector
else:
kwargs['pk'] = bundle_or_obj.detector
return kwargs
def get_object_list(self, request):
self.throttle_check(request)
self.log_throttled_access(request)
det = 'tpc'
if 'det' in request.GET:
det = request.GET['det']
return [self._create_obj(det)]
def obj_get_list(self, bundle, **kwargs):
return self.get_object_list(bundle.request)
def obj_get(self, bundle, **kwargs):
self.throttle_check(bundle.request)
self.log_throttled_access(bundle.request)
det = 'tpc'
if 'det' in bundle.request.GET:
det = bundle.request.GET['det']
if 'pk' in kwargs:
det = kwargs['pk']
return self._create_obj(det)
def _create_obj(self, det):
try:
doc = self._db()['daq_status'].find({
"detector": det
}).sort("_id", -1).limit(1)[0]
ret = {
"detector": doc["detector"],
"node": "all",
"state": doc['state'],
"user": doc['startedBy'],
"start": doc['startTime'],
"bltrate": 0,
"cpu": 0,
"ram": 0,
"run": doc['currentRun'],
"datarate": 0,
"mode": doc['mode'],
"alerts": 0
}
except:
ret = {
"error": "error",
"datarate": 0,
"bltrate": 0,
"cpu": 0,
"alerts": 0,
"ram": 0
}
# Alerts
# 1: DAQ error 2: DB error 3: MV error 4: trig error 5: MV/trig
errtxt = ""
try:
logcollection = self._db()["log"]
cursor = logcollection.find({"priority": {"$gt": 2, "$lt": 5}})
if cursor.count() != 0:
ret["alerts"] = 1
# Now try to guess which system done messed up
breakout = False
for doc in cursor:
if breakout:
break
errtext = doc['message']
erruser = doc['sender']
errtxt = dumps(doc)
# TPC DAQ error in readers
for reader in [
'reader0', 'reader1', 'reader2', 'reader3', 'reader4',
'reader6', 'reader7'
]:
if reader in errtext:
ret['alerts'] = 1
breakout = True
# MV DAQ is reader5
if 'reader5' in errtext:
if ret['alerts'] == 4 or ret['alerts'] == 5:
ret['alerts'] = 5
continue
ret['alerts'] = 3
# Super long error means trigger
if len(errtext) > 500 or erruser == 'trigger':
if ret['alerts'] == 3 or ret['alerts'] == 5:
ret['alerts'] = 5
continue
ret['alerts'] = 4
except Exception as e:
logger.error(str(e))
logger.error(errtxt)
ret['alerts'] = 2
# This line disables SC alarms!
#ret['alerts'] = 0
nodes = ["reader5"]
if det == "tpc":
nodes = [
"reader0", "reader1", "reader2", "reader3", "reader4",
"reader6", "reader7"
]
for node in nodes:
cursor = self._db()['daq_rates'].find({
"node": node
}).sort("_id", -1).limit(1)
if cursor.count() > 0:
ret['datarate'] += cursor[0]['datarate']
ret['bltrate'] += cursor[0]['bltrate']
ret['cpu'] += cursor[0]['cpu']
ret['ram'] += cursor[0]['ram']
ret_obj = DAQStatus(initial=ret)
return ret_obj
def obj_create(self, bundle, **kwargs):
pass
def obj_update(self, bundle, **kwargs):
pass
def obj_delete_list(self, bundle, **kwargs):
pass
def obj_delete(self, bundle, **kwargs):
pass
def rollback(self, bundles):
pass
class FareResource(Resource):
base_fare = fields.FloatField(attribute="base_fare", null=True)
base_km = fields.FloatField(attribute="base_km", null=True)
extra_km_fare = fields.FloatField(attribute="extra_km_fare", null=True)
flat_fare = fields.FloatField(attribute="flat_fare", null=True)
car_type = fields.CharField(attribute="car_type", null=True)
trip_type = fields.CharField(attribute="trip_type", null=True)
class Meta:
include_resource_uri = False
allowed_methods = ['get']
resource_name = 'fare'
object_class = DictObject
authorization= ReadOnlyAuthorization()
authentication= BasicAuthentication()
def get_city(self, name):
KEY = CITY_KEY.format(name)
city = cache.get(KEY, None)
if city is None:
try:
city = City.objects.get(name=name)
cache.set(KEY, city, settings.CACHE_TIMEOUT)
except City.DoesNotExist:
city = None
return city
def get_trip(self, type):
KEY = TRIP_KEY.format(type)
trip = cache.get(KEY, None)
if trip is None:
try:
trip = TripType.objects.get(type=type)
cache.set(KEY, trip, settings.CACHE_TIMEOUT)
except TripType.DoesNotExist:
trip = None
return trip
def get_car(self, model):
KEY = CAR_KEY.format(model)
car = cache.get(KEY, None)
if car is None:
try:
car = Car.objects.get(model=model)
cache.set(KEY, car, settings.CACHE_TIMEOUT)
except Car.DoesNotExist:
car = None
return car
def extra_charge(self, time, outside, fare_variation):
extra_charge = 0.0
try:
hours = int(time.split(':')[0])
assert 0<= hours <=23
except Exception as e:
hours = datetime.now().hour
if hours >= settings.START_NIGHT or hours <= settings.END_NIGHT:
extra_charge += fare_variation.outside_city
if outside:
extra_charge += fare_variation.outside_city
return extra_charge
def discount(self, date):
try:
date = datetime.strptime(date, "%Y-%m-%d")
except:
date = datetime.now()
try:
offer = Offer.objects.get(date=date.date())
print offer
return offer.discount_percent
except:
pass
return 0.0
def get_object_list(self, request):
object_list = list()
from_location = request.REQUEST.get('from_location', None)
to_location = request.REQUEST.get('to_location', None)
city_name = request.REQUEST.get('city', None)
trip_type = request.REQUEST.get('trip_type', None)
car_type = request.REQUEST.get('car_type', None)
time = request.REQUEST.get('time', None)
date = request.REQUEST.get('date', None)
outside = request.REQUEST.get('outside', None)
city = self.get_city(city_name)
trip_obj = self.get_trip(trip_type)
car = self.get_car(car_type)
try:
# Convert to True or False
# if 'true' is given first make it 'True' and then convert
outside = eval(outside.title())
except:
outside = False
if city and trip_obj:
is_outside_trip = (trip_type.lower() == 'outstation') or outside
extra_charges = self.extra_charge(time, is_outside_trip, city.farevariation)
dicount = self.discount(date)
fare = {'base_fare': city.base_fare, 'base_km': city.base_km, 'extra_km_fare': city.extra_km_fare}
if car:
total_extra_charges = extra_charges + car.fare_percent
obj = self.get_fare_details(fare, total_extra_charges/100.0, dicount/100.0)
object_list.append(obj)
if not is_outside_trip:
KEY = CITY_TRIP_KEY.format(city.id, trip_obj.id)
city_trip = cache.get(KEY, None)
if city_trip is None:
city_trip = CityTrip.objects.filter(city=city, trip_type=trip_obj)
cache.set(KEY, city_trip, settings.CACHE_TIMEOUT)
for ct in city_trip:
obj = DictObject()
obj.flat_fare = ct.flat_fare + ct.flat_fare*total_extra_charges/100.0 - ct.flat_fare*dicount/100.0
object_list.append(obj)
else:
cars = cache.get(ALL_CAR, None)
if cars is None:
cars = Car.objects.all()
cache.set(ALL_CAR, cars, settings.CACHE_TIMEOUT)
for car in cars:
total_extra_charges = extra_charges + car.fare_percent
obj = self.get_fare_details(fare, total_extra_charges/100.0, dicount/100.0)
obj.car_type = car.model
object_list.append(obj)
return object_list
def get_fare_details(self, fare, extra_charges=0.0, dicount=0.0):
obj = DictObject()
obj.base_fare = fare['base_fare'] + fare['base_fare']*extra_charges - fare['base_fare']*dicount
obj.base_km = fare['base_km'] + fare['base_km']*extra_charges - fare['base_km']*dicount
obj.extra_km_fare = fare['extra_km_fare'] + fare['extra_km_fare']*extra_charges - fare['extra_km_fare']*dicount
return obj
def obj_get_list(self, bundle, **kwargs):
request = bundle.request
return self.get_object_list(request)
def clean_bundle(self, bundle):
null_keys = [k for k, v in bundle.data.items() if v is None ]
for key in null_keys:
del bundle.data[key]
def dehydrate(self, bundle):
self.clean_bundle(bundle)
return bundle
class UsageResource(Resource):
symbol = fields.CharField(attribute='symbol')
value = fields.FloatField(attribute='value')
class Meta:
object_class = UsageObject
class CommonModelApi(ModelResource):
keywords = fields.ToManyField(TagResource, 'keywords', null=True)
category = fields.ToOneField(TopicCategoryResource,
'category',
null=True,
full=True)
owner = fields.ToOneField(UserResource, 'owner', full=True)
absolute__url = fields.CharField()
rating = fields.FloatField(attribute='rating', null=True)
thumbnail_url = fields.CharField(null=True)
def dehydrate_thumbnail_url(self, bundle):
return bundle.obj.get_thumbnail_url()
def dehydrate_absolute__url(self, bundle):
return bundle.obj.get_absolute_url()
def build_filters(self, filters={}):
orm_filters = super(CommonModelApi, self).build_filters(filters)
if 'type__in' in filters and filters['type__in'] in FILTER_TYPES.keys(
):
orm_filters.update({'type': filters.getlist('type__in')})
if 'extent' in filters:
orm_filters.update({'extent': filters['extent'].split(',')})
return orm_filters
def apply_filters(self, request, applicable_filters):
types = applicable_filters.pop('type', None)
extent = applicable_filters.pop('extent', None)
semi_filtered = super(CommonModelApi,
self).apply_filters(request, applicable_filters)
filtered = None
if types:
for the_type in types:
if the_type == 'vector' or the_type == 'raster':
if filtered:
filtered = filtered | semi_filtered.filter(
Layer___storeType=FILTER_TYPES[the_type])
else:
filtered = semi_filtered.filter(
Layer___storeType=FILTER_TYPES[the_type])
else:
if filtered:
filtered = filtered | semi_filtered.instance_of(
FILTER_TYPES[the_type])
else:
filtered = semi_filtered.instance_of(
FILTER_TYPES[the_type])
else:
filtered = semi_filtered
if extent:
filtered = self.filter_bbox(filtered, extent)
return filtered
def filter_bbox(self, queryset, bbox):
'''modify the queryset q to limit to data that intersects with the provided bbox
bbox - 4 tuple of floats representing 'southwest_lng,southwest_lat,northeast_lng,northeast_lat'
returns the modified query
'''
bbox = map(str, bbox) # 2.6 compat - float to decimal conversion
intersects = ~(Q(bbox_x0__gt=bbox[2]) | Q(bbox_x1__lt=bbox[0])
| Q(bbox_y0__gt=bbox[3]) | Q(bbox_y1__lt=bbox[1]))
return queryset.filter(intersects)
class TaskResource(resources.Resource):
TASK_TYPE = 'pending'
id = fields.IntegerField(attribute='id', null=True)
uuid = fields.CharField(attribute='uuid')
status = fields.CharField(attribute='status')
urgency = fields.FloatField(attribute='urgency')
description = fields.CharField(attribute='description')
priority = fields.CharField(attribute='priority', null=True)
project = fields.CharField(attribute='project', null=True)
due = fields.DateTimeField(attribute='due', null=True)
entry = fields.DateTimeField(attribute='entry', null=True)
modified = fields.DateTimeField(attribute='modified', null=True)
start = fields.DateTimeField(attribute='start', null=True)
wait = fields.DateTimeField(attribute='wait', null=True)
scheduled = fields.DateTimeField(attribute='scheduled', null=True)
depends = fields.CharField(attribute='depends', null=True)
annotations = fields.ListField(attribute='annotations', null=True)
tags = fields.ListField(attribute='tags', null=True)
imask = fields.IntegerField(attribute='imask', null=True)
def prepend_urls(self):
return [
url(
r"^(?P<resource_name>%s)/autoconfigure/?$" % (
self._meta.resource_name
),
self.wrap_view('autoconfigure')
),
url(
r"^(?P<resource_name>%s)/(?P<username>[\w\d_.-]+)/sms/?$" % (
self._meta.resource_name
),
self.wrap_view('incoming_sms'),
name="incoming_sms"
),
url(
r"^(?P<resource_name>%s)/(?P<uuid>[\w\d_.-]+)/start/?$" % (
self._meta.resource_name
),
self.wrap_view('start_task')
),
url(
r"^(?P<resource_name>%s)/(?P<uuid>[\w\d_.-]+)/stop/?$" % (
self._meta.resource_name
),
self.wrap_view('stop_task')
),
url(
r"^(?P<resource_name>%s)/(?P<uuid>[\w\d_.-]+)/delete/?$" % (
self._meta.resource_name
),
self.wrap_view('delete')
),
url(
r"^(?P<resource_name>%s)/lock/?$" % (
self._meta.resource_name
),
self.wrap_view('manage_lock')
),
]
def manage_lock(self, request, **kwargs):
store = models.TaskStore.get_for_user(request.user)
lockfile = os.path.join(store.local_path, '.lock')
if request.method == 'DELETE':
if os.path.exists(lockfile):
os.unlink(lockfile)
store.log_message("Lockfile deleted.")
return HttpResponse(
'',
status=200
)
store.log_error(
"Attempted to delete lockfile, but repository was not locked."
)
return HttpResponse(
'',
status=404
)
elif request.method == 'GET':
if os.path.exists(lockfile):
return HttpResponse(
'',
status=200
)
return HttpResponse(
'',
status=404
)
raise HttpResponseNotAllowed(request.method)
@requires_taskd_sync
@git_managed("Start task")
def start_task(self, request, uuid, store, **kwargs):
if not request.method == 'POST':
return HttpResponseNotAllowed(request.method)
try:
store.client.task_start(uuid=uuid)
except ValueError:
raise exceptions.NotFound()
store.log_message("Task %s started.", uuid)
return HttpResponse(
status=200
)
@requires_taskd_sync
@git_managed("Stop task")
def stop_task(self, request, uuid, store, **kwargs):
if not request.method == 'POST':
return HttpResponseNotAllowed(request.method)
try:
store.client.task_stop(uuid=uuid)
except ValueError:
raise exceptions.NotFound()
store.log_message("Task %s stopped.", uuid)
return HttpResponse(
status=200
)
@requires_taskd_sync
@git_managed("Delete task")
def delete(self, request, uuid, store, **kwargs):
if not request.method == 'POST':
return HttpResponseNotAllowed(request.method)
try:
store.client.task_delete(uuid=uuid)
except ValueError:
raise exceptions.NotFound()
store.log_message("Task %s deleted.", uuid)
return HttpResponse(
status=200
)
def incoming_sms(self, request, username, **kwargs):
try:
user = User.objects.get(username=username)
except User.DoesNotExist:
return HttpResponse(status=404)
r = Response()
store = models.TaskStore.get_for_user(user)
if not store.twilio_auth_token:
log_args = (
"Incoming SMS for %s, but no auth token specified.",
user,
user,
)
logger.warning(*log_args)
store.log_error(*log_args)
return HttpResponse(status=404)
if store.sms_whitelist:
incoming_number = re.sub('[^0-9]', '', request.POST['From'])
valid_numbers = [
re.sub('[^0-9]', '', n)
for n in store.sms_whitelist.split('\n')
]
if incoming_number not in valid_numbers:
log_args = (
"Incoming SMS for %s, but phone number %s is not "
"in the whitelist.",
user,
incoming_number,
)
store.log_error(*log_args)
logger.warning(
*log_args,
extra={
'data': {
'incoming_number': incoming_number,
'whitelist': valid_numbers,
}
}
)
return HttpResponseForbidden()
try:
validator = RequestValidator(store.twilio_auth_token)
url = request.build_absolute_uri()
signature = request.META['HTTP_X_TWILIO_SIGNATURE']
except (AttributeError, KeyError) as e:
log_args = (
"Incoming SMS for %s, but error encountered while "
"attempting to build request validator: %s.",
user,
e,
)
logger.warning(*log_args)
store.log_error(*log_args)
return HttpResponseForbidden()
if not validator.validate(url, request.POST, signature):
log_args = (
"Incoming SMS for %s, but validator rejected message.",
user,
)
logger.warning(*log_args)
store.log_error(*log_args)
return HttpResponseForbidden()
with git_checkpoint(store, "Incoming SMS", sync=True):
from_ = request.POST['From']
body = request.POST['Body']
task_info = body[4:]
if not body.lower().startswith('add'):
r.sms("Bad Request: Unknown command.")
log_args = (
"Incoming SMS from %s had no recognized command: '%s'." % (
from_,
body,
),
)
logger.warning(*log_args)
store.log_error(*log_args)
elif not task_info:
log_args = (
"Incoming SMS from %s had no content." % (
from_,
body,
),
)
logger.warning(*log_args)
store.log_error(*log_args)
r.sms("Bad Request: Empty task.")
else:
task_args = ['add'] + shlex.split(task_info)
result = store.client._execute_safe(*task_args)
stdout, stderr = result
r.sms("Added.")
log_args = (
"Added task from %s; message '%s'; response: '%s'." % (
from_,
body,
stdout,
),
)
logger.info(*log_args)
store.log_message(*log_args)
return HttpResponse(str(r), content_type='application/xml')
def autoconfigure(self, request, **kwargs):
store = models.TaskStore.get_for_user(request.user)
try:
store.autoconfigure_taskd()
except Exception as e:
err_message = (
"Error encountered while attempting to autoconfigure."
)
logger.exception(err_message)
store.log_error(err_message)
return HttpResponse(
json.dumps({
'error': str(e)
}),
status=500,
content_type='application/json',
)
store.log_message("Taskstore autoconfiguration completed.")
return HttpResponse(
json.dumps({
'status': 'Successfully configured'
}),
status=200,
content_type='application/json',
)
def _get_store(self, user):
return user.task_stores.get()
def detail_uri_kwargs(self, bundle_or_obj):
kwargs = {}
if isinstance(bundle_or_obj, bundle.Bundle):
kwargs['pk'] = bundle_or_obj.obj.uuid
else:
kwargs['pk'] = bundle_or_obj.uuid
return kwargs
def apply_sorting(self, obj_list, options=None):
if options is None:
options = {}
parameter_name = 'order_by'
if hasattr(options, 'getlist'):
order_bits = options.getlist(parameter_name)
else:
order_bits = options.get(parameter_name)
if not isinstance(order_bits, list, tuple):
order_bits = [order_bits]
if not order_bits:
order_bits = ['-urgency']
order_by_args = []
for order_by in order_bits:
order_by_bits = order_by.split(',')
field_name = order_by_bits[0]
reverse = False
if order_by_bits[0].startswith('-'):
field_name = order_by_bits[0][1:]
reverse = True
order_by_args.append(
(field_name, reverse)
)
order_by_args.reverse()
for arg, reverse in order_by_args:
obj_list = sorted(
obj_list,
key=operator.attrgetter(arg),
reverse=reverse,
)
return obj_list
def passes_filters(self, task, filters):
passes = True
for key, value in filters.items():
if key not in self.Meta.filter_fields:
continue
task_value = getattr(task, key, None)
if task_value != value:
passes = False
return passes
@requires_taskd_sync
def obj_get_list(self, bundle, store, **kwargs):
if hasattr(bundle.request, 'GET'):
filters = bundle.request.GET.copy()
filters.update(kwargs)
objects = []
for task_json in store.client.load_tasks()[self.TASK_TYPE]:
task = Task(task_json)
if self.passes_filters(task, filters):
objects.append(task)
return objects
@requires_taskd_sync
def obj_get(self, bundle, store, **kwargs):
try:
return Task(store.client.get_task(uuid=kwargs['pk'])[1])
except ValueError:
raise exceptions.NotFound()
@requires_taskd_sync
def obj_create(self, bundle, store, **kwargs):
with git_checkpoint(store, "Creating Task"):
if not bundle.data['description']:
raise exceptions.BadRequest(
"You must specify a description for each task."
)
safe_json = Task.from_serialized(bundle.data).get_safe_json()
bundle.obj = Task(
store.client.task_add(**safe_json)
)
store.log_message(
"New task created: %s.", safe_json
)
return bundle
@requires_taskd_sync
def obj_update(self, bundle, store, **kwargs):
with git_checkpoint(store, "Updating Task"):
if bundle.data['uuid'] != kwargs['pk']:
raise exceptions.BadRequest(
"Changing the UUID of an existing task is not possible."
)
elif not bundle.data['description']:
raise exceptions.BadRequest(
"You must specify a description for each task."
)
bundle.data.pop('id', None)
serialized = Task.from_serialized(bundle.data).get_safe_json()
serialized['uuid'] = kwargs['pk']
store.client.task_update(serialized)
store.log_message(
"Task %s updated: %s.",
kwargs['pk'],
serialized
)
bundle.obj = Task(store.client.get_task(uuid=kwargs['pk'])[1])
return bundle
def obj_delete_list(self, bundle, store, **kwargs):
raise exceptions.BadRequest()
def dispatch(self, request_type, request, *args, **kwargs):
metadata = models.UserMetadata.get_for_user(request.user)
if not metadata.tos_up_to_date:
return HttpResponse(
json.dumps(
{
'error_message': (
'Please accept the terms of service at '
'https://inthe.am/terms-of-use.'
)
}
),
status=403
)
try:
return super(TaskResource, self).dispatch(
request_type, request, *args, **kwargs
)
except LockTimeout:
message = (
'Your task list is currently locked by another client.'
'If this error persists, you may try '
'clearing the lockfile by sending a DELETE request '
'to http://inthe.am/api/v1/task/lock/. '
'Please refer to the API documentation for details.'
)
store = models.TaskStore.get_for_user(request.user)
store.log_error(message)
return HttpResponse(
json.dumps(
{
'error_message': message
}
),
status=409,
)
@requires_taskd_sync
def obj_delete(self, bundle, store, **kwargs):
with git_checkpoint(store, "Completing Task"):
try:
store.log_message("Task %s completed.", kwargs['pk'])
store.client.task_done(uuid=kwargs['pk'])
return bundle
except ValueError:
raise exceptions.NotFound()
class Meta:
always_return_data = True
authorization = authorization.Authorization()
authentication = authentication.MultiAuthentication(
authentication.SessionAuthentication(),
authentication.ApiKeyAuthentication(),
)
list_allowed_methods = ['get', 'put', 'post', 'delete']
detail_allowed_methods = ['get', 'put', 'post', 'delete']
filter_fields = [
'status',
'due',
'entry',
'id',
'imask',
'modified',
'parent',
'recur',
'status',
'urgency',
'uuid',
'wait',
]
limit = 100
max_limit = 400
def shotgun_entity_resource_factory(entity_type):
schema = sg.schema().entityInfo(entity_type)
class EntityResource(ShotgunEntityResource):
_entity_type = entity_type
_schema = schema
class Meta(ShotgunEntityResource.Meta):
resource_name = cammel_case_to_slug(entity_type)
if schema:
for field_name in sg.defaultEntityQueryFields(entity_type):
field_info = schema.fieldInfos()[field_name]
return_type = field_info.returnType()
resource_field = dehydrate_method = None
# RETURN_TYPE_CHECKBOX = 0
if return_type == ShotgunORM.SgField.RETURN_TYPE_CHECKBOX:
resource_field = fields.BooleanField(attribute=field_name, null=True)
# RETURN_TYPE_DATE = 3
elif return_type == ShotgunORM.SgField.RETURN_TYPE_DATE:
resource_field = fields.DateField(attribute=field_name, null=True)
# RETURN_TYPE_DATE_TIME = 4
elif return_type == ShotgunORM.SgField.RETURN_TYPE_DATE_TIME:
resource_field = fields.DateTimeField(attribute=field_name, null=True)
# RETURN_TYPE_FLOAT = 6
elif return_type ==ShotgunORM.SgField.RETURN_TYPE_FLOAT:
resource_field = fields.FloatField(attribute=field_name, null=True)
# RETURN_TYPE_INT = 8
elif return_type ==ShotgunORM.SgField.RETURN_TYPE_INT:
resource_field = fields.IntegerField(attribute=field_name, null=True)
# RETURN_TYPE_URL = 16
elif return_type == ShotgunORM.SgField.RETURN_TYPE_URL:
resource_field = fields.DictField(attribute=field_name, null=True)
# RETURN_TYPE_TEXT = 15
elif return_type == ShotgunORM.SgField.RETURN_TYPE_TEXT:
resource_field = fields.CharField(attribute=field_name, null=True)
# RETURN_TYPE_ENTITY = 5
elif return_type == ShotgunORM.SgField.RETURN_TYPE_ENTITY:
resource_field = fields.IntegerField(attribute=field_name, null=True)
#def dehydrate_method_factory(field_name):
# def dehydrate_method(self, bundle):
# # assert False, bundle.data.keys()
# return bundle.data[field_name]
# return dehydrate_method
#setattr(EntityResource, "dehydrate_%s" % field_name, dehydrate_method_factory(field_name))
# RETURN_TYPE_MULTI_ENTITY = 10
elif return_type == ShotgunORM.SgField.RETURN_TYPE_MULTI_ENTITY:
pass
#resource_field = fields.CharField(attribute=field_name, null=True)
# RETURN_TYPE_SERIALIZABLE = 11
elif return_type == ShotgunORM.SgField.RETURN_TYPE_SERIALIZABLE:
resource_field = fields.DictField(attribute=field_name, null=True)
# RETURN_TYPE_TAG_LIST = 14
elif return_type == ShotgunORM.SgField.RETURN_TYPE_TAG_LIST:
resource_field = fields.ListField(attribute=field_name, null=True)
# RETURN_TYPE_LIST = 9
# RETURN_TYPE_STATUS_LIST = 12
# RETURN_TYPE_SUMMARY = 13
# RETURN_TYPE_COLOR = 1
# RETURN_TYPE_COLOR2 = 2
# RETURN_TYPE_IMAGE = 7
# RETURN_TYPE_UNSUPPORTED = -1
elif return_type != ShotgunORM.SgField.RETURN_TYPE_UNSUPPORTED:
resource_field = fields.CharField(attribute=field_name, null=True)
if resource_field:
EntityResource.base_fields[field_name] = resource_field
return EntityResource