class ActivityResource(ModelResource):
iati_identifier = fields.CharField('id')
reporting_organisation = fields.ForeignKey(
ActivityViewOrganisationResource,
'reporting_organisation',
full=True,
null=True)
participating_organisations = fields.ToManyField(
ActivityViewOrganisationResource,
'participating_organisation',
full=True,
null=True)
activity_status = fields.ForeignKey(ActivityViewActivityStatusResource,
'activity_status',
full=True,
null=True)
recipient_country = fields.ToManyField(OnlyCountryResource,
'recipient_country',
full=True,
null=True)
recipient_region = fields.ToManyField(OnlyRegionResource,
'recipient_region',
full=True,
null=True)
sectors = fields.ToManyField(ActivityViewSectorsResource,
'sector',
full=True,
null=True)
titles = fields.ToManyField(TitleResource,
'title_set',
full=True,
null=True)
descriptions = fields.ToManyField(DescriptionResource,
'description_set',
full=True,
null=True)
collaboration_type = fields.ForeignKey(
ActivityViewCollaborationTypeResource,
attribute='collaboration_type',
full=True,
null=True)
default_flow_type = fields.ForeignKey(ActivityViewFlowTypeResource,
attribute='default_flow_type',
full=True,
null=True)
default_finance_type = fields.ForeignKey(FinanceTypeResource,
attribute='default_finance_type',
full=True,
null=True)
default_aid_type = fields.ForeignKey(ActivityViewAidTypeResource,
attribute='default_aid_type',
full=True,
null=True)
default_tied_status_type = fields.ForeignKey(
ActivityViewTiedStatusResource,
attribute='default_tied_status',
full=True,
null=True)
activity_budgets = fields.ToManyField(ActivityBudgetResource,
'budget_set',
full=True,
null=True)
activity_transactions = fields.ToManyField(ActivityViewTransactionResource,
'transaction_set',
full=True,
null=True)
documents = fields.ToManyField(DocumentResource,
'document_link_set',
full=True,
null=True)
date_updated = fields.DateTimeField('last_updated_datetime', null=True)
class Meta:
queryset = Activity.objects.all()
resource_name = 'activities'
max_limit = 100
serializer = Serializer(formats=['xml', 'json'])
excludes = ['date_created', 'id']
ordering = [
'start_actual', 'start_planned', 'end_actual', 'end_planned',
'activity_sectors', 'statistics', 'last_updated_datetime'
]
filtering = {'iati_identifier': ALL}
cache = NoTransformCache()
def apply_filters(self, request, applicable_filters):
base_object_list = super(ActivityResource,
self).apply_filters(request,
applicable_filters)
query = request.GET.get('query', None)
sectors = request.GET.get('sectors', None)
regions = request.GET.get('regions', None)
countries = request.GET.get('countries', None)
organisations = request.GET.get('organisations', None)
filters = {}
if sectors:
sectors = sectors.replace('|', ',').split(',')
filters.update(dict(activity_sector__sector__code__in=sectors))
if regions:
regions = regions.replace('|', ',').split(',')
filters.update(
dict(activity_recipient_region__region__code__in=regions))
if countries:
countries = countries.replace('|', ',').split(',')
filters.update(
dict(activity_recipient_country__country__code__in=countries))
if organisations:
organisations = organisations.replace('|', ',').split(',')
filters.update(
dict(reporting_organisation__code__in=organisations))
if query:
qset = (Q(activity_recipient_country__country__name__in=query,
**filters) | Q(title__title__icontains=query, **filters)
| Q(description__text__icontains=query, **filters))
return base_object_list.filter(qset).distinct()
return base_object_list.filter(**filters).distinct()
class ConfigurableReportDataResource(HqBaseResource, DomainSpecificResourceMixin):
"""
A resource that replicates the behavior of the ajax part of the
ConfigurableReportView view.
"""
data = fields.ListField(attribute="data", readonly=True)
columns = fields.ListField(attribute="columns", readonly=True)
total_records = fields.IntegerField(attribute="total_records", readonly=True)
next_page = fields.CharField(attribute="next_page", readonly=True)
LIMIT_DEFAULT = 50
LIMIT_MAX = 50
def _get_start_param(self, bundle):
try:
start = int(bundle.request.GET.get('offset', 0))
if start < 0:
raise ValueError
except (ValueError, TypeError):
raise BadRequest("start must be a positive integer.")
return start
def _get_limit_param(self, bundle):
try:
limit = int(bundle.request.GET.get('limit', self.LIMIT_DEFAULT))
if limit < 0:
raise ValueError
except (ValueError, TypeError):
raise BadRequest("limit must be a positive integer.")
if limit > self.LIMIT_MAX:
raise BadRequest("Limit may not exceed {}.".format(self.LIMIT_MAX))
return limit
def _get_next_page(self, domain, id_, start, limit, total_records, get_query_dict):
if total_records > start + limit:
start += limit
new_get_params = get_query_dict.copy()
new_get_params["offset"] = start
# limit has not changed, but it may not have been present in get params before.
new_get_params["limit"] = limit
return reverse('api_dispatch_detail', kwargs=dict(
api_name=self._meta.api_name,
resource_name=self._meta.resource_name,
domain=domain,
pk=id_,
)) + "?" + new_get_params.urlencode()
else:
return ""
def _get_report_data(self, report_config, domain, start, limit, get_params):
report = ConfigurableReportDataSource.from_spec(report_config, include_prefilters=True)
string_type_params = [
filter.name
for filter in report_config.ui_filters
if getattr(filter, 'datatype', 'string') == "string"
]
filter_values = get_filter_values(
report_config.ui_filters,
query_dict_to_dict(get_params, domain, string_type_params)
)
report.set_filter_values(filter_values)
page = list(report.get_data(start=start, limit=limit))
columns = []
for column in report.columns:
simple_column = {
"header": column.header,
"slug": column.slug,
}
if isinstance(column, UCRExpandDatabaseSubcolumn):
simple_column['expand_column_value'] = column.expand_value
columns.append(simple_column)
total_records = report.get_total_records()
return page, columns, total_records
def obj_get(self, bundle, **kwargs):
domain = kwargs['domain']
pk = kwargs['pk']
start = self._get_start_param(bundle)
limit = self._get_limit_param(bundle)
report_config = self._get_report_configuration(pk, domain)
page, columns, total_records = self._get_report_data(
report_config, domain, start, limit, bundle.request.GET)
return ConfigurableReportData(
data=page,
columns=columns,
total_records=total_records,
id=report_config._id,
domain=domain,
get_params=bundle.request.GET,
next_page=self._get_next_page(
domain,
report_config._id,
start,
limit,
total_records,
bundle.request.GET,
)
)
def _get_report_configuration(self, id_, domain):
"""
Fetch the required ReportConfiguration object
:param id_: The id of the ReportConfiguration
:param domain: The domain of the ReportConfiguration
:return: A ReportConfiguration
"""
try:
if report_config_id_is_static(id_):
return StaticReportConfiguration.by_id(id_, domain=domain)
else:
return get_document_or_not_found(ReportConfiguration, domain, id_)
except DocumentNotFound:
raise NotFound
def detail_uri_kwargs(self, bundle_or_obj):
return {
'domain': get_obj(bundle_or_obj).domain,
'pk': get_obj(bundle_or_obj).id,
}
def get_resource_uri(self, bundle_or_obj=None, url_name='api_dispatch_list'):
uri = super(ConfigurableReportDataResource, self).get_resource_uri(bundle_or_obj, url_name)
if bundle_or_obj is not None and uri:
get_params = get_obj(bundle_or_obj).get_params.copy()
if "offset" not in get_params:
get_params["offset"] = 0
if "limit" not in get_params:
get_params["limit"] = self.LIMIT_DEFAULT
uri += "?{}".format(get_params.urlencode())
return uri
class Meta(CustomResourceMeta):
authentication = RequirePermissionAuthentication(Permissions.view_reports, allow_session_auth=True)
list_allowed_methods = []
detail_allowed_methods = ["get"]
class TemporaryFileResource(ModelResource):
uuid = fields.CharField(attribute='uuid', readonly=True)
class Meta:
queryset = TemporaryFile.objects.all()
list_allowed_methods = ['get']
detail_allowed_methods = ['get', 'delete']
resource_name = 'temporary-files'
authentication = MultiAuthentication(SessionAuthentication(),
ApiKeyAuthentication())
authorization = DjangoAuthorization()
fields = ['uuid', 'date', 'filename']
detail_uri_name = 'uuid'
serializer = Serializer(formats=['json', 'jsonp'])
def prepend_urls(self):
return [
url(r"^(?P<resource_name>%s)/(?P<%s>.*?)/describe%s$" %
(self._meta.resource_name, self._meta.detail_uri_name,
trailing_slash()),
self.wrap_view('describe'),
name='temporary_file_describe'),
url(r"^(?P<resource_name>%s)/(?P<%s>.*?)/(?P<dataset_id>[\w\-@\._]+)/deploy%s$"
% (self._meta.resource_name, self._meta.detail_uri_name,
trailing_slash()),
self.wrap_view('deploy'),
name='temporary_file_deploy'),
url(r"^(?P<resource_name>%s)/(?P<%s>.*?)/(?P<dataset_id>[\w\-@\._]+)/append%s$"
% (self._meta.resource_name, self._meta.detail_uri_name,
trailing_slash()),
self.wrap_view('append'),
name='temporary_file_append')
]
def populate_point_data(self, dataset_id, csv_info):
srid = csv_info['srid']
x_column = csv_info['xColumn']
y_column = csv_info['yColumn']
populate_point_data(dataset_id, srid, x_column, y_column)
def describe(self, request, **kwargs):
"""
Describe, located at the ``{tablo-server}/api/v1/temporary-files/{uuid}/describe`` endpoint, will describe
the uploaded CSV file. This allows you to know the column names and data types that were found within the
file.
:return:
A JSON object in the following format:
.. code-block:: json
{
"fieldNames": ["field one", "field two", "latitude", "longitude"],
"dataTypes": ["String", "Integer", "Decimal", "Decimal"],
"optionalFields": ["field one"],
"xColumn": "longitude",
"yColumn": "latitude",
"filename": "uploaded.csv"
}
**fieldNames**
A list of field (column) names within the CSV
**dataTypes**
A list of data types for each of the columns. The index of this list will match the index of the
fieldNames list.
**optionalFields**
A list of fields that had empty values, and are taken to be optional.
**xColumn**
The best guess at which column contains X spatial coordinates.
**yColumn**
The best guess at which column contains Y spatial coordinates.
**filename**
The name of the file being described
"""
self.is_authenticated(request)
bundle = self.build_bundle(request=request)
obj = self.obj_get(bundle, **self.remove_api_resource_names(kwargs))
try:
if obj.extension == 'csv':
csv_file_name = obj.file.name
else:
raise InvalidFileError('Unsupported file format',
extension=obj.extension)
except InvalidFileError as e:
raise ImmediateHttpResponse(
HttpBadRequest(content=json.dumps(
derive_error_response_data(e, code=BAD_DATA)),
content_type='application/json'))
csv_info = json.loads(request.POST.get('csv_info') or '{}') or None
prepared_csv = prepare_csv_rows(obj.file, csv_info)
row_set = prepared_csv['row_set']
if not len(row_set):
raise ImmediateHttpResponse(
HttpBadRequest(content=json.dumps(
derive_error_response_data(InvalidFileError(
'File is empty', lines=0),
code=BAD_DATA)),
content_type='application/json'))
bundle.data['fieldNames'] = row_set.columns.to_list()
bundle.data['dataTypes'] = prepared_csv['data_types']
bundle.data['optionalFields'] = prepared_csv['optional_fields']
x_field, y_field = prepared_csv['coord_fields']
if x_field and y_field:
bundle.data['xColumn'] = x_field
bundle.data['yColumn'] = y_field
bundle.data.update({'file_name': csv_file_name})
return self.create_response(request, bundle)
def deploy(self, request, **kwargs):
"""
The deploy endpoint, at ``{tablo_server}/api/v1/temporary-files/{uuid}/{dataset_id}/deploy/`` deploys
the file specified by {uuid} into a database table named after the {dataset_id}. The {dataset_id} must
be unique for the instance of Tablo.
With the deploy endpoint, this is the start of what Tablo considers an import. The data will be
temporarily stored in an import table until the finalize endpoint for the dataset_id is called.
POST messages to the deploy endpoint should include the following data:
**csv_info**
Information about the CSV file. This is generally that information obtained through the
describe endpoint, but can be modified to send additional information or modify it.
**fields**
A list of field JSON objects in the following format:
.. code-block:: json
{
"name": "field_name",
"type": "text",
"required": true
}
The value can be specified if the field is a constant value throughout the table. This can
be use for adding audit information.
:return:
An empty HTTP 200 response if the deploy was successful. An error response if otherwise.
"""
self.is_authenticated(request)
try:
dataset_id = kwargs.pop('dataset_id', None)
bundle = self.build_bundle(request=request)
obj = self.obj_get(bundle,
**self.remove_api_resource_names(kwargs))
csv_info = json.loads(request.POST.get('csv_info'))
additional_fields = json.loads(request.POST.get('fields'))
prepared_csv = prepare_csv_rows(obj.file, csv_info)
table_name = create_database_table(
prepared_csv['row_set'],
csv_info,
dataset_id,
additional_fields=additional_fields)
add_geometry_column(dataset_id)
self.populate_point_data(dataset_id, csv_info)
bundle.data['table_name'] = table_name
obj.delete(
) # Temporary file has been moved to database, safe to delete
except Exception as e:
logger.exception(e)
raise ImmediateHttpResponse(
HttpBadRequest(content=json.dumps(
derive_error_response_data(e)),
content_type='application/json'))
return self.create_response(request, bundle)
def append(self, request, **kwargs):
self.is_authenticated(request)
try:
dataset_id = kwargs.pop('dataset_id', None)
bundle = self.build_bundle(request=request)
obj = self.obj_get(bundle,
**self.remove_api_resource_names(kwargs))
csv_info = json.loads(request.POST.get('csv_info'))
additional_fields = json.loads(request.POST.get('fields'))
prepared_csv = prepare_csv_rows(obj.file, csv_info)
table_name = create_database_table(
prepared_csv['row_set'],
csv_info,
dataset_id,
append=True,
additional_fields=additional_fields)
self.populate_point_data(dataset_id, csv_info)
bundle.data['table_name'] = table_name
obj.delete(
) # Temporary file has been moved to database, safe to delete
except Exception as e:
logger.exception(e)
raise ImmediateHttpResponse(
HttpBadRequest(content=json.dumps(
derive_error_response_data(e)),
content_type='application/json'))
return self.create_response(request, bundle)
class DocumentResource(DeprecatedModelResourceWithFieldsFilter):
citation = fields.ForeignKey(CitationResource, 'citation', full=True)
court = fields.ForeignKey(JurisdictionResource, 'docket__court')
html = fields.CharField(
attribute='html',
use_in='detail',
null=True,
help_text='HTML of the document, if available in the original')
html_lawbox = fields.CharField(attribute='html_lawbox',
use_in='detail',
null=True,
help_text='HTML of lawbox documents')
html_with_citations = fields.CharField(
attribute='html_with_citations',
use_in='detail',
null=True,
help_text="HTML of the document with citation links and other "
"post-processed markup added",
)
plain_text = fields.CharField(
attribute='plain_text',
use_in='detail',
null=True,
help_text="Plain text of the document after extraction using "
"pdftotext, wpd2txt, etc.",
)
date_modified = fields.DateTimeField(
attribute='date_modified',
null=True,
default='1750-01-01T00:00:00Z',
help_text='The last moment when the item was modified. A value in '
'year 1750 indicates the value is unknown')
class Meta:
authentication = authentication.MultiAuthentication(
BasicAuthenticationWithUser(realm="courtlistener.com"),
authentication.SessionAuthentication())
throttle = PerUserCacheThrottle(throttle_at=1000)
resource_name = 'opinion'
queryset = Document.objects.all().select_related(
'docket__court__pk', 'citation')
max_limit = 20
allowed_methods = ['get']
include_absolute_url = True
excludes = [
'is_stub_document',
'cases_cited',
'supreme_court_db_id',
]
filtering = {
'id': ('exact', ),
'time_retrieved': good_time_filters,
'date_modified': good_time_filters,
'date_filed': good_date_filters,
'sha1': ('exact', ),
'court': ('exact', ),
'citation': ALL,
'citation_count': numerical_filters,
'precedential_status': ('exact', 'in'),
'date_blocked': good_date_filters,
'blocked': ALL,
'extracted_by_ocr': ALL,
}
ordering = [
'time_retrieved', 'date_modified', 'date_filed', 'date_blocked'
]
class SummaryFeedByCountryCodeResource(Resource):
id = fields.CharField(attribute='id')
freq = fields.IntegerField(attribute='freq')
city = fields.CharField(attribute='city', default=None)
longlat = fields.ListField(attribute='longlat')
class Meta:
resource_name = 'summary-country'
object_class = SummaryFeed
# adapted this from ModelResource
# def get_resource_uri(self, bundle_or_obj=None):
# kwargs = {
# 'resource_name': self._meta.resource_name,
# }
# if isinstance(bundle_or_obj, Bundle):
# kwargs['pk'] = bundle_or_obj.obj.id # pk is referenced in ModelResource
# else:
# kwargs['pk'] = bundle_or_obj.id
# if self._meta.api_name is not None:
# kwargs['api_name'] = self._meta.api_name
# return self._build_reverse_url('api_dispatch_detail', kwargs = kwargs)
def prepend_urls(self):
return [
url(r"^(?P<resource_name>%s)/(?P<countrycode>[\w\d_.-]+)/$" %
self._meta.resource_name,
self.wrap_view('dispatch_detail'),
name="api_dispatch_detail"),
]
def get_object_list(self, request):
return []
def obj_get_list(self, request=None, **kwargs):
# outer get of object list... this calls get_object_list and
# could be a point at which additional filtering may be applied
return self.get_object_list(request)
def obj_get(self, request=None, **kwargs):
global resultsByCountry
countryCode = kwargs['countrycode'].upper()
resultsByCountry.clear()
#countryCode = request.GET.get('countryCode', '')
#print 'countryCode: ' + countryCode
cursor = connection.cursor()
print 'test - 1'
cursor.execute(
'SELECT COUNT(*) as freq, longitude, latitude, city FROM web_feed WHERE country = %s GROUP BY longitude, latitude ORDER BY freq DESC',
[countryCode])
print 'test - 2'
rows = cursor.fetchall()
print 'test - 3'
for r in rows:
print 'test - 4'
#print 'args: ',r[0], [r[1], r[2]], r[3]
s = SummaryFeed()
s.freq = r[0]
s.longlat = [r[1], r[2]]
if r[3]:
s.city = r[3]
else:
s.city = '<Unknown>'
s.id = '%f_%f' % (r[1], r[2])
resultsByCountry[s.id] = s
print 'test - 5'
return resultsByCountry.values()
class FilesystemResource(MetricResource, ConfParamResource):
"""
A Lustre file system, associated with exactly one MGT and consisting of
one or mode MDTs and one or more OSTs.
When using POST to create a file system, specify volumes to use like this:
::
{osts: [{volume_id: 22}],
mdt: {volume_id: 23},
mgt: {volume_id: 24}}
To create a file system using an existing MGT instead of creating a new
MGT, set the `id` attribute instead of the `volume_id` attribute for
that target (i.e. `mgt: {id: 123}`).
Note: A Lustre file system is owned by an MGT, and the ``name`` of the file system
is unique within that MGT. Do not use ``name`` as a globally unique identifier
for a file system in your application.
"""
bytes_free = fields.IntegerField()
bytes_total = fields.IntegerField()
files_free = fields.IntegerField()
files_total = fields.IntegerField()
client_count = fields.IntegerField(
help_text=
"Number of Lustre clients which are connected to this file system")
mount_command = fields.CharField(
null=True,
help_text='Example command for\
mounting this file system on a Lustre client, e.g. "mount -t lustre 192.168.0.1:/testfs /mnt/testfs"',
)
mount_path = fields.CharField(
null=True,
help_text='Path for mounting the file system\
on a Lustre client, e.g. "192.168.0.1:/testfs"',
)
osts = fields.ToManyField(
"chroma_api.target.TargetResource",
null=True,
attribute=lambda bundle: ManagedOst.objects.filter(filesystem=bundle.
obj),
help_text="List of OSTs which belong to this file system",
)
mdts = fields.ToManyField(
"chroma_api.target.TargetResource",
null=True,
full=True,
attribute=lambda bundle: ManagedMdt.objects.filter(filesystem=bundle.
obj),
help_text=
"List of MDTs in this file system, should be at least 1 unless the "
"file system is in the process of being deleted",
)
mgt = fields.ToOneField(
"chroma_api.target.TargetResource",
attribute="mgs",
full=True,
help_text="The MGT on which this file system is registered",
)
def _get_stat_simple(self, bundle, klass, stat_name, factor=1.0):
try:
return bundle.obj.metrics.fetch_last(
klass, fetch_metrics=[stat_name])[1][stat_name] * factor
except (KeyError, IndexError, TypeError):
return None
def dehydrate_mount_path(self, bundle):
return bundle.obj.mount_path()
def dehydrate_mount_command(self, bundle):
path = self.dehydrate_mount_path(bundle)
if path:
return "mount -t lustre %s /mnt/%s" % (path, bundle.obj.name)
else:
return None
def dehydrate_bytes_free(self, bundle):
return self._get_stat_simple(bundle, ManagedOst, "kbytesfree", 1024)
def dehydrate_bytes_total(self, bundle):
return self._get_stat_simple(bundle, ManagedOst, "kbytestotal", 1024)
def dehydrate_files_free(self, bundle):
return self._get_stat_simple(bundle, ManagedMdt, "filesfree")
def dehydrate_files_total(self, bundle):
return self._get_stat_simple(bundle, ManagedMdt, "filestotal")
def get_hsm_control_params(self, mdt, bundle):
all_params = set(HSM_CONTROL_PARAMS.keys())
available_params = all_params - set([bundle.data["cdt_status"]])
bundle_params = []
# Strip the mdt down for brevity of transport and also to
# avoid problems with the PUT.
(resource, id) = mdt.data["resource_uri"].split("/")[-3:-1]
safe_mdt = dict(kind=mdt.data["kind"],
resource=resource,
id=id,
conf_params=mdt.data["conf_params"])
for param in available_params:
bundle_params.append(
dict(
mdt=safe_mdt,
param_key=HSM_CONTROL_KEY,
param_value=param,
verb=HSM_CONTROL_PARAMS[param]["verb"],
long_description=HSM_CONTROL_PARAMS[param]
["long_description"],
))
return bundle_params
def dehydrate(self, bundle):
# Have to do this here because we can't guarantee ordering during
# full_dehydrate to ensure that the mdt bundles are available.
try:
mdt = next(m for m in bundle.data["mdts"]
if "mdt.hsm_control" in m.data["conf_params"])
bundle.data["cdt_status"] = mdt.data["conf_params"][
"mdt.hsm_control"]
bundle.data["cdt_mdt"] = mdt.data["resource_uri"]
bundle.data["hsm_control_params"] = self.get_hsm_control_params(
mdt, bundle)
except StopIteration:
pass
# Now the number of MDT's is known calculate the client count. The client count is calculated by the number of connections
# divided by the number of MDT's. In the case, that is possible durring creation and deletion of filesystems, where the mdt
# count is 0 then the connected clients must be zero.
if len(bundle.data["mdts"]) == 0:
bundle.data["client_count"] = 0
else:
bundle.data["client_count"] = self._get_stat_simple(
bundle,
ManagedMdt,
"client_count",
factor=1.0 / len(bundle.data["mdts"]))
return bundle
class Meta:
queryset = ManagedFilesystem.objects.all()
resource_name = "filesystem"
authorization = DjangoAuthorization()
authentication = AnonymousAuthentication()
excludes = ["not_deleted", "ost_next_index", "mdt_next_index"]
ordering = ["name"]
filtering = {"id": ["exact", "in"], "name": ["exact"]}
list_allowed_methods = ["get", "post"]
detail_allowed_methods = ["get", "delete", "put"]
readonly = [
"bytes_free",
"bytes_total",
"files_free",
"files_total",
"client_count",
"mount_command",
"mount_path",
]
validation = FilesystemValidation()
always_return_data = True
@validate
def obj_create(self, bundle, **kwargs):
request = bundle.request
filesystem_id, command_id = JobSchedulerClient.create_filesystem(
bundle.data)
filesystem = ManagedFilesystem.objects.get(pk=filesystem_id)
command = Command.objects.get(pk=command_id)
fs_bundle = self.full_dehydrate(self.build_bundle(obj=filesystem))
filesystem_data = self.alter_detail_data_to_serialize(
request, fs_bundle).data
raise custom_response(self, request, http.HttpAccepted, {
"command": dehydrate_command(command),
"filesystem": filesystem_data
})
class ProfileResource(ModelResource):
"""Profile api"""
user = fields.ToOneField(UserResource, 'user')
avatar_100 = fields.CharField(null=True)
profile_detail_url = fields.CharField()
email = fields.CharField(default='')
layers_count = fields.IntegerField(default=0)
maps_count = fields.IntegerField(default=0)
documents_count = fields.IntegerField(default=0)
current_user = fields.BooleanField(default=False)
activity_stream_url = fields.CharField(null=True)
def build_filters(self, filters={}):
"""adds filtering by group functionality"""
orm_filters = super(ProfileResource, self).build_filters(filters)
if 'group' in filters:
orm_filters['group'] = filters['group']
return orm_filters
def apply_filters(self, request, applicable_filters):
"""filter by group if applicable by group functionality"""
group = applicable_filters.pop('group', None)
semi_filtered = super(ProfileResource, self).apply_filters(request, applicable_filters)
if group is not None:
semi_filtered = semi_filtered.filter(user__groupmember__group__slug=group)
return semi_filtered
def dehydrate_email(self, bundle):
email = ''
if bundle.request.user.is_authenticated():
email = bundle.obj.email
return email
def dehydrate_layers_count(self, bundle):
return bundle.obj.user.resourcebase_set.instance_of(Layer).count()
def dehydrate_maps_count(self, bundle):
return bundle.obj.user.resourcebase_set.instance_of(Map).count()
def dehydrate_documents_count(self, bundle):
return bundle.obj.user.resourcebase_set.instance_of(Document).count()
def dehydrate_avatar_100(self, bundle):
return avatar_url(bundle.obj.user, 100)
def dehydrate_profile_detail_url(self, bundle):
return bundle.obj.get_absolute_url()
def dehydrate_current_user(self, bundle):
return bundle.request.user.username == bundle.obj.user.username
def dehydrate_activity_stream_url(self, bundle):
return reverse('actstream_actor', kwargs={
'content_type_id': ContentType.objects.get_for_model(bundle.obj.user).pk,
'object_id': bundle.obj.user.pk})
class Meta:
queryset = Profile.objects.all()
resource_name = 'profiles'
allowed_methods = ['get',]
ordering = ['user','name']
class ConfigResource(Resource):
"""
The resource for configuration data to be sent to the front end
so that site config only need be set in 1 location (the django app).
"""
uuid = fields.CharField(attribute='uuid')
application_name = fields.CharField(attribute='application_name')
application_tag_line = fields.CharField(attribute='application_tag_line')
front_end_url = fields.CharField(attribute='front_end_url')
login_url = fields.CharField(attribute='login_url')
login_redirect_url = fields.CharField(attribute='login_redirect_url')
show_latest_idea_count = fields.CharField(
attribute='show_latest_idea_count')
show_latest_project_count = fields.CharField(
attribute='show_latest_project_count')
admin_name = fields.CharField(attribute='admin_name')
admin_email = fields.CharField(attribute='admin_email')
admin_phone = fields.CharField(attribute='admin_phone')
class Meta:
"""
The basics for a tastypie resource
"""
resource_name = 'config'
object_class = ConfigObject
serializer = CustomSerializer()
allowed_methods = ('get')
authentication = Authentication()
authorization = Authorization()
#filtering
#ordering
# ------------------------------------------------------------------------------------------------------------
def determine_format(self, request):
""" Override the default format, so that format=json is not required """
content_types = {
'json': 'application/json',
'jsonp': 'text/javascript',
'xml': 'application/xml',
'yaml': 'text/yaml'
}
format = request.GET.get('format', None)
if format == None:
return 'application/json'
else:
return content_types[format]
# ------------------------------------------------------------------------------------------------------------
def detail_uri_kwargs(self, bundle_or_obj):
""" Grab the id of the specific resource """
kwargs = {}
if isinstance(bundle_or_obj, Bundle):
kwargs['pk'] = bundle_or_obj.obj.uuid
else:
kwargs['pk'] = bundle_or_obj.uuid
return kwargs
# ------------------------------------------------------------------------------------------------------------
def get_object_list(self, request):
""" Returns a list of results"""
# This setting should be globally available from the settings import
try:
api_settings = settings.API_SETTINGS
except:
error_text = 'Failed to access settings to be used in config API. \n'
error_text += 'Local config file needs a setting named CONFIG_API_INCLUDE which contains a list '
error_text += 'of the settings to be shared via the API.'
logging.error(error_text)
raise NotFound(
"Settings for config API not available. Administrator required to check settings."
)
new_obj = ConfigObject(initial=api_settings)
new_obj.uuid = 'api_settings'
results = [new_obj]
return results
# ------------------------------------------------------------------------------------------------------------
def obj_get_list(self, bundle, **kwargs):
""" Get the list """
# Filtering disabled for brevity...
return self.get_object_list(bundle.request)
class QGISStyleResource(ModelResource):
"""Styles API for QGIS Server backend."""
body = fields.CharField(attribute='body', use_in='detail')
name = fields.CharField(attribute='name')
title = fields.CharField(attribute='title')
layer = fields.ForeignKey('geonode.api.resourcebase_api.LayerResource',
attribute='layer',
null=True)
style_url = fields.CharField(attribute='style_url')
type = fields.CharField(attribute='type')
class Meta:
paginator_class = CrossSiteXHRPaginator
queryset = QGISServerStyle.objects.all()
resource_name = 'styles'
detail_uri_name = 'id'
allowed_methods = ['get', 'post', 'delete']
authorization = GeoNodeStyleAuthorization()
filtering = {
'id': ALL,
'title': ALL,
'name': ALL,
'layer': ALL_WITH_RELATIONS
}
def populate_object(self, style):
"""Populate results with necessary fields
:param style: Style objects
:type style: QGISServerStyle
:return:
"""
try:
qgis_layer = style.layer_styles.first()
""":type: geonode.qgis_server.QGISServerLayer"""
style.layer = qgis_layer.layer
style.type = 'qml'
except BaseException:
pass
return style
def build_filters(self, filters=None, **kwargs):
"""Apply custom filters for layer."""
filters = super(QGISStyleResource,
self).build_filters(filters, **kwargs)
# Convert layer__ filters into layer_styles__layer__
updated_filters = {}
for key, value in filters.iteritems():
key = key.replace('layer__', 'layer_styles__layer__')
updated_filters[key] = value
return updated_filters
def build_bundle(self, obj=None, data=None, request=None, **kwargs):
"""Override build_bundle method to add additional info."""
if obj is None and self._meta.object_class:
obj = self._meta.object_class()
elif obj:
obj = self.populate_object(obj)
return Bundle(obj=obj, data=data, request=request, **kwargs)
def post_list(self, request, **kwargs):
"""Attempt to redirect to QGIS Server Style management.
A post method should have the following field:
name: Slug name of style
title: Title of style
style: the style file uploaded
Also, should have kwargs:
layername or layer__name: The layer name associated with the style
or
layer__id: The layer id associated with the style
"""
from geonode.qgis_server.views import qml_style
# Extract layer name information
POST = request.POST
FILES = request.FILES
layername = POST.get('layername') or POST.get('layer__name')
if not layername:
layer_id = POST.get('layer__id')
layer = Layer.objects.get(id=layer_id)
layername = layer.name
# move style file
FILES['qml'] = FILES['style']
response = qml_style(request, layername)
if isinstance(response, TemplateResponse):
if response.status_code == 201:
obj = QGISServerStyle.objects.get(
layer_styles__layer__name=layername, name=POST['name'])
updated_bundle = self.build_bundle(obj=obj, request=request)
location = self.get_resource_uri(updated_bundle)
if not self._meta.always_return_data:
return http.HttpCreated(location=location)
else:
updated_bundle = self.full_dehydrate(updated_bundle)
updated_bundle = self.alter_detail_data_to_serialize(
request, updated_bundle)
return self.create_response(
request,
updated_bundle,
response_class=http.HttpCreated,
location=location)
else:
context = response.context_data
# Check form valid
style_upload_form = context['style_upload_form']
if not style_upload_form.is_valid():
raise BadRequest(style_upload_form.errors.as_text())
alert_message = context['alert_message']
raise BadRequest(alert_message)
elif isinstance(response, HttpResponse):
response_class = None
if response.status_code == 403:
response_class = http.HttpForbidden
return self.error_response(request,
response.content,
response_class=response_class)
def delete_detail(self, request, **kwargs):
"""Attempt to redirect to QGIS Server Style management."""
from geonode.qgis_server.views import qml_style
style_id = kwargs.get('id')
qgis_style = QGISServerStyle.objects.get(id=style_id)
layername = qgis_style.layer_styles.first().layer.name
response = qml_style(request, layername, style_name=qgis_style.name)
if isinstance(response, TemplateResponse):
if response.status_code == 200:
# style deleted
return http.HttpNoContent()
else:
context = response.context_data
# Check form valid
style_upload_form = context['style_upload_form']
if not style_upload_form.is_valid():
raise BadRequest(style_upload_form.errors.as_text())
alert_message = context['alert_message']
raise BadRequest(alert_message)
elif isinstance(response, HttpResponse):
response_class = None
if response.status_code == 403:
response_class = http.HttpForbidden
return self.error_response(request,
response.content,
response_class=response_class)
def delete_list(self, request, **kwargs):
"""Do not allow delete list"""
return http.HttpForbidden()
class JobResource(ChromaModelResource):
"""
Jobs refer to individual units of work that the server is doing. Jobs
may either run as part of a Command, or on their own. Jobs which are necessary
to the completion of more than one command may belong to more than one command.
For example:
* a Command to start a filesystem has a Job for starting each OST.
* a Command to setup an OST has a series of Jobs for formatting, registering etc
Jobs which are part of the same command may run in parallel to one another.
The lock objects in the ``read_locks`` and ``write_locks`` fields have the
following form:
::
{
id: "1",
locked_item_id: 2,
locked_item_content_type_id: 4,
}
The ``id`` and ``content_type_id`` of the locked object form a unique identifier
which can be compared with API-readable objects which have such attributes.
"""
description = fields.CharField(help_text="Human readable string around\
one sentence long describing what the job is doing")
wait_for = fields.ListField(
'wait_for',
null=True,
help_text=
"List of other jobs which must complete before this job can run")
read_locks = fields.ListField(
null=True,
help_text="List of objects which must stay in the required state while\
this job runs")
write_locks = fields.ListField(
null=True,
help_text="List of objects which must be in a certain state for\
this job to run, and may be modified by this job while it runs.")
commands = fields.ToManyField(
'chroma_api.command.CommandResource',
lambda bundle: bundle.obj.command_set.all(),
null=True,
help_text="Commands which require this job to complete\
sucessfully in order to succeed themselves")
steps = fields.ToManyField('chroma_api.step.StepResource',
lambda bundle: bundle.obj.stepresult_set.all(),
null=True,
help_text="Steps executed within this job")
step_results = fields.DictField(help_text="List of step results")
class_name = fields.CharField(help_text="Internal class name of job")
available_transitions = fields.DictField()
def _dehydrate_locks(self, bundle, write):
if bundle.obj.locks_json:
locks = json.loads(bundle.obj.locks_json)
locks = [
StateLock.from_dict(bundle.obj, lock) for lock in locks
if lock['write'] == write
]
slr = StateLockResource()
return [
slr.full_dehydrate(slr.build_bundle(obj=l)).data for l in locks
]
else:
return []
def dehydrate_wait_for(self, bundle):
if not bundle.obj.wait_for_json:
return []
else:
wait_fors = json.loads(bundle.obj.wait_for_json)
return [
JobResource().get_resource_uri(Job.objects.get(pk=i))
for i in wait_fors
]
def dehydrate_read_locks(self, bundle):
return self._dehydrate_locks(bundle, write=False)
def dehydrate_write_locks(self, bundle):
return self._dehydrate_locks(bundle, write=True)
def dehydrate_class_name(self, bundle):
return bundle.obj.content_type.model_class().__name__
def dehydrate_available_transitions(self, bundle):
job = bundle.obj.downcast()
if job.state == 'complete' or not job.cancellable:
return []
elif job.cancellable:
return [{'state': 'cancelled', 'label': 'Cancel'}]
def dehydrate_step_results(self, bundle):
result = {}
for step_result in bundle.obj.stepresult_set.all():
result[StepResource().get_resource_uri(step_result)] = json.loads(
step_result.result) if step_result.result else None
return result
def dehydrate_description(self, bundle):
return bundle.obj.downcast().description()
class Meta:
queryset = Job.objects.all()
resource_name = 'job'
authorization = DjangoAuthorization()
authentication = AnonymousAuthentication()
excludes = ['task_id', 'locks_json', 'wait_for_json']
ordering = ['created_at']
list_allowed_methods = ['get']
detail_allowed_methods = ['get', 'put']
filtering = {'id': ['exact', 'in'], 'state': ['exact', 'in']}
always_return_data = True
validation = JobValidation()
@validate
def obj_update(self, bundle, **kwargs):
job = Job.objects.get(pk=kwargs['pk'])
new_state = bundle.data['state']
if new_state == 'cancelled':
JobSchedulerClient.cancel_job(job.pk)
Job.objects.get(pk=kwargs['pk'])
bundle.obj = job
return bundle
class MemberResource(BaseResource):
''' The Parliament Member API '''
class Meta(BaseResource.Meta):
queryset = Member.objects.exclude(
current_party__isnull=True).select_related('current_party')
allowed_methods = ['get']
ordering = [
'name',
'is_current',
'bills_stats_proposed',
'bills_stats_pre',
'bills_stats_first',
'bills_stats_approved',
]
filtering = dict(
name=ALL,
is_current=ALL,
)
excludes = ['website', 'backlinks_enabled', 'area_of_residence']
list_fields = ['name', 'id', 'img_url', 'is_current']
include_absolute_url = True
party_name = fields.CharField()
party_url = fields.CharField()
mmms_count = fields.IntegerField(null=True)
votes_count = fields.IntegerField(null=True)
video_about = fields.ToManyField(
VideoResource,
attribute=lambda b: get_videos_queryset(b.obj, group='about'),
null=True,
full=True)
videos_related = fields.ToManyField(
VideoResource,
attribute=lambda b: get_videos_queryset(b.obj, group='related'),
null=True)
links = fields.ToManyField(
LinkResource,
attribute=lambda b: Link.objects.for_model(b.obj),
full=True,
null=True)
bills_uri = fields.CharField()
agendas_uri = fields.CharField()
committees = fields.ListField()
detailed_roles = fields.ToManyField(
RoleResource,
attribute=lambda b: Person.objects.get(mk=b.obj).roles.all(),
full=True,
null=True)
fields.ToOneField(PartyResource, 'current_party', full=True)
average_weekly_presence_rank = fields.IntegerField()
def obj_get_list(self, bundle, **kwargs):
simple = super(MemberResource, self).obj_get_list(bundle, **kwargs)
if hasattr(bundle.request, 'GET'):
# Grab a mutable copy.
filters = bundle.request.GET.copy()
# Update with the provided kwargs.
filters.update(kwargs)
name = filters.get('name')
if name and not simple:
try:
return Member.objects.filter(person__aliases__name=name)
except PersonAlias.DoesNotExist:
return simple
return simple
def dehydrate_committees(self, bundle):
temp_list = bundle.obj.committee_meetings.exclude(
committee__type='plenum')
temp_list = temp_list.values("committee", "committee__name").annotate(
Count("id")).order_by('-id__count')[:5]
return (map(
lambda item:
(item['committee__name'],
reverse('committee-detail', args=[item['committee']])),
temp_list))
def dehydrate_bills_uri(self, bundle):
return '%s?%s' % (reverse(
'api_dispatch_list',
kwargs={
'resource_name': 'bill',
'api_name': 'v2',
}), urllib.urlencode(dict(proposer=bundle.obj.id)))
def dehydrate_gender(self, bundle):
return bundle.obj.get_gender_display()
def dehydrate_agendas_uri(self, bundle):
return reverse('api_dispatch_detail',
kwargs={
'resource_name': 'member-agendas',
'api_name': 'v2',
'pk': bundle.obj.id
})
def dehydrate_party_name(self, bundle):
party = bundle.obj.current_party
return party.name if party else None
def dehydrate_party_url(self, bundle):
party = bundle.obj.current_party
return party.get_absolute_url() if party else None
def dehydrate_mmms_count(self, bundle):
_cache_key = 'api_v2_member_mmms_' + str(bundle.obj.pk)
count = cache.get(_cache_key)
if count is None:
count = bundle.obj.mmm_documents.count()
cache.set(_cache_key, count, 24 * 3600)
return count
def dehydrate_votes_count(self, bundle):
_cache_key = 'api_v2_member_votes_' + str(bundle.obj.pk)
count = cache.get(_cache_key)
if count is None:
count = bundle.obj.votes.count()
cache.set(_cache_key, count, 24 * 3600)
return count
def dehydrate_average_weekly_presence_rank(self, bundle):
''' Calculate the distribution of presence and place the user on a 5 level scale '''
SCALE = 5
member = bundle.obj
rel_location = cache.get('average_presence_location_%d' % member.id)
if not rel_location:
presence_list = sorted(
map(lambda member: member.average_weekly_presence_hours,
Member.objects.all()))
presence_groups = int(math.ceil(len(presence_list) / float(SCALE)))
# Generate cache for all members
for mk in Member.objects.all():
avg = mk.average_weekly_presence_hours
if avg:
mk_location = 1 + (presence_list.index(avg) /
presence_groups)
else:
mk_location = 0
cache.set('average_presence_location_%d' % mk.id, mk_location,
60 * 60 * 24)
if mk.id == member.id:
rel_location = mk_location
return rel_location
def build_filters(self, filters=None):
if filters is None:
filters = {}
try:
knesset = int(filters.get('knesset', 0))
except KeyError:
knesset = 0
orm_filters = super(MemberResource, self).build_filters(filters)
if knesset:
knesset = Knesset.objects.get(number=knesset)
orm_filters['parties__knesset'] = knesset
return orm_filters
class TaskFlowInstanceResource(GCloudModelResource):
business = fields.ForeignKey(BusinessResource, 'business', full=True)
pipeline_instance = fields.ForeignKey(PipelineInstanceResource,
'pipeline_instance')
name = fields.CharField(attribute='name', readonly=True, null=True)
instance_id = fields.IntegerField(attribute='instance_id', readonly=True)
category_name = fields.CharField(attribute='category_name', readonly=True)
create_time = fields.DateTimeField(attribute='create_time',
readonly=True,
null=True)
start_time = fields.DateTimeField(attribute='start_time',
readonly=True,
null=True)
finish_time = fields.DateTimeField(attribute='finish_time',
readonly=True,
null=True)
elapsed_time = fields.IntegerField(attribute='elapsed_time', readonly=True)
is_started = fields.BooleanField(attribute='is_started',
readonly=True,
null=True)
is_finished = fields.BooleanField(attribute='is_finished',
readonly=True,
null=True)
creator_name = fields.CharField(attribute='creator_name',
readonly=True,
null=True)
executor_name = fields.CharField(attribute='executor_name',
readonly=True,
null=True)
pipeline_tree = fields.DictField(attribute='pipeline_tree',
use_in='detail',
readonly=True,
null=True)
subprocess_info = fields.DictField(attribute='subprocess_info',
use_in='detail',
readonly=True)
class Meta:
queryset = TaskFlowInstance.objects.filter(
pipeline_instance__isnull=False, is_deleted=False)
resource_name = 'taskflow'
authorization = TaskflowAuthorization()
always_return_data = True
serializer = AppSerializer()
filtering = {
"id": ALL,
"business": ALL_WITH_RELATIONS,
"name": ALL,
"category": ALL,
"create_method": ALL,
"create_info": ALL,
"template_source": ALL,
"template_id": ALL,
"pipeline_instance": ALL_WITH_RELATIONS,
}
q_fields = ["id", "pipeline_instance__name"]
limit = 0
@staticmethod
def handle_task_name_attr(data):
data['name'] = name_handler(data['name'], TASK_NAME_MAX_LENGTH)
pipeline_node_name_handle(data['pipeline_tree'])
def dehydrate_pipeline_tree(self, bundle):
return json.dumps(bundle.data['pipeline_tree'])
def obj_create(self, bundle, **kwargs):
model = bundle.obj.__class__
try:
template_id = bundle.data['template_id']
template_source = bundle.data.get('template_source', 'business')
creator = bundle.request.user.username
pipeline_instance_kwargs = {
'name': bundle.data.pop('name'),
'creator': creator,
'pipeline_tree': json.loads(bundle.data.pop('pipeline_tree')),
}
if 'description' in bundle.data:
pipeline_instance_kwargs['description'] = bundle.data.pop(
'description')
except (KeyError, ValueError) as e:
raise BadRequest(e.message)
# XSS handle
self.handle_task_name_attr(pipeline_instance_kwargs)
# validate pipeline tree
try:
validate_web_pipeline_tree(
pipeline_instance_kwargs['pipeline_tree'])
except PipelineException as e:
raise BadRequest(e.message)
if template_source == 'business':
try:
template = TaskTemplate.objects.get(pk=template_id)
except TaskTemplate.DoesNotExist:
raise BadRequest('template[pk=%s] does not exist' %
template_id)
else:
try:
template = CommonTemplate.objects.get(pk=str(template_id),
is_deleted=False)
except CommonTemplate.DoesNotExist:
raise BadRequest('common template[pk=%s] does not exist' %
template_id)
try:
pipeline_instance = model.objects.__class__.create_pipeline_instance(
template, **pipeline_instance_kwargs)
except PipelineException as e:
raise BadRequest(e.message)
kwargs['category'] = template.category
if bundle.data['flow_type'] == 'common_func':
kwargs['current_flow'] = 'func_claim'
else:
kwargs['current_flow'] = 'execute_task'
kwargs['pipeline_instance_id'] = pipeline_instance.id
super(TaskFlowInstanceResource, self).obj_create(bundle, **kwargs)
return bundle
class FeedbackResource(PotatoCaptchaResource, CORSResource,
MarketplaceResource):
feedback = fields.CharField(attribute='feedback')
platform = fields.CharField(attribute='platform', null=True)
chromeless = fields.CharField(attribute='chromeless', null=True)
from_url = fields.CharField(attribute='from_url', null=True)
user = fields.CharField(attribute='user', null=True)
user_agent = fields.CharField(attribute='user_agent', blank=True)
ip_address = fields.CharField(attribute='ip_address', blank=True)
class Meta(MarketplaceResource.Meta):
resource_name = 'feedback'
always_return_data = True
list_allowed_methods = ['post']
detail_allowed_methods = []
authentication = OptionalOAuthAuthentication()
authorization = Authorization()
object_class = GenericObject
include_resource_uri = False
throttle = CacheThrottle(throttle_at=30)
def _send_email(self, bundle):
"""
Send feedback email from the valid bundle.
"""
user = bundle.data.get('user')
sender = getattr(user, 'email', settings.NOBODY_EMAIL)
send_mail_jinja(u'Marketplace Feedback',
'account/email/feedback.txt',
bundle.data,
from_email=sender,
recipient_list=[settings.MKT_FEEDBACK_EMAIL])
def hydrate(self, bundle):
"""
Add the authenticated user to the bundle.
"""
if 'platform' not in bundle.data:
bundle.data['platform'] = bundle.request.GET.get('dev', '')
bundle.data.update({
'user':
bundle.request.amo_user,
'user_agent':
bundle.request.META.get('HTTP_USER_AGENT', ''),
'ip_address':
bundle.request.META.get('REMOTE_ADDR', '')
})
return bundle
def dehydrate(self, bundle):
"""
Strip the `user_agent` and `ip_address` fields before presenting to the
consumer.
"""
del bundle.data['user_agent']
del bundle.data['ip_address']
return bundle
def get_resource_uri(self, bundle_or_obj=None):
"""
Noop needed to prevent NotImplementedError.
"""
return ''
def obj_create(self, bundle, request=None, **kwargs):
bundle.obj = self._meta.object_class(**kwargs)
bundle = self.full_hydrate(bundle)
form = FeedbackForm(bundle.data, request=request)
if not form.is_valid():
raise self.form_errors(form)
self._send_email(bundle)
return bundle
class FileItemResource(Resource):
name = fields.CharField(attribute='name')
class Meta:
resource_name = 'fileservice'
object_class = FileItem
fields = ['name']
include_resource_uri = False
allowed_methods = ['get', 'post', 'put']
list_allowed_methods = ['post']
always_return_data = True
authentication = SessionAuthentication()
authorization = Authorization()
def determine_format(self, request):
return 'application/json'
@staticmethod
def get_file_items():
file_names = helpers.get_fileservice_files()
file_items = []
for name in file_names:
file_item = FileItem()
file_item.name = name
file_items.append(file_item)
return file_items
@staticmethod
def get_file_item(kwargs):
if 'name' in kwargs:
return FileItemResource.get_file_item_by_name(kwargs['name'])
elif 'pk' in kwargs:
return FileItemResource.get_file_items()[int(kwargs['pk'])]
return None
@staticmethod
def get_file_item_by_name(name):
file_items = FileItemResource.get_file_items()
for file_item in file_items:
if file_item.name == helpers.u_to_str(name):
return file_item
def deserialize(self, request, data, format=None):
if not format:
format = request.META.get('CONTENT_TYPE', 'application/json')
if format == 'application/x-www-form-urlencoded':
return request.POST
if format.startswith('multipart'):
data = request.POST.copy()
data.update(request.FILES)
return data
return super(Resource, self).deserialize(request, data, format)
def detail_uri_kwargs(self, bundle_or_obj):
if isinstance(bundle_or_obj, Bundle):
return {'name': bundle_or_obj.obj.name}
else:
return {'name': bundle_or_obj.name}
def get_object_list(self, request):
# inner get of object list... this is where you'll need to
# fetch the data from what ever data source
return FileItemResource.get_file_items()
def obj_get_list(self, request=None, **kwargs):
# outer get of object list... this calls get_object_list and
# could be a point at which additional filtering may be applied
return self.get_object_list(request)
def obj_get(self, request=None, **kwargs):
# get one object from data source
file_item = FileItemResource.get_file_item(kwargs)
# if not file_item: raise NotFound("Object not found")
return file_item
def obj_create(self, bundle, request=None, **kwargs):
# create a new File
bundle.obj = FileItem()
# full_hydrate does the heavy lifting mapping the
# POST-ed payload key/values to object attribute/values
bundle = self.full_hydrate(bundle)
filename_name, file_extension = os.path.splitext(
bundle.data[u'file'].name)
# -- only allow uploading of files of types specified
# in FILESERVICE_CONFIG.types_allowed
types_allowed = helpers.get_fileservice_whitelist()
if '*' not in types_allowed and file_extension.lower() \
not in types_allowed:
raise BadRequest('file type is not whitelisted in '
'FILESERVICE_CONFIG.types_allowed')
file_data = bundle.data[u'file'].read()
# TODO: support optional unique name generation from sha1 and uuid.
# file_sha1 = hashlib.sha1(file_data).hexdigest()
# is file_data only the bytes without filename etc?
# if file_extension:
# filename_name = '{}{}'.format(file_sha1, file_extension)
# else:
# filename_name = file_sha1
# TODO: if the filename uploaded is not a valid sha1,
# warn that it should at least be unique.
bundle.obj.name = bundle.data[u'file'].name
with open(helpers.get_filename_absolute(bundle.data[u'file'].name),
'wb+') as destination_file:
destination_file.write(file_data)
# remove the file object passed in so that the response is
# more concise about what this file will be referred to
bundle.data.pop(u'file', None)
return bundle
def prepend_urls(self):
""" Add the following array of urls to the resource base urls """
view_action = 'download'
if helpers.get_streaming_supported():
view_action = 'view'
return [
# --- download
url(r"^(?P<resource_name>%s)/download/(?P<name>[\w\d_.-]+)%s$" %
(self._meta.resource_name, trailing_slash()),
self.wrap_view('download'),
name="api_fileitem_download"),
url(r"^(?P<resource_name>%s)/(?P<pk>\w[\w/-]*)/download%s$" %
(self._meta.resource_name, trailing_slash()),
self.wrap_view('download'),
name="api_fileitem_download"),
url(r"^(?P<resource_name>%s)/(?P<name>[\w\d_.-]+)/download%s$" %
(self._meta.resource_name, trailing_slash()),
self.wrap_view('download'),
name="api_fileitem_download"),
# --- view
url(r"^(?P<resource_name>%s)/view/(?P<name>[\w\d_.-]+)%s$" %
(self._meta.resource_name, trailing_slash()),
self.wrap_view(view_action),
name='api_fileitem_{}'.format(view_action)),
url(r"^(?P<resource_name>%s)/(?P<pk>\w[\w/-]*)/view%s$" %
(self._meta.resource_name, trailing_slash()),
self.wrap_view(view_action),
name='api_fileitem_{}'.format(view_action)),
url(r"^(?P<resource_name>%s)/(?P<name>[\w\d_.-]+)/view%s$" %
(self._meta.resource_name, trailing_slash()),
self.wrap_view(view_action),
name='api_fileitem_{}'.format(view_action)),
# --- dispatch
url(r"^(?P<resource_name>%s)/(?P<name>[\w\d_.-]+)/$" %
self._meta.resource_name,
self.wrap_view('dispatch_detail'),
name="api_dispatch_detail_name"),
url(r"^(?P<resource_name>%s)/(?P<id>[\d]+)/$" %
self._meta.resource_name,
self.wrap_view('dispatch_detail'),
name="api_dispatch_detail"),
]
def download(self, request, **kwargs):
'''
example use:
http://.../fileservice/download/med.mp4/
or
http://.../fileservice/med.mp4/download/
'''
# method check to avoid bad requests
self.method_check(request, allowed=['get'])
# Must be done otherwise endpoint will be wide open
self.is_authenticated(request)
response = None
file_item_name = kwargs.get('name', None)
if file_item_name:
filename_absolute = helpers.get_filename_absolute(file_item_name)
if os.path.isfile(filename_absolute):
response = serve(request, os.path.basename(filename_absolute),
os.path.dirname(filename_absolute))
response['Content-Disposition'] = 'attachment; ' \
'filename="{}"'.format(
os.path.basename(filename_absolute))
if not response:
response = self.create_response(request=request,
data={},
response_class=HttpNotFound)
return response
def view(self, request, **kwargs):
'''
allow a file to be viewed as opposed to download. This is particularly
needed when a video file is stored
in the fileservice and user wants to be able to use a view the
video as opposed to having to download it
first. It passes the serving of the file to nginx/apache which will
return all the proper headers allowing,
say, html5's video viewer's 'seek' indicator/knob to work. Otherwise
the video is only played sequentially
Note that nginx/apache need to be configured accordingly. nginx
for example:
location /var/lib/geoserver_data/file-service-store/ {
# forces requests to be authorized
internal;
alias /var/lib/geoserver_data/file-service-store/;
}
for apache, need to install xsendfile module, enable it, set
the path and then
XSendFile on
XSendFilePath /var/lib/geoserver_data/file-service-store
example use:
/fileservice/view/med.mp4
or
/fileservice/med.mp4/view
Note that media players tend to require the route to end with the
filename like /fileservice/view/med.mp4
'''
# method check to avoid bad requests
self.method_check(request, allowed=['get'])
# Must be done otherwise endpoint will be wide open
self.is_authenticated(request)
file_item_name = kwargs.get('name', None)
url = urllib.pathname2url(file_item_name)
mime = MimeTypes()
mime_type = mime.guess_type(url)
response = HttpResponse(content_type=mime_type[0])
file_with_route = smart_str('{}{}'.format(
helpers.get_fileservice_dir(), file_item_name))
# apache header
response['X-Sendfile'] = file_with_route
# nginx header
response['X-Accel-Redirect'] = file_with_route
return response
class BaseApplicationResource(OpeniResource):
category = fields.CharField(attribute='category', null=True, blank=True)
class Meta(GenericMeta):
resource_name = "Application"
object_class = ApplicationModel
class GeoserverStyleResource(ModelResource):
"""Styles API for Geoserver backend."""
body = fields.CharField(attribute='sld_body', use_in='detail')
name = fields.CharField(attribute='name')
title = fields.CharField(attribute='sld_title')
# layer_default_style is polymorphic, so it will have many to many
# relation
layer = fields.ManyToManyField(
'geonode.api.resourcebase_api.LayerResource',
attribute='layer_default_style',
null=True)
version = fields.CharField(attribute='sld_version', null=True, blank=True)
style_url = fields.CharField(attribute='sld_url')
workspace = fields.CharField(attribute='workspace', null=True)
type = fields.CharField(attribute='type')
class Meta:
paginator_class = CrossSiteXHRPaginator
queryset = Style.objects.all()
resource_name = 'styles'
detail_uri_name = 'id'
authorization = GeoNodeStyleAuthorization()
allowed_methods = ['get']
filtering = {
'id': ALL,
'title': ALL,
'name': ALL,
'layer': ALL_WITH_RELATIONS
}
def build_filters(self, filters=None, **kwargs):
"""Apply custom filters for layer."""
filters = super(GeoserverStyleResource,
self).build_filters(filters, **kwargs)
# Convert layer__ filters into layer_styles__layer__
updated_filters = {}
for key, value in filters.iteritems():
key = key.replace('layer__', 'layer_default_style__')
updated_filters[key] = value
return updated_filters
def populate_object(self, style):
"""Populate results with necessary fields
:param style: Style objects
:type style: Style
:return:
"""
style.type = 'sld'
return style
def build_bundle(self, obj=None, data=None, request=None, **kwargs):
"""Override build_bundle method to add additional info."""
if obj is None and self._meta.object_class:
obj = self._meta.object_class()
elif obj:
obj = self.populate_object(obj)
return Bundle(obj=obj, data=data, request=request, **kwargs)
class RatingResource(CORSResource, MarketplaceModelResource):
app = fields.ToOneField(AppResource, 'addon', readonly=True)
user = fields.ToOneField(AccountResource, 'user', readonly=True, full=True)
version = CompatToOneField(None,
'version',
rest='version',
readonly=True,
null=True,
extra_fields=('version', ))
report_spam = fields.CharField()
class Meta(MarketplaceModelResource.Meta):
# Unfortunately, the model class name for ratings is "Review".
queryset = Review.objects.valid()
resource_name = 'rating'
list_allowed_methods = ['get', 'post']
detail_allowed_methods = ['get', 'put', 'delete']
always_return_data = True
authentication = (SharedSecretAuthentication(),
OptionalOAuthAuthentication())
authorization = AnonymousReadOnlyAuthorization()
fields = ['rating', 'body', 'modified', 'created']
filtering = {
'app': ('exact', ),
'user': ('exact', ),
'pk': ('exact', ),
}
ordering = ['created']
def dehydrate(self, bundle):
bundle = super(RatingResource, self).dehydrate(bundle)
if bundle.request.amo_user:
amo_user = bundle.request.amo_user
bundle.data['is_author'] = bundle.obj.user.pk == amo_user.pk
bundle.data['has_flagged'] = (
not bundle.data['is_author']
and bundle.obj.reviewflag_set.filter(user=amo_user).exists())
return bundle
def dehydrate_report_spam(self, bundle):
return self._build_reverse_url('api_post_flag',
kwargs={
'api_name': self._meta.api_name,
'resource_name':
self._meta.resource_name,
'review_id': bundle.obj.pk
})
def _review_data(self, request, app, form):
data = dict(addon_id=app.id,
user_id=request.user.id,
ip_address=request.META.get('REMOTE_ADDR', ''))
if app.is_packaged:
data['version_id'] = app.current_version.id
data.update(**form.cleaned_data)
return data
def get_app(self, ident):
try:
app = Webapp.objects.valid().get(id=ident)
except (Webapp.DoesNotExist, ValueError):
try:
app = Webapp.objects.valid().get(app_slug=ident)
except Webapp.DoesNotExist:
raise self.non_form_errors([('app', 'Invalid app')])
if not app.listed_in(region=REGIONS_DICT[get_region()]):
raise self.non_form_errors([('app', 'Not available in this region')
])
return app
def build_filters(self, filters=None):
"""
If `addon__exact` is a filter and its value cannot be coerced into an
int, assume that it's a slug lookup.
Run the query necessary to determine the app, and substitute the slug
with the PK in the filter so tastypie will continue doing its thing.
"""
built = super(RatingResource, self).build_filters(filters)
if 'addon__exact' in built:
try:
int(built['addon__exact'])
except ValueError:
app = self.get_app(built['addon__exact'])
if app:
built['addon__exact'] = str(app.pk)
if built.get('user__exact', None) == 'mine':
# This is a cheat. Would prefer /mine/ in the URL.
user = get_user()
if not user:
# You must be logged in to use "mine".
raise http_error(http.HttpUnauthorized,
'You must be logged in to access "mine".')
built['user__exact'] = user.pk
return built
def obj_create(self, bundle, request=None, **kwargs):
"""
Handle POST requests to the resource. If the data validates, create a
new Review from bundle data.
"""
form = ReviewForm(bundle.data)
if not form.is_valid():
raise self.form_errors(form)
app = self.get_app(bundle.data['app'])
# Return 409 if the user has already reviewed this app.
qs = self._meta.queryset.filter(addon=app, user=request.user)
if app.is_packaged:
qs = qs.filter(version_id=bundle.data['version'])
if qs.exists():
raise http_error(http.HttpConflict,
'You have already reviewed this app.')
# Return 403 if the user is attempting to review their own app:
if app.has_author(request.user):
raise http_error(http.HttpForbidden,
'You may not review your own app.')
# Return 403 if not a free app and the user hasn't purchased it.
if app.is_premium() and not app.is_purchased(request.amo_user):
raise http_error(
http.HttpForbidden,
"You may not review paid apps you haven't purchased.")
bundle.obj = Review.objects.create(
**self._review_data(request, app, form))
amo.log(amo.LOG.ADD_REVIEW, app, bundle.obj)
log.debug('[Review:%s] Created by user %s ' %
(bundle.obj.id, request.user.id))
record_action('new-review', request, {'app-id': app.id})
return bundle
def obj_update(self, bundle, request, **kwargs):
"""
Handle PUT requests to the resource. If authorized and the data
validates, update the indicated resource with bundle data.
"""
form = ReviewForm(bundle.data)
if not form.is_valid():
raise self.form_errors(form)
if 'app' in bundle.data:
error = ('app', "Cannot update a rating's `app`")
raise self.non_form_errors([error])
sup = super(RatingResource, self).obj_update(bundle, request, **kwargs)
amo.log(amo.LOG.EDIT_REVIEW, bundle.obj.addon, bundle.obj)
log.debug('[Review:%s] Edited by %s' %
(bundle.obj.id, request.user.id))
return sup
def obj_delete(self, request, **kwargs):
obj = self.get_by_resource_or_404(request, **kwargs)
if not (AppOwnerAuthorization().is_authorized(request,
object=obj.addon)
or OwnerAuthorization().is_authorized(request, object=obj) or
PermissionAuthorization('Users', 'Edit').is_authorized(request)
or PermissionAuthorization('Addons',
'Edit').is_authorized(request)):
raise http_error(
http.HttpForbidden,
'You do not have permission to delete this review.')
log.info('Rating %s deleted from addon %s' % (obj.pk, obj.addon.pk))
return super(RatingResource, self).obj_delete(request, **kwargs)
def get_object_list(self, request):
qs = MarketplaceModelResource.get_object_list(self, request)
# Mature regions show only reviews from within that region.
if not request.REGION.adolescent:
qs = qs.filter(client_data__region=request.REGION.id)
return qs
def alter_list_data_to_serialize(self, request, data):
if 'app' in request.GET:
addon = self.get_app(request.GET['app'])
data['info'] = {
'average': addon.average_rating,
'slug': addon.app_slug,
'current_version': addon.current_version.version
}
filters = dict(addon=addon)
if addon.is_packaged:
filters['version'] = addon.current_version
if not request.user.is_anonymous():
filters['user'] = request.user
existing_review = Review.objects.valid().filter(**filters)
if addon.is_premium():
can_rate = addon.has_purchased(request.amo_user)
else:
can_rate = not addon.has_author(request.user)
data['user'] = {
'can_rate': can_rate,
'has_rated': existing_review.exists()
}
else:
data['user'] = None
return data
def override_urls(self):
# Based on 'nested resource' example in tastypie cookbook.
return [
url(r'^(?P<resource_name>%s)/(?P<review_id>\w[\w/-]*)/flag%s$' %
(self._meta.resource_name, trailing_slash()),
self.wrap_view('post_flag'),
name='api_post_flag')
]
def post_flag(self, request, **kwargs):
return RatingFlagResource().dispatch('list',
request,
review_id=kwargs['review_id'])
class UserResource(ChromaModelResource):
"""
A user account
"""
groups = fields.ToManyField(
"chroma_api.group.GroupResource",
attribute="groups",
full=True,
null=True,
readonly=True,
help_text="List of groups that this user is a member of.",
)
alert_subscriptions = fields.ToManyField(
"chroma_api.alert.AlertSubscriptionResource",
attribute="alert_subscriptions",
null=True,
full=True,
help_text="List of alert subscriptions (alerts for which this user"
"will be sent emails. See alert_subscription resource"
"for format",
)
full_name = fields.CharField(
help_text=
"Human readable form derived from ``first_name`` and ``last_name``")
password1 = fields.CharField(
help_text=
"Used when creating a user (request must be made by a superuser)")
password2 = fields.CharField(
help_text="Password confirmation, must match ``password1``")
new_password1 = fields.CharField(
help_text="Used for modifying password (request must be "
"made by the same user or by a superuser)")
new_password2 = fields.CharField(
help_text="Password confirmation, must match ``new_password1``")
is_superuser = fields.BooleanField(readonly=True,
help_text="Is the user a superuser",
attribute="is_superuser")
def alter_deserialized_detail_data(self, request, data):
def handle_groups(group):
if isinstance(group, dict):
return group["resource_uri"]
elif isinstance(group, basestring):
return group
elif isinstance(group, Bundle):
return group
else:
raise NotImplementedError(group.__class__)
if "groups" in data:
data["groups"] = map(handle_groups, data["groups"])
return data
def hydrate_groups(self, bundle):
from chroma_api.group import GroupResource
# Prevent non-superusers from modifying their groups
if not bundle.request.user.is_superuser:
if bundle.data.get("groups") is not None:
group_ids = []
for group in bundle.data["groups"]:
if isinstance(group, dict):
group_ids.append(int(group["id"]))
elif isinstance(group, basestring):
group_ids.append(
int(GroupResource().get_via_uri(
group, bundle.request).id))
elif isinstance(group, Bundle):
group_ids.append(int(group.obj.id))
else:
raise NotImplementedError(group.__class__)
user_group_ids = [
int(group.pk)
for group in bundle.request.user.groups.all()
]
if not set(group_ids) == set(user_group_ids):
raise ImmediateHttpResponse(HttpForbidden())
return bundle
# This seems wrong. Without it, the hydration goes awry with what
# comes in via PUT. We aren't managing user alert subscriptions
# via the User resource, though, so perhaps this is not so bad.
def hydrate_alert_subscriptions(self, bundle):
try:
del bundle.data["alert_subscriptions"][:]
except KeyError:
pass
return bundle
def _hydrate_password(self, bundle, key):
try:
new_password = bundle.data[key]
if new_password:
bundle.obj.set_password(new_password)
except KeyError:
pass
return bundle
def hydrate_password2(self, bundle):
return self._hydrate_password(bundle, "password2")
def hydrate_new_password2(self, bundle):
return self._hydrate_password(bundle, "new_password2")
def obj_create(self, bundle, **kwargs):
bundle = super(UserResource, self).obj_create(bundle, **kwargs)
from django.contrib.auth.models import Group
superuser_group = Group.objects.get(name="superusers")
for g in bundle.obj.groups.all():
if g == superuser_group:
bundle.obj.is_superuser = True
bundle.obj.save()
return bundle
def dehydrate_full_name(self, bundle):
return bundle.obj.get_full_name()
def delete_detail(self, request, **kwargs):
if int(kwargs["pk"]) == request.user.id:
return self.create_response(
request,
{"id": ["Cannot delete currently authenticated user"]},
response_class=HttpBadRequest)
else:
return super(UserResource, self).delete_detail(request, **kwargs)
class Meta:
authentication = AnonymousAuthentication()
authorization = UserAuthorization()
queryset = User.objects.all()
validation = UserValidation()
fields = [
"first_name",
"full_name",
"groups",
"id",
"last_name",
"new_password1",
"new_password2",
"password1",
"password2",
"resource_uri",
"username",
"email",
]
ordering = ["username", "email"]
list_allowed_methods = ["get", "post"]
detail_allowed_methods = ["get", "put", "patch", "delete"]
always_return_data = True
class FaceSearchResource(Resource):
name = fields.CharField(attribute='name')
class Meta:
resource_name = 'facesearchservice'
object_class = FaceSearch
fields = ['name']
include_resource_uri = False
allowed_methods = ['post']
always_return_data = True
authentication = BasicAuthentication()
authorization = Authorization()
def determine_format(self, request):
return 'application/json'
def deserialize(self, request, data, format=None):
if not format:
format = request.META.get('CONTENT_TYPE', 'application/json')
if format == 'application/x-www-form-urlencoded':
return request.POST
if format.startswith('multipart'):
data = request.POST.copy()
data.update(request.FILES)
return data
return super(Resource, self).deserialize(request, data, format)
def detail_uri_kwargs(self, bundle_or_obj):
if isinstance(bundle_or_obj, Bundle):
return {'name': bundle_or_obj.obj.name}
else:
return {'name': bundle_or_obj.name}
def prepend_urls(self):
""" Add the following array of urls to the Tileset base urls """
return [
url(r"^(?P<resource_name>%s)/test_getfilename%s$" %
(self._meta.resource_name, trailing_slash()),
self.wrap_view('test_getfilename'), name="test_getfilename"),
url(r"^(?P<resource_name>%s)/reload_gallery%s$" %
(self._meta.resource_name, trailing_slash()),
self.wrap_view('reload_gallery'), name="reload_gallery"),
]
# This function is a REST end-point to perform a simple test of br_get_filename, which is giving us a problem
# Make sure to have a picture /tmp/testimage1.jpg
def test_getfilename(self, request, **kwargs):
self.method_check(request, allowed=['post'])
res = []
# create a new File
filename_name = u'/tmp/testimage1.jpg'
file_data = open(filename_name, 'rb').read()
facetmpl = br.br_load_img(file_data, len(file_data))
print "Setting file name on template to " + os.path.basename(filename_name)
logger.debug("Setting file name on template to " + os.path.basename(filename_name))
br.br_set_filename(facetmpl, os.path.basename(filename_name))
print "Retrieving file name from template"
logger.debug("Retriving file name from template")
filename = br.br_get_filename(facetmpl)
logger.debug("Filename=" + filename)
print filename
response = self.create_response(request, res)
return response
def reload_gallery(self, request, **kwargs):
reindex_gallery()
galleryGalPath = get_gallery_file()
galGallery = br.br_make_gallery(galleryGalPath)
galTemplateList = br.br_load_from_gallery(galGallery)
# compare and collect scores
ntargets = br.br_num_templates(galTemplateList)
logger.debug(str(ntargets) + " templates found in the gallery at " + galleryGalPath)
res = str(ntargets) + " templates found in the gallery at " + galleryGalPath
for r in range(ntargets):
tmpl = br.br_get_template(galTemplateList, r)
logger.debug("Filename = " + br.br_get_filename(tmpl))
response = self.create_response(request, res)
br.br_free_template_list(galTemplateList)
br.br_close_gallery(galGallery)
return response
def obj_create(self, bundle, request=None, **kwargs):
# create a new File
bundle.obj = FaceSearch()
# full_hydrate does the heavy lifting mapping the
# POST-ed payload key/values to object attribute/values
bundle = self.full_hydrate(bundle)
file_data = bundle.data[u'file'].read()
# write the file data to a temporary file
filename_name, file_extension = os.path.splitext(bundle.data[u'file'].name)
destination_file = tempfile.NamedTemporaryFile(suffix=file_extension)
destination_file.write(bytearray(file_data))
destination_file.flush()
os.fsync(destination_file) # Need this, we were not getting all bytes written to file before proceeding
temp_gal = tempfile.NamedTemporaryFile(suffix='.gal')
args = ['br', '-algorithm', 'FaceRecognition', '-enroll', destination_file.name, temp_gal.name]
subprocess.call(args)
out_file = tempfile.NamedTemporaryFile(suffix='.csv')
galleryGalPath = get_gallery_file()
args = args[:3] + ['-compare', galleryGalPath, temp_gal.name, out_file.name]
subprocess.call(args)
with open(out_file.name, 'r') as scores_file:
files_line = scores_file.readline().strip().split(',')[1:]
scores_line = scores_file.readline().strip().split(',')
probe = scores_line[0]
score_list = scores_line[1:]
scores= []
for f, s in zip(files_line, score_list):
logger.debug('%s -> %s: %s' % (probe, f, s))
scores.append((os.path.basename(f), s))
destination_file.close()
out_file.close()
temp_gal.close()
peeps = Person.objects.filter(pic_filename__in=dict(scores).keys()).values()
sorted_peeps = []
scores.sort(key=lambda s: s[1], reverse=True)
# TODO: Make 15 a parameter - right now we return the top 15 results
logger.debug("Building result package")
for s in scores[:15]:
currfilename = s[0]
logger.debug(currfilename)
foundPeep = filter(lambda p: p['pic_filename'] == os.path.basename(s[0]), peeps)
if foundPeep is not None and len(foundPeep) > 0:
sorted_peeps.append(foundPeep[0])
else:
logger.info("Found picture with no link person " + currfilename)
# bundle the search results
bundle.obj.name = bundle.data[u'file'].name
bundle.data.pop(u'file', None)
bundle.data['meta'] = {
"limit": len(peeps),
"next": None,
"offset": 0,
"previous": None,
"total_count": len(peeps)
}
bundle.data['objects'] = sorted_peeps
bundle.data['scores'] = scores
return bundle
class InvoiceRevisionResource(VosaeResource):
revision = base_fields.CharField(
attribute='revision',
readonly=True,
help_text=HELP_TEXT['invoice_revision']['revision']
)
issue_date = base_fields.DateTimeField(
attribute='issue_date',
readonly=True,
help_text=HELP_TEXT['invoice_revision']['issue_date']
)
sender = base_fields.CharField(
attribute='sender',
null=True,
blank=True,
help_text=HELP_TEXT['invoice_revision']['sender']
)
sender_organization = base_fields.CharField(
attribute='sender_organization',
readonly=True,
null=True,
help_text=HELP_TEXT['invoice_revision']['sender_organization']
)
quotation_date = base_fields.DateField(
attribute='quotation_date',
null=True,
blank=True,
help_text=HELP_TEXT['invoice_revision']['quotation_date']
)
quotation_validity = base_fields.DateField(
attribute='quotation_validity',
null=True,
blank=True,
help_text=HELP_TEXT['invoice_revision']['quotation_validity']
)
purchase_order_date = base_fields.DateField(
attribute='purchase_order_date',
null=True,
blank=True,
help_text=HELP_TEXT['invoice_revision']['purchase_order_date']
)
invoicing_date = base_fields.DateField(
attribute='invoicing_date',
null=True,
blank=True,
help_text=HELP_TEXT['invoice_revision']['invoicing_date']
)
due_date = base_fields.DateField(
attribute='due_date',
null=True,
blank=True,
help_text=HELP_TEXT['invoice_revision']['due_date']
)
credit_note_emission_date = base_fields.DateField(
attribute='credit_note_emission_date',
readonly=True,
null=True,
blank=True,
help_text=HELP_TEXT['invoice_revision']['credit_note_emission_date']
)
custom_payment_conditions = base_fields.CharField(
attribute='custom_payment_conditions',
null=True,
blank=True,
help_text=HELP_TEXT['invoice_revision']['custom_payment_conditions']
)
customer_reference = base_fields.CharField(
attribute='customer_reference',
null=True,
blank=True,
help_text=HELP_TEXT['invoice_revision']['customer_reference']
)
taxes_application = base_fields.CharField(
attribute='taxes_application',
help_text=HELP_TEXT['invoice_revision']['taxes_application']
)
issuer = fields.ReferenceField(
to='core.api.resources.VosaeUserResource',
attribute='issuer',
readonly=True,
null=True,
help_text=HELP_TEXT['invoice_revision']['issuer']
)
sender_address = fields.EmbeddedDocumentField(
embedded='contacts.api.resources.AddressResource',
attribute='sender_address',
null=True,
help_text=HELP_TEXT['invoice_revision']['sender_address']
)
contact = fields.ReferenceField(
to='contacts.api.resources.ContactResource',
attribute='contact',
null=True,
help_text=HELP_TEXT['invoice_revision']['contact']
)
organization = fields.ReferenceField(
to='contacts.api.resources.OrganizationResource',
attribute='organization',
null=True,
help_text=HELP_TEXT['invoice_revision']['organization']
)
billing_address = fields.EmbeddedDocumentField(
embedded='contacts.api.resources.AddressResource',
attribute='billing_address',
null=True,
help_text=HELP_TEXT['invoice_revision']['billing_address']
)
delivery_address = fields.EmbeddedDocumentField(
embedded='contacts.api.resources.AddressResource',
attribute='delivery_address',
null=True,
help_text=HELP_TEXT['invoice_revision']['delivery_address']
)
currency = fields.EmbeddedDocumentField(
embedded='invoicing.api.resources.SnapshotCurrencyResource',
attribute='currency',
help_text=HELP_TEXT['invoice_revision']['currency']
)
line_items = fields.EmbeddedListField(
of='invoicing.api.resources.InvoiceItemResource',
attribute='line_items',
full=True,
null=True,
help_text=HELP_TEXT['invoice_revision']['line_items']
)
pdf = ReferencedDictField(
of='core.api.resources.VosaeFileResource',
attribute='pdf',
readonly=True,
null=True,
help_text=HELP_TEXT['invoice_revision']['pdf']
)
class Meta(VosaeResource.Meta):
object_class = InvoiceRevision
def hydrate(self, bundle):
"""Set issue data and issuer on POST, extracted from request"""
bundle = super(InvoiceRevisionResource, self).hydrate(bundle)
bundle.obj.issuer = bundle.request.vosae_user
bundle.obj.issue_date = datetime_now()
return bundle
class SearchResource(DeprecatedModelResourceWithFieldsFilter):
# Roses to the clever person that makes this introspect the model and
# removes all this code.
absolute_url = fields.CharField(
attribute='absolute_url',
help_text="The URL on CourtListener for the item.",
null=True,
)
case_name = fields.CharField(
attribute='caseName',
help_text="The full name of the case",
null=True,
)
case_number = fields.CharField(
attribute='caseNumber',
help_text="The combination of the citation and the docket number.",
null=True,
)
citation = fields.CharField(
attribute='citation',
help_text="A concatenated list of all the citations for an opinion.",
null=True,
)
cite_count = fields.IntegerField(
attribute='citeCount',
help_text="The number of times this document is cited by other cases",
)
court = fields.CharField(
attribute='court',
help_text="The name of the court where the document was filed",
null=True,
)
court_id = fields.CharField(
attribute='court_id',
help_text='The court where the document was filed',
null=True,
)
date_filed = fields.DateField(
attribute='dateFiled',
help_text='The date filed by the court',
null=True,
)
docket_number = fields.CharField(
attribute='docketNumber',
help_text=
'The docket numbers of a case, can be consolidated and quite long',
null=True,
)
download_url = fields.CharField(
attribute='download_url',
help_text=
'The URL on the court website where the document was originally scraped',
null=True,
)
id = fields.CharField(
attribute='id',
help_text='The primary key for an opinion.',
)
judge = fields.CharField(
attribute='judge',
help_text='The judges that brought the opinion as a simple text string',
null=True,
)
local_path = fields.CharField(
attribute='local_path',
help_text=
'The location, relative to MEDIA_ROOT on the CourtListener server, where files are stored',
null=True,
)
score = fields.FloatField(
attribute='score',
help_text='The relevance of the result. Will vary from query to query.',
)
source = fields.CharField(
attribute='source',
help_text='the source of the document, one of: %s' %
', '.join(['%s (%s)' % (t[0], t[1]) for t in SOURCES]),
null=True,
)
snippet = fields.CharField(
attribute='snippet',
help_text=
'a snippet as found in search results, utilizing <mark> for highlighting and … for ellipses',
null=True,
)
status = fields.CharField(
attribute='status',
help_text='The precedential status of document, one of: %s' %
', '.join([('stat_%s' % t[1]).replace(' ', '+')
for t in DOCUMENT_STATUSES]),
null=True,
)
suit_nature = fields.CharField(
attribute='suitNature',
help_text=
"The nature of the suit. For the moment can be codes or laws or whatever",
null=True,
)
text = fields.CharField(
attribute='text',
use_in='detail', # Only shows on the detail page.
help_text=
"A concatenated copy of most fields in the item so those fields are available for search."
)
timestamp = fields.DateField(
attribute='timestamp',
help_text='The moment when an item was indexed by Solr.')
class Meta:
authentication = authentication.MultiAuthentication(
BasicAuthenticationWithUser(realm="courtlistener.com"),
authentication.SessionAuthentication())
throttle = PerUserCacheThrottle(throttle_at=1000)
resource_name = 'search'
max_limit = 20
include_absolute_url = True
allowed_methods = ['get']
filtering = {
'q': ('search', ),
'case_name': ('search', ),
'judge': ('search', ),
'stat_': ('boolean', ),
'filed_after': ('date', ),
'filed_before': ('date', ),
'citation': ('search', ),
'neutral_cite': ('search', ),
'docket_number': ('search', ),
'cited_gt': ('int', ),
'cited_lt': ('int', ),
'court': ('csv', ),
}
ordering = [
'dateFiled+desc',
'dateFiled+asc',
'citeCount+desc',
'citeCount+asc',
'score+desc',
]
def get_resource_uri(self,
bundle_or_obj=None,
url_name='api_dispatch_list'):
"""Creates a URI like /api/v1/search/$id/
"""
url_str = '/api/rest/%s/%s/%s/'
if bundle_or_obj:
return url_str % (
self.api_name,
'opinion',
bundle_or_obj.obj.id,
)
else:
return ''
def get_object_list(self, request=None, **kwargs):
"""Performs the Solr work."""
main_query = {'caller': 'api_search'}
try:
main_query.update(build_main_query(kwargs['cd'], highlight='text'))
sl = SolrList(main_query=main_query,
offset=request.GET.get('offset', 0),
limit=request.GET.get('limit', 20),
type=kwargs['cd']['type'])
except KeyError:
sf = forms.SearchForm({'q': "*:*"})
if sf.is_valid():
main_query.update(
build_main_query(sf.cleaned_data, highlight='text'))
sl = SolrList(
main_query=main_query,
offset=request.GET.get('offset', 0),
limit=request.GET.get('limit', 20),
)
return sl
def obj_get_list(self, bundle, **kwargs):
search_form = forms.SearchForm(bundle.request.GET)
if search_form.is_valid():
cd = search_form.cleaned_data
if cd['q'] == '':
cd['q'] = '*:*' # Get everything.
return self.get_object_list(bundle.request, cd=cd)
else:
BadRequest("Invalid resource lookup data provided. Unable to "
"complete your query.")
def obj_get(self, bundle, **kwargs):
search_form = forms.SearchForm(bundle.request.GET)
if search_form.is_valid():
cd = search_form.cleaned_data
cd['q'] = 'id:%s' % kwargs['pk']
return self.get_object_list(bundle.request, cd=cd)[0]
else:
BadRequest("Invalid resource lookup data provided. Unable to "
"complete your request.")
def apply_sorting(self, obj_list, options=None):
"""Since we're not using Django Model sorting, we just want to use our
own, which is already passed into the search form anyway.
Thus: Do nothing here.
"""
return obj_list
class ThesaurusKeywordResource(TypeFilteredResource):
"""ThesaurusKeyword api"""
thesaurus_identifier = fields.CharField(null=False)
label_id = fields.CharField(null=False)
def build_filters(self, filters={}, ignore_bad_filters=False):
"""adds filtering by current language"""
_filters = filters.copy()
id = _filters.pop('id', None)
orm_filters = super().build_filters(_filters)
if id is not None:
orm_filters['id__in'] = id
if 'thesaurus' in _filters:
orm_filters['thesaurus__identifier'] = _filters['thesaurus']
return orm_filters
def serialize(self, request, data, format, options={}):
options['count_type'] = 'tkeywords__id'
return super().serialize(request, data, format, options)
def dehydrate_id(self, bundle):
return bundle.obj.id
def dehydrate_label_id(self, bundle):
return bundle.obj.id
def dehydrate_thesaurus_identifier(self, bundle):
return bundle.obj.thesaurus.identifier
def dehydrate(self, bundle):
lang = get_language()
label = ThesaurusKeywordLabel.objects.filter(
keyword=bundle.data['id']).filter(lang=lang)
if label.exists():
bundle.data['label_id'] = label.get().id
bundle.data['label'] = label.get().label
bundle.data['alt_label'] = label.get().label
else:
bundle.data['label'] = bundle.data['alt_label']
return bundle
class Meta:
queryset = ThesaurusKeyword.objects \
.all() \
.order_by('alt_label') \
.select_related('thesaurus')
resource_name = 'thesaurus/keywords'
allowed_methods = ['get']
filtering = {
'id': ALL,
'alt_label': ALL,
'thesaurus': ALL,
}
serializer = CountJSONSerializer()
authorization = ApiLockdownAuthorization()
class CommonTemplateResource(GCloudModelResource):
pipeline_template = fields.ForeignKey(
PipelineTemplateResource,
'pipeline_template')
name = fields.CharField(
attribute='name',
readonly=True,
null=True)
category_name = fields.CharField(
attribute='category_name',
readonly=True,
null=True)
creator_name = fields.CharField(
attribute='creator_name',
readonly=True,
null=True)
editor_name = fields.CharField(
attribute='editor_name',
readonly=True,
null=True)
create_time = fields.DateTimeField(
attribute='create_time',
readonly=True,
null=True)
edit_time = fields.DateTimeField(
attribute='edit_time',
readonly=True,
null=True)
pipeline_tree = fields.DictField(
attribute='pipeline_tree',
use_in='detail',
readonly=True,
null=True)
template_id = fields.IntegerField(
attribute='template_id',
readonly=True)
subprocess_info = fields.DictField(
attribute='subprocess_info',
use_in='detail',
readonly=True
)
version = fields.CharField(
attribute='version',
readonly=True,
null=True
)
subprocess_has_update = fields.BooleanField(
attribute='subprocess_has_update',
use_in='list',
readonly=True
)
class Meta:
queryset = CommonTemplate.objects.filter(pipeline_template__isnull=False, is_deleted=False)
resource_name = 'common_template'
authorization = CommonAuthorization()
always_return_data = True
serializer = AppSerializer()
filtering = {
"id": ALL,
"name": ALL,
"category": ALL,
"pipeline_template": ALL_WITH_RELATIONS,
"subprocess_has_update": ALL,
"has_subprocess": ALL
}
q_fields = ["id", "pipeline_template__name"]
limit = 0
paginator_class = TemplateFilterPaginator
@staticmethod
def handle_template_name_attr(data):
data['name'] = name_handler(data['name'],
TEMPLATE_NODE_NAME_MAX_LENGTH)
pipeline_node_name_handle(data['pipeline_tree'])
def dehydrate_pipeline_tree(self, bundle):
return json.dumps(bundle.data['pipeline_tree'])
def alter_list_data_to_serialize(self, request, data):
user_model = get_user_model()
user = request.user
collected_templates = user_model.objects.get(username=user.username) \
.tasktemplate_set.all() \
.values_list('id', flat=True)
for bundle in data['objects']:
if bundle.obj.id in collected_templates:
bundle.data['is_add'] = 1
else:
bundle.data['is_add'] = 0
return data
def obj_create(self, bundle, **kwargs):
model = bundle.obj.__class__
try:
pipeline_template_kwargs = {
'name': bundle.data.pop('name'),
'creator': bundle.request.user.username,
'pipeline_tree': json.loads(bundle.data.pop('pipeline_tree')),
'description': bundle.data.pop('description', ''),
}
except (KeyError, ValueError) as e:
raise BadRequest(e.message)
# XSS handle
self.handle_template_name_attr(pipeline_template_kwargs)
# validate pipeline tree
try:
validate_web_pipeline_tree(pipeline_template_kwargs['pipeline_tree'])
except PipelineException as e:
raise BadRequest(e.message)
# Note: tastypie won't use model's create method
try:
pipeline_template = model.objects.create_pipeline_template(
**pipeline_template_kwargs)
except PipelineException as e:
raise BadRequest(e.message)
except CommonTemplate.DoesNotExist:
raise BadRequest('flow template referred as SubProcess does not exist')
kwargs['pipeline_template_id'] = pipeline_template.template_id
return super(CommonTemplateResource, self).obj_create(bundle, **kwargs)
def obj_update(self, bundle, skip_errors=False, **kwargs):
obj = bundle.obj
try:
pipeline_template_kwargs = {
'name': bundle.data.pop('name'),
'editor': bundle.request.user.username,
'pipeline_tree': json.loads(bundle.data.pop('pipeline_tree')),
}
if 'description' in bundle.data:
pipeline_template_kwargs['description'] = bundle.data.pop('description')
except (KeyError, ValueError) as e:
raise BadRequest(e.message)
# XSS handle
self.handle_template_name_attr(pipeline_template_kwargs)
try:
obj.update_pipeline_template(**pipeline_template_kwargs)
except PipelineException as e:
raise BadRequest(e.message)
bundle.data['pipeline_template'] = '/api/v3/pipeline_template/%s/' % obj.pipeline_template.pk
return super(CommonTemplateResource, self).obj_update(bundle, **kwargs)
def obj_delete(self, bundle, **kwargs):
try:
common_tmpl = CommonTemplate.objects.get(id=kwargs['pk'], is_deleted=False)
except CommonTemplate.DoesNotExist:
raise NotFound('flow template does not exist')
referencer = common_tmpl.referencer()
if referencer:
flat = ','.join(['%s:%s' % (item['id'], item['name']) for item in referencer])
raise BadRequest('flow template are referenced by other templates[%s], please delete them first' % flat)
result = super(CommonTemplateResource, self).obj_delete(bundle, **kwargs)
if result:
common_tmpl.set_deleted()
return result
def build_filters(self, filters=None, ignore_bad_filters=False):
filters = super(CommonTemplateResource, self).build_filters(filters=filters,
ignore_bad_filters=ignore_bad_filters)
if 'subprocess_has_update__exact' in filters:
filters.pop('subprocess_has_update__exact')
if 'has_subprocess__exact' in filters:
filters.pop('has_subprocess__exact')
return filters
class ProfileResource(TypeFilteredResource):
"""Profile api"""
avatar_100 = fields.CharField(null=True)
profile_detail_url = fields.CharField()
email = fields.CharField(default='')
layers_count = fields.IntegerField(default=0)
maps_count = fields.IntegerField(default=0)
documents_count = fields.IntegerField(default=0)
current_user = fields.BooleanField(default=False)
activity_stream_url = fields.CharField(null=True)
def build_filters(self, filters=None, ignore_bad_filters=False):
"""adds filtering by group functionality"""
if filters is None:
filters = {}
orm_filters = super().build_filters(filters)
if 'group' in filters:
orm_filters['group'] = filters['group']
if 'name__icontains' in filters:
orm_filters['username__icontains'] = filters['name__icontains']
return orm_filters
def apply_filters(self, request, applicable_filters):
"""filter by group if applicable by group functionality"""
group = applicable_filters.pop('group', None)
name = applicable_filters.pop('name__icontains', None)
semi_filtered = super().apply_filters(request, applicable_filters)
if group is not None:
semi_filtered = semi_filtered.filter(
groupmember__group__slug=group)
if name is not None:
semi_filtered = semi_filtered.filter(
profile__first_name__icontains=name)
return semi_filtered
def dehydrate_email(self, bundle):
email = ''
if bundle.request.user.is_superuser:
email = bundle.obj.email
return email
def dehydrate_datasets_count(self, bundle):
obj_with_perms = get_objects_for_user(
bundle.request.user, 'base.view_resourcebase').filter(
polymorphic_ctype__model='dataset')
return bundle.obj.resourcebase_set.filter(id__in=obj_with_perms.values('id')).filter(metadata_only=False)\
.distinct().count()
def dehydrate_maps_count(self, bundle):
obj_with_perms = get_objects_for_user(
bundle.request.user,
'base.view_resourcebase').filter(polymorphic_ctype__model='map')
return bundle.obj.resourcebase_set.filter(id__in=obj_with_perms.values('id')).filter(metadata_only=False)\
.distinct().count()
def dehydrate_documents_count(self, bundle):
obj_with_perms = get_objects_for_user(
bundle.request.user, 'base.view_resourcebase').filter(
polymorphic_ctype__model='document')
return bundle.obj.resourcebase_set.filter(id__in=obj_with_perms.values('id')).filter(metadata_only=False)\
.distinct().count()
def dehydrate_avatar_100(self, bundle):
return avatar_url(bundle.obj, 240)
def dehydrate_profile_detail_url(self, bundle):
return bundle.obj.get_absolute_url()
def dehydrate_current_user(self, bundle):
return bundle.request.user.username == bundle.obj.username
def dehydrate_activity_stream_url(self, bundle):
return reverse('actstream_actor',
kwargs={
'content_type_id':
ContentType.objects.get_for_model(bundle.obj).pk,
'object_id':
bundle.obj.pk
})
def dehydrate(self, bundle):
"""
Protects user's personal information from non staff
"""
is_owner = bundle.request.user == bundle.obj
is_admin = bundle.request.user.is_staff or bundle.request.user.is_superuser
if not (is_owner or is_admin):
bundle.data = dict(
id=bundle.data.get('id', ''),
username=bundle.data.get('username', ''),
first_name=bundle.data.get('first_name', ''),
last_name=bundle.data.get('last_name', ''),
avatar_100=bundle.data.get('avatar_100', ''),
profile_detail_url=bundle.data.get('profile_detail_url', ''),
documents_count=bundle.data.get('documents_count', 0),
maps_count=bundle.data.get('maps_count', 0),
layers_count=bundle.data.get('layers_count', 0),
)
return bundle
def prepend_urls(self):
if settings.HAYSTACK_SEARCH:
return [
url(r"^(?P<resource_name>{})/search{}$".format(
self._meta.resource_name, trailing_slash()),
self.wrap_view('get_search'),
name="api_get_search"),
]
else:
return []
def serialize(self, request, data, format, options=None):
if options is None:
options = {}
options['count_type'] = 'owner'
return super().serialize(request, data, format, options)
class Meta:
queryset = get_user_model().objects.exclude(
Q(username='******') | Q(is_active=False))
resource_name = 'profiles'
allowed_methods = ['get']
ordering = ['username', 'date_joined']
excludes = [
'is_staff', 'password', 'is_superuser', 'is_active', 'last_login'
]
filtering = {
'username': ALL,
}
serializer = CountJSONSerializer()
authorization = ApiLockdownAuthorization()
class BulkUserResource(HqBaseResource, DomainSpecificResourceMixin):
"""
A read-only user data resource based on elasticsearch.
Supported Params: limit offset q fields
"""
type = "bulk-user"
id = fields.CharField(attribute='id', readonly=True, unique=True)
email = fields.CharField(attribute='email')
username = fields.CharField(attribute='username', unique=True)
first_name = fields.CharField(attribute='first_name', null=True)
last_name = fields.CharField(attribute='last_name', null=True)
phone_numbers = fields.ListField(attribute='phone_numbers', null=True)
@staticmethod
def to_obj(user):
'''
Takes a flat dict and returns an object
'''
if '_id' in user:
user['id'] = user.pop('_id')
return namedtuple('user', list(user))(**user)
class Meta(CustomResourceMeta):
authentication = RequirePermissionAuthentication(Permissions.edit_commcare_users)
list_allowed_methods = ['get']
detail_allowed_methods = ['get']
object_class = object
resource_name = 'bulk-user'
def dehydrate(self, bundle):
fields = bundle.request.GET.getlist('fields')
data = {}
if not fields:
return bundle
for field in fields:
data[field] = bundle.data[field]
bundle.data = data
return bundle
def obj_get_list(self, bundle, **kwargs):
request_fields = bundle.request.GET.getlist('fields')
for field in request_fields:
if field not in self.fields:
raise BadRequest('{0} is not a valid field'.format(field))
params = bundle.request.GET
param = lambda p: params.get(p, None)
fields = list(self.fields)
fields.remove('id')
fields.append('_id')
fn = MOCK_BULK_USER_ES or user_es_call
users = fn(
domain=kwargs['domain'],
q=param('q'),
fields=fields,
size=param('limit'),
start_at=param('offset'),
)
return list(map(self.to_obj, users))
def detail_uri_kwargs(self, bundle_or_obj):
return {
'pk': get_obj(bundle_or_obj).id
}
class MemberResource(BaseResource):
''' The Parliament Member API '''
class Meta(BaseResource.Meta):
queryset = Member.objects.exclude(
current_party__isnull=True).select_related('current_party')
allowed_methods = ['get']
ordering = [
'name',
'is_current',
'bills_stats_proposed',
'bills_stats_pre',
'bills_stats_first',
'bills_stats_approved',
]
filtering = dict(
name=ALL,
is_current=ALL,
)
excludes = ['website', 'backlinks_enabled', 'area_of_residence']
list_fields = ['name', 'id', 'img_url', 'is_current']
include_absolute_url = True
party_name = fields.CharField()
party_url = fields.CharField()
mmms_count = fields.IntegerField(null=True)
votes_count = fields.IntegerField(null=True)
video_about = fields.ToManyField(
VideoResource,
attribute=lambda b: get_videos_queryset(b.obj, group='about'),
null=True,
full=True)
videos_related = fields.ToManyField(
VideoResource,
attribute=lambda b: get_videos_queryset(b.obj, group='related'),
null=True)
links = fields.ToManyField(
LinkResource,
attribute=lambda b: Link.objects.for_model(b.obj),
full=True,
null=True)
bills_uri = fields.CharField()
agendas_uri = fields.CharField()
committees = fields.ListField()
def dehydrate_committees(self, bundle):
temp_list = bundle.obj.committee_meetings.values(
"committee",
"committee__name").annotate(Count("id")).order_by('-id__count')[:5]
return (map(
lambda item:
(item['committee__name'],
reverse('committee-detail', args=[item['committee']])),
temp_list))
def dehydrate_bills_uri(self, bundle):
return '%s?%s' % (reverse(
'api_dispatch_list',
kwargs={
'resource_name': 'bill',
'api_name': 'v2',
}), urllib.urlencode(dict(proposer=bundle.obj.id)))
def dehydrate_gender(self, bundle):
return bundle.obj.get_gender_display()
def dehydrate_agendas_uri(self, bundle):
return reverse('api_dispatch_detail',
kwargs={
'resource_name': 'member-agendas',
'api_name': 'v2',
'pk': bundle.obj.id
})
def dehydrate_party_name(self, bundle):
return bundle.obj.current_party.name
def dehydrate_party_url(self, bundle):
return bundle.obj.current_party.get_absolute_url()
def dehydrate_mmms_count(self, bundle):
_cache_key = 'api_v2_member_mmms_' + str(bundle.obj.pk)
count = cache.get(_cache_key)
if count is None:
count = bundle.obj.mmm_documents.count()
cache.set(_cache_key, count, 24 * 3600)
return count
def dehydrate_votes_count(self, bundle):
_cache_key = 'api_v2_member_votes_' + str(bundle.obj.pk)
count = cache.get(_cache_key)
if count is None:
count = bundle.obj.votes.count()
cache.set(_cache_key, count, 24 * 3600)
return count
fields.ToOneField(PartyResource, 'current_party', full=True)
def build_filters(self, filters=None):
if filters is None:
filters = {}
try:
knesset = int(filters.get('knesset', 0))
except KeyError:
knesset = 0
orm_filters = super(MemberResource, self).build_filters(filters)
if knesset:
knesset = Knesset.objects.get(number=knesset)
orm_filters['parties__knesset'] = knesset
return orm_filters
class WFPDocumentResource(ModelResource):
"""Resource for Static Map"""
keywords = fields.ToManyField(TagResource, 'keywords', null=True)
categories = fields.ToManyField(CategoryResource, 'categories', full=True)
regions = fields.ToManyField(RegionResource, 'regions', full=True)
file_size = fields.CharField(attribute='get_file_size', readonly=True)
geonode_page = fields.CharField(attribute='detail_url', readonly=True)
geonode_file = fields.FileField(attribute='doc_file')
thumbnail = fields.CharField(attribute='thumbnail',
readonly=True,
null=True)
is_public = fields.BooleanField(default=True)
class Meta(CommonMetaApi):
queryset = WFPDocument.objects.all().order_by('-date')
resource_name = 'staticmaps'
filtering = {
'title': ALL,
'keywords': ALL_WITH_RELATIONS,
'categories': ALL_WITH_RELATIONS,
'regions': ALL_WITH_RELATIONS,
'date': ALL,
}
excludes = [
'abstract',
'bbox_x0',
'bbox_x1',
'bbox_y0',
'bbox_y1',
'constraints_other',
'csw_anytext',
'csw_insert_date',
'csw_mdsource',
'csw_schema',
'csw_type',
'csw_typename',
'csw_wkt_geometry',
'data_quality_statement',
'date_type',
'distribution_description',
'distribution_url',
'edition',
'extension',
'featured',
'is_published',
'language',
'maintenance_frequency',
'metadata_uploaded',
'metadata_xml',
'owner',
'share_count',
'srid',
'supplemental_information',
'temporal_extent_end',
'temporal_extent_start',
# renamed
'doc_file',
]
def dehydrate_is_public(self, bundle):
anonymous_user = get_anonymous_user()
public_wfpdocs_ids = get_objects_for_user(
anonymous_user,
'base.view_resourcebase').instance_of(WFPDocument).values_list(
'id', flat=True)
return bundle.obj.id in public_wfpdocs_ids
def dehydrate_page_format(self, bundle):
return WFPDocument.FORMAT_CHOICES[bundle.data['page_format']][1]
def dehydrate_orientation(self, bundle):
return WFPDocument.ORIENTATION_CHOICES[bundle.data['orientation']][1]
def dehydrate_thumbnail(self, bundle):
url = urlparse(bundle.obj.thumbnail_url)
return url.path
def build_schema(self):
base_schema = super(WFPDocumentResource, self).build_schema()
for f in self._meta.object_class._meta.fields:
if f.name in base_schema['fields'] and f.choices:
base_schema['fields'][f.name].update({
'choices': f.choices,
})
return base_schema
class MediaResource(ModelResource):
release = fields.ForeignKey('alibrary.api.ReleaseResource',
'release',
null=True,
full=True,
max_depth=2)
artist = fields.ForeignKey('alibrary.api.ArtistResource',
'artist',
null=True,
full=True,
max_depth=2)
message = fields.CharField(attribute='message', null=True)
class Meta:
queryset = Media.objects.order_by('tracknumber').all()
list_allowed_methods = [
'get',
]
detail_allowed_methods = [
'get',
]
resource_name = 'library/track'
detail_uri_name = 'uuid'
excludes = ['updated', 'release__media']
include_absolute_url = True
authentication = MultiAuthentication(ApiKeyAuthentication(),
SessionAuthentication(),
Authentication())
authorization = Authorization()
limit = 50
filtering = {
'created': ['exact', 'range', 'gt', 'gte', 'lt', 'lte'],
'id': ['exact', 'in'],
}
def apply_sorting(self, obj_list, options=None):
sorting = options.get('id__in', None)
if not sorting:
return obj_list
obj_list_sorted = list()
for pk in sorting.split(','):
obj_list_sorted.append(obj_list.get(pk=int(pk)))
return obj_list_sorted
def dehydrate(self, bundle):
obj = bundle.obj
if obj.master:
stream = {
'uri':
reverse_lazy('mediaasset-format',
kwargs={
'media_uuid': bundle.obj.uuid,
'quality': 'default',
'encoding': 'mp3',
}),
}
else:
stream = None
bundle.data['stream'] = stream
bundle.data['duration'] = bundle.obj.get_duration()
bundle.data['waveform_image'] = reverse_lazy('mediaasset-waveform',
kwargs={
'media_uuid':
bundle.obj.uuid,
'type': 'w'
})
bundle.data['spectrogram_image'] = reverse_lazy('mediaasset-waveform',
kwargs={
'media_uuid':
bundle.obj.uuid,
'type':
's'
})
bundle.data['relations'] = relations_for_object(bundle.obj)
# TODO: find a nicer way - disable ops-cache
obj = Media.objects.filter(pk=obj.pk).nocache()[0]
# votes
try:
user_vote = obj.votes.filter(user=bundle.request.user)[0].vote
except (TypeError, IndexError) as e:
user_vote = None
try:
votes = {
'up': obj.total_upvotes,
'down': obj.total_downvotes,
'total': obj.vote_total,
'user': user_vote,
}
except AttributeError as e:
votes = None
bundle.data['votes'] = votes
"""
count playlist usage - #914
"""
bundle.data['playlist_usage'] = len(bundle.obj.get_appearances())
bundle.data['bitrate'] = bundle.obj.bitrate
bundle.data['tags'] = [tag.name for tag in bundle.obj.tags]
"""
TODO: verry hackish and incomplete imnplementation.
label includes are needed for on-air app. should be built more flexible in the future!
"""
if bundle.request.GET.get('includes', None):
includes = bundle.request.GET['includes'].split(',')
if 'label' in includes:
try:
from alibrary.api.labelapi import LabelResource
label = bundle.obj.release.label
label_bundle = LabelResource().build_bundle(
obj=label, request=bundle.request)
label_resource = LabelResource().full_dehydrate(
label_bundle)
except:
label_resource = None
bundle.data['label'] = label_resource
return bundle
def build_filters(self, filters=None):
""" Enable querying by fingerprint """
if filters is None:
filters = {}
orm_filters = super(MediaResource, self).build_filters(filters)
if "code" in filters:
# re-implement fprint based lookups
pass
return orm_filters
def prepend_urls(self):
return [
url(r"^(?P<resource_name>%s)/autocomplete%s$" %
(self._meta.resource_name, trailing_slash()),
self.wrap_view('autocomplete'),
name="alibrary-media_api-autocomplete"),
url(r"^(?P<resource_name>%s)/(?P<uuid>\w[\w/-]*)/vote%s$" %
(self._meta.resource_name, trailing_slash()),
self.wrap_view('vote'),
name="alibrary-media_api-vote"),
url(r"^(?P<resource_name>%s)/(?P<uuid>\w[\w/-]*)/stats%s$" %
(self._meta.resource_name, trailing_slash()),
self.wrap_view('stats'),
name="alibrary-media_api-stats"),
url(r"^(?P<resource_name>%s)/(?P<uuid>\w[\w/-]*)/stream.mp3$" %
self._meta.resource_name,
self.wrap_view('stream_file'),
name="alibrary-media_api-stream"),
]
def autocomplete(self, request, **kwargs):
self.method_check(request, allowed=['get'])
self.throttle_check(request)
q = request.GET.get('q', None)
result = []
object_list = []
objects = []
object_count = 0
qs = None
if q and len(q) > 1:
# haystack version
#sqs = SearchQuerySet().models(Media).filter(SQ(content__contains=q) | SQ(content_auto=q))
sqs = SearchQuerySet().models(Media).filter(content=AutoQuery(q))
qs = Media.objects.filter(
id__in=[result.object.pk for result in sqs]).distinct()
# ORM version
#qs = Media.objects.order_by('name').filter(name__icontains=q)
object_list = qs.distinct()[0:50]
object_count = qs.distinct().count()
for result in object_list:
bundle = self.build_bundle(obj=result, request=request)
bundle = self.autocomplete_dehydrate(bundle, q)
objects.append(bundle)
data = {
'meta': {
'query': q,
'total_count': object_count
},
'objects': objects,
}
self.log_throttled_access(request)
return self.create_response(request, data)
def autocomplete_dehydrate(self, bundle, q):
bundle.data['name'] = bundle.obj.name
bundle.data['id'] = bundle.obj.pk
bundle.data['ct'] = 'media'
bundle.data['get_absolute_url'] = bundle.obj.get_absolute_url()
bundle.data['resource_uri'] = bundle.obj.get_api_url()
bundle.data['main_image'] = None
bundle.data['duration'] = bundle.obj.get_duration()
try:
bundle.data['artist'] = bundle.obj.artist.name
except:
bundle.data['artist'] = None
try:
bundle.data['release'] = bundle.obj.release.name
except:
bundle.data['release'] = None
try:
opt = THUMBNAIL_OPT
main_image = get_thumbnailer(
bundle.obj.release.main_image).get_thumbnail(opt)
bundle.data['main_image'] = main_image.url
except:
pass
return bundle
def vote(self, request, **kwargs):
self.method_check(request, allowed=['get'])
self.is_authenticated(request)
self.throttle_check(request)
obj = Media.objects.get(**self.remove_api_resource_names(kwargs))
# votes
try:
user_vote = obj.votes.filter(user=request.user)[0].vote
except (TypeError, IndexError) as e:
user_vote = None
votes = {
'up': obj.total_upvotes,
'down': obj.total_downvotes,
'total': obj.vote_total,
'user': user_vote,
}
self.log_throttled_access(request)
return self.create_response(request, votes)
def stats(self, request, **kwargs):
self.method_check(request, allowed=['get'])
#self.is_authenticated(request)
self.throttle_check(request)
obj = Media.objects.get(**self.remove_api_resource_names(kwargs))
from statistics.util import ObjectStatistics
ostats = ObjectStatistics(obj=obj)
stats = ostats.generate()
self.log_throttled_access(request)
return self.create_response(request, stats)
def stream_file(self, request, **kwargs):
"""
provides the default stream file as download.
method is only used by API clients (radio website) at the moment
"""
self.method_check(request, allowed=['get'])
self.is_authenticated(request)
self.throttle_check(request)
if not (request.user.has_perm('alibrary.play_media')):
return HttpResponseForbidden('sorry. no permissions here!')
obj = Media.objects.get(**self.remove_api_resource_names(kwargs))
from media_asset.util import get_format
format = get_format(obj, wait=True)
return sendfile(request, format.path)
class ProfileResource(TypeFilteredResource):
"""Profile api"""
avatar_100 = fields.CharField(null=True)
profile_detail_url = fields.CharField()
email = fields.CharField(default='')
layers_count = fields.IntegerField(default=0)
maps_count = fields.IntegerField(default=0)
documents_count = fields.IntegerField(default=0)
current_user = fields.BooleanField(default=False)
activity_stream_url = fields.CharField(null=True)
def build_filters(self, filters={}):
"""adds filtering by group functionality"""
orm_filters = super(ProfileResource, self).build_filters(filters)
if 'group' in filters:
orm_filters['group'] = filters['group']
return orm_filters
def apply_filters(self, request, applicable_filters):
"""filter by group if applicable by group functionality"""
group = applicable_filters.pop('group', None)
semi_filtered = super(
ProfileResource,
self).apply_filters(
request,
applicable_filters)
if group is not None:
semi_filtered = semi_filtered.filter(
groupmember__group__slug=group)
return semi_filtered
def dehydrate_email(self, bundle):
email = ''
if bundle.request.user.is_authenticated():
email = bundle.obj.email
return email
def dehydrate_layers_count(self, bundle):
obj_with_perms = get_objects_for_user(bundle.request.user,
'base.view_resourcebase').instance_of(Layer)
return bundle.obj.resourcebase_set.filter(id__in=obj_with_perms.values('id')).distinct().count()
def dehydrate_maps_count(self, bundle):
obj_with_perms = get_objects_for_user(bundle.request.user,
'base.view_resourcebase').instance_of(Map)
return bundle.obj.resourcebase_set.filter(id__in=obj_with_perms.values('id')).distinct().count()
def dehydrate_documents_count(self, bundle):
obj_with_perms = get_objects_for_user(bundle.request.user,
'base.view_resourcebase').instance_of(Document)
return bundle.obj.resourcebase_set.filter(id__in=obj_with_perms.values('id')).distinct().count()
def dehydrate_avatar_100(self, bundle):
return avatar_url(bundle.obj, 100)
def dehydrate_profile_detail_url(self, bundle):
return bundle.obj.get_absolute_url()
def dehydrate_current_user(self, bundle):
return bundle.request.user.username == bundle.obj.username
def dehydrate_activity_stream_url(self, bundle):
return reverse(
'actstream_actor',
kwargs={
'content_type_id': ContentType.objects.get_for_model(
bundle.obj).pk,
'object_id': bundle.obj.pk})
def prepend_urls(self):
if settings.HAYSTACK_SEARCH:
return [
url(r"^(?P<resource_name>%s)/search%s$" % (
self._meta.resource_name, trailing_slash()
),
self.wrap_view('get_search'), name="api_get_search"),
]
else:
return []
def serialize(self, request, data, format, options={}):
options['count_type'] = 'owner'
return super(ProfileResource, self).serialize(request, data, format, options)
class Meta:
queryset = get_user_model().objects.exclude(username='******')
resource_name = 'profiles'
allowed_methods = ['get']
ordering = ['username', 'date_joined']
excludes = ['is_staff', 'password', 'is_superuser',
'is_active', 'last_login']
filtering = {
'username': ALL,
}
serializer = CountJSONSerializer()
class UserResource(ClientCacheResourceMixIn, GraphiteMixIn, ModelResource):
"""User Resource."""
email = fields.CharField(attribute='user__email', null=True, readonly=True)
username = fields.CharField(attribute='user__username',
null=True,
readonly=True)
vouched_by = fields.IntegerField(attribute='vouched_by__id',
null=True,
readonly=True)
groups = fields.CharField()
skills = fields.CharField()
languages = fields.CharField()
url = fields.CharField()
accounts = fields.CharField()
class Meta:
queryset = UserProfile.objects.all()
authentication = AppAuthentication()
authorization = ReadOnlyAuthorization()
serializer = Serializer(formats=['json', 'jsonp'])
paginator_class = Paginator
cache_control = {'max-age': 0}
list_allowed_methods = ['get']
detail_allowed_methods = ['get']
resource_name = 'users'
restrict_fields = False
restricted_fields = ['email', 'is_vouched']
fields = [
'id', 'full_name', 'is_vouched', 'vouched_by', 'date_vouched',
'groups', 'skills', 'bio', 'photo', 'ircname', 'country', 'region',
'city', 'date_mozillian', 'timezone', 'email',
'allows_mozilla_sites', 'allows_community_sites'
]
def build_filters(self, filters=None):
database_filters = {}
valid_filters = [
f for f in filters if f in [
'email', 'country', 'region', 'city', 'ircname', 'username',
'groups', 'skills', 'is_vouched', 'name', 'accounts'
]
]
getvalue = lambda x: unquote(filters[x].lower())
if 'accounts' in valid_filters:
database_filters['accounts'] = Q(
externalaccount__identifier__icontains=getvalue('accounts'))
if 'email' in valid_filters:
database_filters['email'] = Q(
user__email__iexact=getvalue('email'))
if 'username' in valid_filters:
database_filters['username'] = Q(
user__username__iexact=getvalue('username'))
if 'name' in valid_filters:
database_filters['name'] = Q(full_name__iexact=getvalue('name'))
if 'is_vouched' in valid_filters:
value = getvalue('is_vouched')
if value == 'true':
database_filters['is_vouched'] = Q(is_vouched=True)
elif value == 'false':
database_filters['is_vouched'] = Q(is_vouched=False)
for possible_filter in ['country', 'region', 'city', 'ircname']:
if possible_filter in valid_filters:
database_filters[possible_filter] = Q(
**{
'{0}__iexact'.format(possible_filter):
getvalue(possible_filter)
})
for group_filter in ['groups', 'skills']:
if group_filter in valid_filters:
database_filters[group_filter] = Q(
**{
'{0}__name__in'.format(group_filter):
getvalue(group_filter).split(',')
})
return database_filters
def dehydrate(self, bundle):
if (bundle.request.GET.get('restricted', False)
or not bundle.data['allows_mozilla_sites']):
data = {}
for key in self._meta.restricted_fields:
data[key] = bundle.data[key]
bundle = Bundle(obj=bundle.obj, data=data, request=bundle.request)
return bundle
def dehydrate_accounts(self, bundle):
accounts = [{
'identifier': a.identifier,
'type': a.type
} for a in bundle.obj.externalaccount_set.all()]
return accounts
def dehydrate_groups(self, bundle):
groups = bundle.obj.groups.values_list('name', flat=True)
return list(groups)
def dehydrate_skills(self, bundle):
skills = bundle.obj.skills.values_list('name', flat=True)
return list(skills)
def dehydrate_languages(self, bundle):
languages = bundle.obj.languages.values_list('code', flat=True)
return list(languages)
def dehydrate_photo(self, bundle):
if bundle.obj.photo:
return urljoin(settings.SITE_URL, bundle.obj.photo.url)
return ''
def dehydrate_url(self, bundle):
url = reverse('phonebook:profile_view',
args=[bundle.obj.user.username])
return utils.absolutify(url)
def get_detail(self, request, **kwargs):
if request.GET.get('restricted', False):
raise ImmediateHttpResponse(response=http.HttpForbidden())
return super(UserResource, self).get_detail(request, **kwargs)
def apply_filters(self, request, applicable_filters):
if (request.GET.get('restricted', False)
and 'email' not in applicable_filters
and len(applicable_filters) != 1):
raise ImmediateHttpResponse(response=http.HttpForbidden())
mega_filter = Q()
for db_filter in applicable_filters.values():
mega_filter &= db_filter
if request.GET.get('restricted', False):
mega_filter &= Q(allows_community_sites=True)
return UserProfile.objects.complete().filter(
mega_filter).distinct().order_by('id')