def process_request(self):
'''
Default request processing: Apply any query parameters and get fields for the values. Respond with requested format.
'''
fields = self.getFields()
geomFields = self.getGeomFields()
self.assign_qs()
if self.stoqs_object_name == 'measured_parameter':
# Check if the query contains parametervalue constraints, in which case we need to wrap RawQuerySet in an MPQuerySet
pvConstraints = self.parameterValueConstraints()
if pvConstraints:
mpq = MPQuery(self.request)
pq = PQuery(self.request)
sql = postgresifySQL(str(self.qs.query))
sql = pq.addParameterValuesSelfJoins(
sql, pvConstraints, select_items=MPQuery.rest_select_items)
self.qs = MPQuerySet(sql, MPQuerySet.rest_columns)
else:
self.qs = MPQuerySet(None,
MPQuerySet.rest_columns,
qs_mp=self.qs)
# Process request based on format requested
if self.format == 'csv' or self.format == 'tsv':
response = HttpResponse()
if self.format == 'tsv':
response['Content-type'] = 'text/tab-separated-values'
response[
'Content-Disposition'] = 'attachment; filename=%s.tsv' % self.stoqs_object_name
writer = csv.writer(response, delimiter='\t')
else:
response['Content-type'] = 'text/csv'
response[
'Content-Disposition'] = 'attachment; filename=%s.csv' % self.stoqs_object_name
writer = csv.writer(response)
writer.writerow(fields)
for obj in self.qs:
writer.writerow([obj[f] for f in fields])
return response
elif self.format == 'xml':
return HttpResponse(serializers.serialize('xml', self.query_set),
'application/xml')
elif self.format == 'json':
return HttpResponse(
simplejson.dumps(self.qs, cls=encoders.STOQSJSONEncoder),
'application/json')
elif self.format == 'kml':
kml = KML(self.request, self.qs, self.qparams,
self.stoqs_object_name)
return kml.kmlResponse()
elif self.format == 'kmln':
kml = KML(self.request,
self.qs,
self.qparams,
self.stoqs_object_name,
withTimeStamps=False,
withLineStrings=False,
withFullIconURL=False)
return kml.kmlResponse()
elif self.format == 'count':
count = self.qs.count()
logger.debug('count = %d', count)
return HttpResponse('%d' % count, mimetype='text/plain')
elif self.format == 'help':
helpText = 'Fields: %s\n\nField Lookups: %s' % (fields,
self.fieldLookups)
if geomFields:
helpText += '\n\nSpatial and distance Lookups that may be appended to: %s\n\n%s\n\n%s' % (
geomFields, self.distanceLookups, self.spatialLookups)
helpText += '\n\nResponses: %s' % (self.responses, )
response = HttpResponse(helpText, mimetype="text/plain")
return response
else:
self.build_html_template()
response = render_to_response(
self.html_tmpl_file, {
'cols': fields,
'google_analytics_code': settings.GOOGLE_ANALYTICS_CODE
},
context_instance=RequestContext(self.request))
fh = open(self.html_tmpl_path, 'w')
for line in response:
fh.write(line)
fh.close()
return render_to_response(self.html_tmpl_path, {'list': self.qs})
class BaseOutputer(object):
'''
Base methods for supported responses for all STOQS objects: csv, json, kml, html, etc.
'''
html_tmpl_file = 'html_template.html'
# Django's additional field lookups - applicable to all fields
fieldLookups = ('exact', 'iexact', 'contains', 'icontains', 'in', 'gt',
'gte', 'lt', 'lte', 'startswith', 'istartswith',
'endswith', 'iendswith', 'range', 'year', 'month', 'day',
'week_day', 'isnull', 'search', 'regex', 'iregex')
# GeoDjango's field lookups for geometry fields
distanceLookups = ('distance_lt', 'distance_lte', 'distance_gt',
'distance_gte', 'dwithin')
spatialLookups = ('bbcontains', 'bboverlaps', 'contained', 'contains',
'contains_properly', 'coveredby', 'covers', 'crosses',
'disjoint', 'distance_gt', 'distance_gte', 'distance_lt',
'distance_lte', 'dwithin', 'equals', 'exact',
'intersects', 'overlaps', 'relate', 'same_as', 'touches',
'within', 'left', 'right', 'overlaps_left',
'overlaps_right', 'overlaps_above', 'overlaps_below',
'strictly_above', 'strictly_below')
fields = []
geomFields = []
def __init__(self, request, format, query_set, stoqs_object=None):
'''
@query_set is the Query Set without any values assigned, qs is the Query set with values assigned.
'''
self.request = request
self.format = format
self.query_set = query_set
self.stoqs_object = stoqs_object
self.stoqs_object_name = stoqs_object._meta.verbose_name.lower(
).replace(' ', '_')
self.html_template = '%s_tmpl.html' % self.stoqs_object_name
# This file must be writable by the server running this Django app, wherever tempfile puts it should work.
# /tmp should occasionally be scrubbed of old tempfiles by a cron(1) job.
self.html_tmpl_path = tempfile.NamedTemporaryFile(
dir='/tmp', prefix=self.stoqs_object_name + '_',
suffix='.html').name
# May be overridden by classes that provide other responses, such as '.png' in an overridden process_request() method
self.responses = [
'.help', '.html', '.json', '.csv', '.tsv', '.xml', '.count'
]
def build_html_template(self):
'''
Build template for stoqs_object using generic html template with a column for each attribute
'''
response = render_to_response(
self.html_tmpl_file,
{'cols': [field.name for field in self.stoqs_object._meta.fields]},
context_instance=RequestContext(self.request))
logger.debug("Writing template: %s", self.html_tmpl_path)
fh = open(self.html_tmpl_path, 'w')
for line in response:
fh.write(line)
fh.close()
def getFields(self):
'''
Default fields for model class retreived by introspection. Override fields in subclasses to add other fields from joined tables.
'''
if self.fields:
return self.fields
else:
fields = []
for f in self.stoqs_object._meta.fields:
fields.append(f.name)
self.fields = fields
return fields
def getGeomFields(self):
'''
Return list of any Geometry Field Types in the class - useful for knowing if we can append distanceLookups and spatialLookups.
'''
geomTypes = ('GeometryField', 'PointField', 'LineStringField',
'PolygonField', 'MultiPointField', 'MultiLineStringField',
'MultiPolygonField', 'GeometryCollectionField')
if self.geomFields:
return self.geomFields
else:
geomFields = []
for f in self.stoqs_object._meta.fields:
if f.get_internal_type() in geomTypes:
geomFields.append(f.name)
self.geomFields = geomFields
return geomFields
def ammendFields(self, fields):
# Append Django field lookups to the field names, see: https://docs.djangoproject.com/en/dev/ref/models/querysets/#field-lookups
# and https://docs.djangoproject.com/en/dev/ref/contrib/gis/db-api/ for the distance and spatial lookups
ammendedFields = []
ammendedFields.extend(fields)
for addition in self.fieldLookups:
for f in fields:
ammendedFields.append('%s__%s' % (
f,
addition,
))
for addition in self.distanceLookups:
for f in self.geomFields:
ammendedFields.append('%s__%s' % (
f,
addition,
))
for addition in self.spatialLookups:
for f in self.geomFields:
ammendedFields.append('%s__%s' % (
f,
addition,
))
return ammendedFields
def parameterValueConstraints(self):
'''
Look for any parameter _MIN & _MAX input from the Query String. Return a dictionary of (min, max) tuples
keyed by parameter name. Unlike pvConstraints in the QueryUI, here we allow unbounded constraints;
either pmin or pmax may be None.
'''
pmin = {}
pmax = {}
for key, value in self.request.GET.iteritems():
if key.endswith('_MIN'):
name = key.split('_MIN')[0]
try:
pmin[name] = self.request.GET.getlist(name + '_MIN')[0]
except:
logger.exception(
'Could not get min parameter value even though ' +
key + ' ends with _MIN')
if key.endswith('_MAX'):
name = key.split('_MAX')[0]
try:
pmax[name] = self.request.GET.getlist(name + '_MAX')[0]
except:
logger.exception(
'Could not get max parameter value even though ' +
key + ' ends with _MAX')
pvDicts = []
for name in set(
pmin.keys() +
pmax.keys()): # set() uniquifies the keys from each dictionary
if name in mod.Parameter.objects.values_list('name', flat=True):
try:
pn = pmin[name]
except KeyError:
pn = None
try:
px = pmax[name]
except KeyError:
px = None
pvDicts.append({name: (pn, px)})
logger.debug('parametervalues = %s', pvDicts)
return pvDicts
def applyQueryParams(self, fields):
'''
Apply any constraints specified in the query string with ammened Django field lookups
'''
qparams = {}
logger.debug(self.request.GET)
for f in self.ammendFields(fields):
##logger.debug(f)
if self.request.GET.getlist(f):
##logger.debug('len(self.request.GET.getlist(f)) = %d', len(self.request.GET.getlist(f)))
if len(self.request.GET.getlist(f)) == 1:
qparams[f] = self.request.GET.getlist(f)[0]
else:
qparams[f + '__in'] = self.request.GET.getlist(f)
if self.request.GET.get('mplabels', []):
# Special addition for labeled data constraint
qparams[
'measurement__id__in'] = mod.MeasuredParameterResource.objects.filter(
resource__id__in=[self.request.GET.get('mplabels')
]).values_list(
'measuredparameter__measurement__id',
flat=True)
self.qparams = qparams
logger.debug(qparams)
self.query_set = self.query_set.filter(**qparams)
def assign_qs(self):
'''
Assign the processed query string 'qs' with query parameters and fields. May be overridden to restructure response as needed.
Creates self.qs which is self.query_string with values added.
'''
fields = self.getFields()
geomFields = self.getGeomFields()
logger.debug(fields)
self.applyQueryParams(fields)
self.qs = self.query_set
self.qs = self.qs.values(*fields)
def process_request(self):
'''
Default request processing: Apply any query parameters and get fields for the values. Respond with requested format.
'''
fields = self.getFields()
geomFields = self.getGeomFields()
self.assign_qs()
if self.stoqs_object_name == 'measured_parameter':
# Check if the query contains parametervalue constraints, in which case we need to wrap RawQuerySet in an MPQuerySet
pvConstraints = self.parameterValueConstraints()
if pvConstraints:
mpq = MPQuery(self.request)
pq = PQuery(self.request)
sql = postgresifySQL(str(self.qs.query))
sql = pq.addParameterValuesSelfJoins(
sql, pvConstraints, select_items=MPQuery.rest_select_items)
self.qs = MPQuerySet(sql, MPQuerySet.rest_columns)
else:
self.qs = MPQuerySet(None,
MPQuerySet.rest_columns,
qs_mp=self.qs)
# Process request based on format requested
if self.format == 'csv' or self.format == 'tsv':
response = HttpResponse()
if self.format == 'tsv':
response['Content-type'] = 'text/tab-separated-values'
response[
'Content-Disposition'] = 'attachment; filename=%s.tsv' % self.stoqs_object_name
writer = csv.writer(response, delimiter='\t')
else:
response['Content-type'] = 'text/csv'
response[
'Content-Disposition'] = 'attachment; filename=%s.csv' % self.stoqs_object_name
writer = csv.writer(response)
writer.writerow(fields)
for obj in self.qs:
writer.writerow([obj[f] for f in fields])
return response
elif self.format == 'xml':
return HttpResponse(serializers.serialize('xml', self.query_set),
'application/xml')
elif self.format == 'json':
return HttpResponse(
simplejson.dumps(self.qs, cls=encoders.STOQSJSONEncoder),
'application/json')
elif self.format == 'kml':
kml = KML(self.request, self.qs, self.qparams,
self.stoqs_object_name)
return kml.kmlResponse()
elif self.format == 'kmln':
kml = KML(self.request,
self.qs,
self.qparams,
self.stoqs_object_name,
withTimeStamps=False,
withLineStrings=False,
withFullIconURL=False)
return kml.kmlResponse()
elif self.format == 'count':
count = self.qs.count()
logger.debug('count = %d', count)
return HttpResponse('%d' % count, mimetype='text/plain')
elif self.format == 'help':
helpText = 'Fields: %s\n\nField Lookups: %s' % (fields,
self.fieldLookups)
if geomFields:
helpText += '\n\nSpatial and distance Lookups that may be appended to: %s\n\n%s\n\n%s' % (
geomFields, self.distanceLookups, self.spatialLookups)
helpText += '\n\nResponses: %s' % (self.responses, )
response = HttpResponse(helpText, mimetype="text/plain")
return response
else:
self.build_html_template()
response = render_to_response(
self.html_tmpl_file, {
'cols': fields,
'google_analytics_code': settings.GOOGLE_ANALYTICS_CODE
},
context_instance=RequestContext(self.request))
fh = open(self.html_tmpl_path, 'w')
for line in response:
fh.write(line)
fh.close()
return render_to_response(self.html_tmpl_path, {'list': self.qs})
class BaseOutputer(object):
'''
Base methods for supported responses for all STOQS objects: csv, json, kml, html, etc.
'''
html_tmpl_file = 'html_template.html'
# Django's additional field lookups - applicable to all fields
fieldLookups = ('exact', 'iexact', 'contains', 'icontains', 'in', 'gt', 'gte', 'lt', 'lte', 'startswith', 'istartswith',
'endswith', 'iendswith', 'range', 'year', 'month', 'day', 'week_day', 'isnull', 'search', 'regex', 'iregex')
# GeoDjango's field lookups for geometry fields
distanceLookups = ('distance_lt', 'distance_lte', 'distance_gt', 'distance_gte', 'dwithin')
spatialLookups = ('bbcontains', 'bboverlaps', 'contained', 'contains', 'contains_properly', 'coveredby', 'covers', 'crosses',
'disjoint', 'distance_gt', 'distance_gte', 'distance_lt', 'distance_lte', 'dwithin', 'equals', 'exact',
'intersects', 'overlaps', 'relate', 'same_as', 'touches', 'within', 'left', 'right', 'overlaps_left',
'overlaps_right', 'overlaps_above', 'overlaps_below', 'strictly_above', 'strictly_below')
fields = []
geomFields = []
def __init__(self, request, fmt, query_set, stoqs_object=None):
'''
@query_set is the Query Set without any values assigned, qs is the Query set with values assigned.
'''
self.request = request
self.format = fmt
self.query_set = query_set.using(self.request.META['dbAlias'])
self.stoqs_object = stoqs_object
self.stoqs_object_name = stoqs_object._meta.verbose_name.lower().replace(' ', '_')
self.html_template = '%s_tmpl.html' % self.stoqs_object_name
# This file must be writable by the server running this Django app, wherever tempfile puts it should work.
# /tmp should occasionally be scrubbed of old tempfiles by a cron(1) job.
self.html_tmpl_path = tempfile.NamedTemporaryFile(dir='/tmp', prefix=self.stoqs_object_name+'_', suffix='.html').name
# May be overridden by classes that provide other responses, such as '.png' in an overridden process_request() method
self.responses = ['.help', '.html', '.json', '.csv', '.tsv', '.xml', '.count']
def build_html_template(self):
'''
Build template for stoqs_object using generic html template with a column for each attribute
'''
response = render_to_response(self.html_tmpl_file, {'cols': [field.name for field in self.stoqs_object._meta.fields] },
context_instance = RequestContext(self.request))
logger.debug("Writing template: %s", self.html_tmpl_path)
fh = open(self.html_tmpl_path, 'w')
for line in response:
fh.write(line)
fh.close()
def getFields(self):
'''
Default fields for model class retreived by introspection. Override fields in subclasses to add other fields from joined tables.
'''
if self.fields:
return self.fields
else:
fields = []
for f in self.stoqs_object._meta.fields:
fields.append(f.name)
self.fields = fields
return fields
def getGeomFields(self):
'''
Return list of any Geometry Field Types in the class - useful for knowing if we can append distanceLookups and spatialLookups.
'''
geomTypes = ('GeometryField', 'PointField', 'LineStringField', 'PolygonField', 'MultiPointField', 'MultiLineStringField',
'MultiPolygonField', 'GeometryCollectionField')
if self.geomFields:
return self.geomFields
else:
geomFields = []
for f in self.stoqs_object._meta.fields:
if f.get_internal_type() in geomTypes:
geomFields.append(f.name)
self.geomFields = geomFields
return geomFields
def ammendFields(self, fields):
# Append Django field lookups to the field names, see: https://docs.djangoproject.com/en/dev/ref/models/querysets/#field-lookups
# and https://docs.djangoproject.com/en/dev/ref/contrib/gis/db-api/ for the distance and spatial lookups
ammendedFields = []
ammendedFields.extend(fields)
for addition in self.fieldLookups:
for f in fields:
ammendedFields.append('%s__%s' % (f, addition, ))
for addition in self.distanceLookups:
for f in self.geomFields:
ammendedFields.append('%s__%s' % (f, addition, ))
for addition in self.spatialLookups:
for f in self.geomFields:
ammendedFields.append('%s__%s' % (f, addition, ))
return ammendedFields
def parameterValueConstraints(self):
'''
Look for any parameter _MIN & _MAX input from the Query String. Return a dictionary of (min, max) tuples
keyed by parameter name. Unlike pvConstraints in the QueryUI, here we allow unbounded constraints;
either pmin or pmax may be None.
'''
pmin = {}
pmax = {}
for key, _ in self.request.GET.iteritems():
if key.endswith('_MIN'):
name = key.split('_MIN')[0]
try:
pmin[name] = self.request.GET.getlist(name + '_MIN')[0]
except Exception:
logger.exception('Could not get min parameter value even though ' + key + ' ends with _MIN')
if key.endswith('_MAX'):
name = key.split('_MAX')[0]
try:
pmax[name] = self.request.GET.getlist(name + '_MAX')[0]
except Exception:
logger.exception('Could not get max parameter value even though ' + key + ' ends with _MAX')
pvDicts = []
for name in set(pmin.keys() + pmax.keys()): # set() uniquifies the keys from each dictionary
if name in mod.Parameter.objects.using(self.request.META['dbAlias']).values_list('name', flat=True):
try:
pn = pmin[name]
except KeyError:
pn = None
try:
px = pmax[name]
except KeyError:
px = None
pvDicts.append({name: (pn, px)})
logger.debug('parametervalues = %s', pvDicts)
return pvDicts
def applyQueryParams(self, fields):
'''
Apply any constraints specified in the query string with ammened Django field lookups
'''
qparams = {}
logger.debug(self.request.GET)
for f in self.ammendFields(fields):
##logger.debug(f)
if self.request.GET.getlist(f):
##logger.debug('len(self.request.GET.getlist(f)) = %d', len(self.request.GET.getlist(f)))
if len(self.request.GET.getlist(f)) == 1:
qparams[f] = self.request.GET.getlist(f)[0]
else:
qparams[f + '__in'] = self.request.GET.getlist(f)
if self.request.GET.get('mplabels', []):
# Special addition for labeled data constraint
qparams['measurement__id__in'] = mod.MeasuredParameterResource.objects.filter(resource__id__in=[self.request.GET.get('mplabels')]
).values_list('measuredparameter__measurement__id', flat=True)
self.qparams = qparams
logger.debug(qparams)
self.query_set = self.query_set.filter(**qparams)
def assign_qs(self):
'''
Assign the processed query string 'qs' with query parameters and fields. May be overridden to restructure response as needed.
Creates self.qs which is self.query_string with values added.
'''
fields = self.getFields()
logger.debug(fields)
self.applyQueryParams(fields)
self.qs = self.query_set
self.qs = self.qs.values(*fields)
def process_request(self):
'''
Default request processing: Apply any query parameters and get fields for the values. Respond with requested format.
'''
fields = self.getFields()
geomFields = self.getGeomFields()
try:
self.assign_qs()
except EmptyQuerySetException:
raise Http404
if self.stoqs_object_name == 'measured_parameter':
# Check if the query contains parametervalue constraints, in which case we need to wrap RawQuerySet in an MPQuerySet
pvConstraints = self.parameterValueConstraints()
if pvConstraints:
pq = PQuery(self.request)
sql = postgresifySQL(str(self.qs.query))
sql = pq.addParameterValuesSelfJoins(sql, pvConstraints, select_items=MPQuery.rest_select_items)
self.qs = MPQuerySet(self.request.META['dbAlias'], sql, MPQuerySet.rest_columns)
else:
self.qs = MPQuerySet(self.request.META['dbAlias'], None, MPQuerySet.rest_columns, qs_mp=self.qs)
# Process request based on format requested
if self.format == 'csv' or self.format == 'tsv':
response = HttpResponse()
if self.format == 'tsv':
response['Content-type'] = 'text/tab-separated-values'
response['Content-Disposition'] = 'attachment; filename=%s.tsv' % self.stoqs_object_name
writer = csv.writer(response, delimiter='\t')
else:
response['Content-type'] = 'text/csv'
response['Content-Disposition'] = 'attachment; filename=%s.csv' % self.stoqs_object_name
writer = csv.writer(response)
writer.writerow(fields)
for obj in self.qs:
writer.writerow([obj[f] for f in fields])
return response
elif self.format == 'xml':
return HttpResponse(serializers.serialize('xml', self.query_set), 'application/xml')
elif self.format == 'json':
return HttpResponse(json.dumps(self.qs, cls=encoders.STOQSJSONEncoder), 'application/json')
elif self.format == 'kml':
kml = KML(self.request, self.qs, self.qparams, self.stoqs_object_name)
return kml.kmlResponse()
elif self.format == 'kmln':
kml = KML(self.request, self.qs, self.qparams, self.stoqs_object_name, withTimeStamps=False, withLineStrings=False, withFullIconURL=False)
return kml.kmlResponse()
elif self.format == 'count':
count = self.qs.count()
logger.debug('count = %d', count)
return HttpResponse('%d' % count, content_type='text/plain')
elif self.format == 'help':
helpText = 'Fields: %s\n\nField Lookups: %s' % (fields, self.fieldLookups)
if geomFields:
helpText += '\n\nSpatial and distance Lookups that may be appended to: %s\n\n%s\n\n%s' % (geomFields,
self.distanceLookups, self.spatialLookups)
helpText += '\n\nResponses: %s' % (self.responses,)
response = HttpResponse(helpText, content_type="text/plain")
return response
else:
self.build_html_template()
try:
response = render_to_response(self.html_tmpl_file, {'cols': fields,
'google_analytics_code': settings.GOOGLE_ANALYTICS_CODE },
context_instance = RequestContext(self.request))
except AttributeError:
response = render_to_response(self.html_tmpl_file, {'cols': fields},
context_instance = RequestContext(self.request))
fh = open(self.html_tmpl_path, 'w')
for line in response:
fh.write(line)
fh.close()
return render_to_response(self.html_tmpl_path, {'list': self.qs})
def process_request(self):
'''
Default request processing: Apply any query parameters and get fields for the values. Respond with requested format.
'''
fields = self.getFields()
geomFields = self.getGeomFields()
try:
self.assign_qs()
except EmptyQuerySetException:
raise Http404
if self.stoqs_object_name == 'measured_parameter':
# Check if the query contains parametervalue constraints, in which case we need to wrap RawQuerySet in an MPQuerySet
pvConstraints = self.parameterValueConstraints()
if pvConstraints:
pq = PQuery(self.request)
sql = postgresifySQL(str(self.qs.query))
sql = pq.addParameterValuesSelfJoins(sql, pvConstraints, select_items=MPQuery.rest_select_items)
self.qs = MPQuerySet(self.request.META['dbAlias'], sql, MPQuerySet.rest_columns)
else:
self.qs = MPQuerySet(self.request.META['dbAlias'], None, MPQuerySet.rest_columns, qs_mp=self.qs)
# Process request based on format requested
if self.format == 'csv' or self.format == 'tsv':
response = HttpResponse()
if self.format == 'tsv':
response['Content-type'] = 'text/tab-separated-values'
response['Content-Disposition'] = 'attachment; filename=%s.tsv' % self.stoqs_object_name
writer = csv.writer(response, delimiter='\t')
else:
response['Content-type'] = 'text/csv'
response['Content-Disposition'] = 'attachment; filename=%s.csv' % self.stoqs_object_name
writer = csv.writer(response)
writer.writerow(fields)
for obj in self.qs:
writer.writerow([obj[f] for f in fields])
return response
elif self.format == 'xml':
return HttpResponse(serializers.serialize('xml', self.query_set), 'application/xml')
elif self.format == 'json':
return HttpResponse(json.dumps(self.qs, cls=encoders.STOQSJSONEncoder), 'application/json')
elif self.format == 'kml':
kml = KML(self.request, self.qs, self.qparams, self.stoqs_object_name)
return kml.kmlResponse()
elif self.format == 'kmln':
kml = KML(self.request, self.qs, self.qparams, self.stoqs_object_name, withTimeStamps=False, withLineStrings=False, withFullIconURL=False)
return kml.kmlResponse()
elif self.format == 'count':
count = self.qs.count()
logger.debug('count = %d', count)
return HttpResponse('%d' % count, content_type='text/plain')
elif self.format == 'help':
helpText = 'Fields: %s\n\nField Lookups: %s' % (fields, self.fieldLookups)
if geomFields:
helpText += '\n\nSpatial and distance Lookups that may be appended to: %s\n\n%s\n\n%s' % (geomFields,
self.distanceLookups, self.spatialLookups)
helpText += '\n\nResponses: %s' % (self.responses,)
response = HttpResponse(helpText, content_type="text/plain")
return response
else:
self.build_html_template()
try:
response = render_to_response(self.html_tmpl_file, {'cols': fields,
'google_analytics_code': settings.GOOGLE_ANALYTICS_CODE },
context_instance = RequestContext(self.request))
except AttributeError:
response = render_to_response(self.html_tmpl_file, {'cols': fields},
context_instance = RequestContext(self.request))
fh = open(self.html_tmpl_path, 'w')
for line in response:
fh.write(line)
fh.close()
return render_to_response(self.html_tmpl_path, {'list': self.qs})
class BaseOutputer(object):
"""
Base methods for supported responses for all STOQS objects: csv, json, kml, html, etc.
"""
html_tmpl_file = "html_template.html"
# Django's additional field lookups - applicable to all fields
fieldLookups = (
"exact",
"iexact",
"contains",
"icontains",
"in",
"gt",
"gte",
"lt",
"lte",
"startswith",
"istartswith",
"endswith",
"iendswith",
"range",
"year",
"month",
"day",
"week_day",
"isnull",
"search",
"regex",
"iregex",
)
# GeoDjango's field lookups for geometry fields
distanceLookups = ("distance_lt", "distance_lte", "distance_gt", "distance_gte", "dwithin")
spatialLookups = (
"bbcontains",
"bboverlaps",
"contained",
"contains",
"contains_properly",
"coveredby",
"covers",
"crosses",
"disjoint",
"distance_gt",
"distance_gte",
"distance_lt",
"distance_lte",
"dwithin",
"equals",
"exact",
"intersects",
"overlaps",
"relate",
"same_as",
"touches",
"within",
"left",
"right",
"overlaps_left",
"overlaps_right",
"overlaps_above",
"overlaps_below",
"strictly_above",
"strictly_below",
)
fields = []
geomFields = []
def __init__(self, request, format, query_set, stoqs_object=None):
"""
@query_set is the Query Set without any values assigned, qs is the Query set with values assigned.
"""
self.request = request
self.format = format
self.query_set = query_set.using(self.request.META["dbAlias"])
self.stoqs_object = stoqs_object
self.stoqs_object_name = stoqs_object._meta.verbose_name.lower().replace(" ", "_")
self.html_template = "%s_tmpl.html" % self.stoqs_object_name
# This file must be writable by the server running this Django app, wherever tempfile puts it should work.
# /tmp should occasionally be scrubbed of old tempfiles by a cron(1) job.
self.html_tmpl_path = tempfile.NamedTemporaryFile(
dir="/tmp", prefix=self.stoqs_object_name + "_", suffix=".html"
).name
# May be overridden by classes that provide other responses, such as '.png' in an overridden process_request() method
self.responses = [".help", ".html", ".json", ".csv", ".tsv", ".xml", ".count"]
def build_html_template(self):
"""
Build template for stoqs_object using generic html template with a column for each attribute
"""
response = render_to_response(
self.html_tmpl_file,
{"cols": [field.name for field in self.stoqs_object._meta.fields]},
context_instance=RequestContext(self.request),
)
logger.debug("Writing template: %s", self.html_tmpl_path)
fh = open(self.html_tmpl_path, "w")
for line in response:
fh.write(line)
fh.close()
def getFields(self):
"""
Default fields for model class retreived by introspection. Override fields in subclasses to add other fields from joined tables.
"""
if self.fields:
return self.fields
else:
fields = []
for f in self.stoqs_object._meta.fields:
fields.append(f.name)
self.fields = fields
return fields
def getGeomFields(self):
"""
Return list of any Geometry Field Types in the class - useful for knowing if we can append distanceLookups and spatialLookups.
"""
geomTypes = (
"GeometryField",
"PointField",
"LineStringField",
"PolygonField",
"MultiPointField",
"MultiLineStringField",
"MultiPolygonField",
"GeometryCollectionField",
)
if self.geomFields:
return self.geomFields
else:
geomFields = []
for f in self.stoqs_object._meta.fields:
if f.get_internal_type() in geomTypes:
geomFields.append(f.name)
self.geomFields = geomFields
return geomFields
def ammendFields(self, fields):
# Append Django field lookups to the field names, see: https://docs.djangoproject.com/en/dev/ref/models/querysets/#field-lookups
# and https://docs.djangoproject.com/en/dev/ref/contrib/gis/db-api/ for the distance and spatial lookups
ammendedFields = []
ammendedFields.extend(fields)
for addition in self.fieldLookups:
for f in fields:
ammendedFields.append("%s__%s" % (f, addition))
for addition in self.distanceLookups:
for f in self.geomFields:
ammendedFields.append("%s__%s" % (f, addition))
for addition in self.spatialLookups:
for f in self.geomFields:
ammendedFields.append("%s__%s" % (f, addition))
return ammendedFields
def parameterValueConstraints(self):
"""
Look for any parameter _MIN & _MAX input from the Query String. Return a dictionary of (min, max) tuples
keyed by parameter name. Unlike pvConstraints in the QueryUI, here we allow unbounded constraints;
either pmin or pmax may be None.
"""
pmin = {}
pmax = {}
for key, value in self.request.GET.iteritems():
if key.endswith("_MIN"):
name = key.split("_MIN")[0]
try:
pmin[name] = self.request.GET.getlist(name + "_MIN")[0]
except:
logger.exception("Could not get min parameter value even though " + key + " ends with _MIN")
if key.endswith("_MAX"):
name = key.split("_MAX")[0]
try:
pmax[name] = self.request.GET.getlist(name + "_MAX")[0]
except:
logger.exception("Could not get max parameter value even though " + key + " ends with _MAX")
pvDicts = []
for name in set(pmin.keys() + pmax.keys()): # set() uniquifies the keys from each dictionary
if name in mod.Parameter.objects.values_list("name", flat=True):
try:
pn = pmin[name]
except KeyError:
pn = None
try:
px = pmax[name]
except KeyError:
px = None
pvDicts.append({name: (pn, px)})
logger.debug("parametervalues = %s", pvDicts)
return pvDicts
def applyQueryParams(self, fields):
"""
Apply any constraints specified in the query string with ammened Django field lookups
"""
qparams = {}
logger.debug(self.request.GET)
for f in self.ammendFields(fields):
##logger.debug(f)
if self.request.GET.getlist(f):
##logger.debug('len(self.request.GET.getlist(f)) = %d', len(self.request.GET.getlist(f)))
if len(self.request.GET.getlist(f)) == 1:
qparams[f] = self.request.GET.getlist(f)[0]
else:
qparams[f + "__in"] = self.request.GET.getlist(f)
if self.request.GET.get("mplabels", []):
# Special addition for labeled data constraint
qparams["measurement__id__in"] = mod.MeasuredParameterResource.objects.filter(
resource__id__in=[self.request.GET.get("mplabels")]
).values_list("measuredparameter__measurement__id", flat=True)
self.qparams = qparams
logger.debug(qparams)
self.query_set = self.query_set.filter(**qparams)
def assign_qs(self):
"""
Assign the processed query string 'qs' with query parameters and fields. May be overridden to restructure response as needed.
Creates self.qs which is self.query_string with values added.
"""
fields = self.getFields()
geomFields = self.getGeomFields()
logger.debug(fields)
self.applyQueryParams(fields)
self.qs = self.query_set
self.qs = self.qs.values(*fields)
def process_request(self):
"""
Default request processing: Apply any query parameters and get fields for the values. Respond with requested format.
"""
fields = self.getFields()
geomFields = self.getGeomFields()
try:
self.assign_qs()
except EmptyQuerySetException:
raise Http404
if self.stoqs_object_name == "measured_parameter":
# Check if the query contains parametervalue constraints, in which case we need to wrap RawQuerySet in an MPQuerySet
pvConstraints = self.parameterValueConstraints()
if pvConstraints:
mpq = MPQuery(self.request)
pq = PQuery(self.request)
sql = postgresifySQL(str(self.qs.query))
sql = pq.addParameterValuesSelfJoins(sql, pvConstraints, select_items=MPQuery.rest_select_items)
self.qs = MPQuerySet(self.request.META["dbAlias"], sql, MPQuerySet.rest_columns)
else:
self.qs = MPQuerySet(self.request.META["dbAlias"], None, MPQuerySet.rest_columns, qs_mp=self.qs)
# Process request based on format requested
if self.format == "csv" or self.format == "tsv":
response = HttpResponse()
if self.format == "tsv":
response["Content-type"] = "text/tab-separated-values"
response["Content-Disposition"] = "attachment; filename=%s.tsv" % self.stoqs_object_name
writer = csv.writer(response, delimiter="\t")
else:
response["Content-type"] = "text/csv"
response["Content-Disposition"] = "attachment; filename=%s.csv" % self.stoqs_object_name
writer = csv.writer(response)
writer.writerow(fields)
for obj in self.qs:
writer.writerow([obj[f] for f in fields])
return response
elif self.format == "xml":
return HttpResponse(serializers.serialize("xml", self.query_set), "application/xml")
elif self.format == "json":
return HttpResponse(json.dumps(self.qs, cls=encoders.STOQSJSONEncoder), "application/json")
elif self.format == "kml":
kml = KML(self.request, self.qs, self.qparams, self.stoqs_object_name)
return kml.kmlResponse()
elif self.format == "kmln":
kml = KML(
self.request,
self.qs,
self.qparams,
self.stoqs_object_name,
withTimeStamps=False,
withLineStrings=False,
withFullIconURL=False,
)
return kml.kmlResponse()
elif self.format == "count":
count = self.qs.count()
logger.debug("count = %d", count)
return HttpResponse("%d" % count, content_type="text/plain")
elif self.format == "help":
helpText = "Fields: %s\n\nField Lookups: %s" % (fields, self.fieldLookups)
if geomFields:
helpText += "\n\nSpatial and distance Lookups that may be appended to: %s\n\n%s\n\n%s" % (
geomFields,
self.distanceLookups,
self.spatialLookups,
)
helpText += "\n\nResponses: %s" % (self.responses,)
response = HttpResponse(helpText, content_type="text/plain")
return response
else:
self.build_html_template()
response = render_to_response(
self.html_tmpl_file,
{"cols": fields, "google_analytics_code": settings.GOOGLE_ANALYTICS_CODE},
context_instance=RequestContext(self.request),
)
fh = open(self.html_tmpl_path, "w")
for line in response:
fh.write(line)
fh.close()
return render_to_response(self.html_tmpl_path, {"list": self.qs})
def process_request(self):
"""
Default request processing: Apply any query parameters and get fields for the values. Respond with requested format.
"""
fields = self.getFields()
geomFields = self.getGeomFields()
try:
self.assign_qs()
except EmptyQuerySetException:
raise Http404
if self.stoqs_object_name == "measured_parameter":
# Check if the query contains parametervalue constraints, in which case we need to wrap RawQuerySet in an MPQuerySet
pvConstraints = self.parameterValueConstraints()
if pvConstraints:
mpq = MPQuery(self.request)
pq = PQuery(self.request)
sql = postgresifySQL(str(self.qs.query))
sql = pq.addParameterValuesSelfJoins(sql, pvConstraints, select_items=MPQuery.rest_select_items)
self.qs = MPQuerySet(self.request.META["dbAlias"], sql, MPQuerySet.rest_columns)
else:
self.qs = MPQuerySet(self.request.META["dbAlias"], None, MPQuerySet.rest_columns, qs_mp=self.qs)
# Process request based on format requested
if self.format == "csv" or self.format == "tsv":
response = HttpResponse()
if self.format == "tsv":
response["Content-type"] = "text/tab-separated-values"
response["Content-Disposition"] = "attachment; filename=%s.tsv" % self.stoqs_object_name
writer = csv.writer(response, delimiter="\t")
else:
response["Content-type"] = "text/csv"
response["Content-Disposition"] = "attachment; filename=%s.csv" % self.stoqs_object_name
writer = csv.writer(response)
writer.writerow(fields)
for obj in self.qs:
writer.writerow([obj[f] for f in fields])
return response
elif self.format == "xml":
return HttpResponse(serializers.serialize("xml", self.query_set), "application/xml")
elif self.format == "json":
return HttpResponse(json.dumps(self.qs, cls=encoders.STOQSJSONEncoder), "application/json")
elif self.format == "kml":
kml = KML(self.request, self.qs, self.qparams, self.stoqs_object_name)
return kml.kmlResponse()
elif self.format == "kmln":
kml = KML(
self.request,
self.qs,
self.qparams,
self.stoqs_object_name,
withTimeStamps=False,
withLineStrings=False,
withFullIconURL=False,
)
return kml.kmlResponse()
elif self.format == "count":
count = self.qs.count()
logger.debug("count = %d", count)
return HttpResponse("%d" % count, content_type="text/plain")
elif self.format == "help":
helpText = "Fields: %s\n\nField Lookups: %s" % (fields, self.fieldLookups)
if geomFields:
helpText += "\n\nSpatial and distance Lookups that may be appended to: %s\n\n%s\n\n%s" % (
geomFields,
self.distanceLookups,
self.spatialLookups,
)
helpText += "\n\nResponses: %s" % (self.responses,)
response = HttpResponse(helpText, content_type="text/plain")
return response
else:
self.build_html_template()
response = render_to_response(
self.html_tmpl_file,
{"cols": fields, "google_analytics_code": settings.GOOGLE_ANALYTICS_CODE},
context_instance=RequestContext(self.request),
)
fh = open(self.html_tmpl_path, "w")
for line in response:
fh.write(line)
fh.close()
return render_to_response(self.html_tmpl_path, {"list": self.qs})
def process_request(self):
'''
Default request processing: Apply any query parameters and get fields for the values. Respond with requested format.
'''
fields = self.getFields()
geomFields = self.getGeomFields()
try:
self.assign_qs()
except EmptyQuerySetException:
raise Http404
# A terrible hack to add latitude and longitude columns to the response - must call following assign_qs()
fields = self.add_lon_lat_cols()
if self.stoqs_object_name == 'measured_parameter':
# Check if the query contains parametervalue constraints, in which case we need to wrap RawQuerySet in an MPQuerySet
pvConstraints = self.parameterValueConstraints()
if pvConstraints:
pq = PQuery(self.request)
sql = postgresifySQL(str(self.qs.query))
sql = pq.addParameterValuesSelfJoins(
sql, pvConstraints, select_items=MPQuery.rest_select_items)
self.qs = MPQuerySet(self.request.META['dbAlias'], sql,
MPQuerySet.rest_columns)
else:
self.qs = MPQuerySet(self.request.META['dbAlias'],
None,
MPQuerySet.rest_columns,
qs_mp=self.qs)
# Process request based on format requested
if self.format == 'csv' or self.format == 'tsv':
response = HttpResponse()
if self.format == 'tsv':
response['Content-type'] = 'text/tab-separated-values'
response[
'Content-Disposition'] = 'attachment; filename=%s.tsv' % self.stoqs_object_name
writer = csv.writer(response, delimiter='\t')
else:
response['Content-type'] = 'text/csv'
response[
'Content-Disposition'] = 'attachment; filename=%s.csv' % self.stoqs_object_name
writer = csv.writer(response)
writer.writerow(fields)
for obj in self.qs:
writer.writerow(self.row_of_fields(obj))
return response
elif self.format == 'xml':
return HttpResponse(serializers.serialize('xml', self.query_set),
'application/xml')
elif self.format == 'json':
return HttpResponse(
json.dumps(self.qs, cls=encoders.STOQSJSONEncoder),
'application/json')
elif self.format == 'kml':
kml = KML(self.request, self.qs, self.qparams,
self.stoqs_object_name)
return kml.kmlResponse()
elif self.format == 'kmln':
kml = KML(self.request,
self.qs,
self.qparams,
self.stoqs_object_name,
withTimeStamps=False,
withLineStrings=False,
withFullIconURL=False)
return kml.kmlResponse()
elif self.format == 'count':
count = self.qs.count()
logger.debug('count = %d', count)
return HttpResponse('%d' % count, content_type='text/plain')
elif self.format == 'help':
helpText = 'Fields: %s\n\nField Lookups: %s' % (fields,
self.fieldLookups)
if geomFields:
helpText += '\n\nSpatial and distance Lookups that may be appended to: %s\n\n%s\n\n%s' % (
geomFields, self.distanceLookups, self.spatialLookups)
helpText += '\n\nResponses: %s' % (self.responses, )
response = HttpResponse(helpText, content_type="text/plain")
return response
else:
self.build_html_template()
try:
response = render(self.request,
self.html_tmpl_file,
context={
'cols':
fields,
'google_analytics_code':
settings.GOOGLE_ANALYTICS_CODE
})
except AttributeError:
response = render(self.request,
self.html_tmpl_file,
context={'cols': fields})
fh = open(self.html_tmpl_path, 'w')
for line in response:
# Override Django's default datetime formatting with ISO 8601 format that includes seconds
# STOQS model field names ending in 'timevalue' and 'date' are convertable datetimes
# See: https://docs.djangoproject.com/en/1.11/ref/templates/builtins/#date
line = line.decode("utf-8").replace('timevalue }',
'timevalue|date:"c" }')
line = line.replace('date }', 'date|date:"c" }')
fh.write(line)
fh.close()
return render(self.request,
self.html_tmpl_path,
context={'list': self.qs})
