def timeseries():
validator = ParamValidator()\
.set_optional('agg', agg_validator, 'week')\
.set_optional('data_type', make_format_validator(['json', 'csv']), 'json')\
.set_optional('dataset_name__in', list_of_datasets_validator, MetaTable.index)\
.set_optional('obs_date__ge', date_validator, datetime.now() - timedelta(days=90))\
.set_optional('obs_date__le', date_validator, datetime.now())\
.set_optional('location_geom__within', geom_validator, None)\
.set_optional('buffer', int_validator, 100)
err = validator.validate(request.args)
if err:
return bad_request(err)
geom = validator.get_geom()
table_names = validator.vals['dataset_name__in']
start_date = validator.vals['obs_date__ge']
end_date = validator.vals['obs_date__le']
agg = validator.vals['agg']
# Only examine tables that have a chance of containing records within the date and space boundaries.
try:
table_names = MetaTable.narrow_candidates(table_names, start_date,
end_date, geom)
except Exception as e:
msg = 'Failed to gather candidate tables.'
return internal_error(msg, e)
try:
panel = MetaTable.timeseries_all(table_names=table_names,
agg_unit=agg,
start=start_date,
end=end_date,
geom=geom)
except Exception as e:
msg = 'Failed to construct timeseries.'
return internal_error(msg, e)
panel = MetaTable.attach_metadata(panel)
resp = json_response_base(validator, panel)
datatype = validator.vals['data_type']
if datatype == 'json':
resp = make_response(json.dumps(resp, default=dthandler), 200)
resp.headers['Content-Type'] = 'application/json'
elif datatype == 'csv':
# response format
# temporal_group,dataset_name_1,dataset_name_2
# 2014-02-24 00:00:00,235,653
# 2014-03-03 00:00:00,156,624
fields = ['temporal_group']
for o in resp['objects']:
fields.append(o['dataset_name'])
csv_resp = []
i = 0
for k, g in groupby(resp['objects'], key=itemgetter('dataset_name')):
l_g = list(g)[0]
j = 0
for row in l_g['items']:
# first iteration, populate the first column with temporal_groups
if i == 0:
csv_resp.append([row['datetime']])
csv_resp[j].append(row['count'])
j += 1
i += 1
csv_resp.insert(0, fields)
csv_resp = make_csv(csv_resp)
resp = make_response(csv_resp, 200)
resp.headers['Content-Type'] = 'text/csv'
filedate = datetime.now().strftime('%Y-%m-%d')
resp.headers[
'Content-Disposition'] = 'attachment; filename=%s.csv' % filedate
return resp
def _timeseries(args):
meta_params = ['geom', 'dataset', 'dataset_name__in', 'obs_date__ge', 'obs_date__le', 'agg']
meta_vals = [args.data.get(k) for k in meta_params]
geom, dataset, table_names, start_date, end_date, agg = meta_vals
ctrees = {}
if has_tree_filters(args.data):
# Timeseries is a little tricky. If there aren't filters,
# it would be ridiculous to build a condition tree for every one.
for field, value in args.data.items():
if 'filter' in field:
# This pattern matches the last occurrence of the '__' pattern.
# Prevents an error that is caused by dataset names with trailing
# underscores.
tablename = re.split(r'__(?!_)', field)[0]
metarecord = MetaTable.get_by_dataset_name(tablename)
pt = metarecord.point_table
ctrees[pt.name] = parse_tree(pt, value)
# Just cleanliness, since we don't use this argument. Doesn't have
# to show up in the JSON response.
del args.data['dataset']
# If no dataset_name__in list was provided, have to fill it in by invoking
# MetaTable.index() here! Not in the validator. This way the list stays up
# to date.
if table_names is None:
table_names = MetaTable.index()
args.data['dataset_name__in'] = table_names
# If a single dataset was provided, it's the only thing we need to consider.
if dataset is not None:
table_names = [dataset.name]
del args.data['dataset_name__in']
# remove table names which wouldn't return anything for the query, given
# the time and geom constraints
try:
table_names = MetaTable.narrow_candidates(table_names, start_date, end_date, geom)
except Exception as e:
msg = 'Failed to gather candidate tables.'
return internal_error(msg, e)
# If there aren't any table names, it causes an error down the code. Better
# to return and inform them that the request wouldn't have found anything.
if not table_names:
return bad_request("Your request doesn't return any results. Try "
"adjusting your time constraint or location "
"parameters.")
try:
panel = MetaTable.timeseries_all(
table_names, agg, start_date, end_date, geom, ctrees
)
except Exception as e:
msg = 'Failed to construct timeseries.'
return internal_error(msg, e)
panel = MetaTable.attach_metadata(panel)
resp = json_response_base(args, panel, args.data)
datatype = args.data['data_type']
if datatype == 'json':
resp = make_response(json.dumps(resp, default=unknown_object_json_handler), 200)
resp.headers['Content-Type'] = 'application/json'
elif datatype == 'csv':
# response format
# temporal_group,dataset_name_1,dataset_name_2
# 2014-02-24 00:00:00,235,653
# 2014-03-03 00:00:00,156,624
fields = ['temporal_group']
for o in resp['objects']:
fields.append(o['dataset_name'])
csv_resp = []
i = 0
for k, g in groupby(resp['objects'], key=itemgetter('dataset_name')):
l_g = list(g)[0]
j = 0
for row in l_g['items']:
# first iteration, populate the first column with temporal_groups
if i == 0:
csv_resp.append([row['datetime']])
csv_resp[j].append(row['count'])
j += 1
i += 1
csv_resp.insert(0, fields)
csv_resp = make_csv(csv_resp)
resp = make_response(csv_resp, 200)
resp.headers['Content-Type'] = 'text/csv'
filedate = datetime.now().strftime('%Y-%m-%d')
resp.headers['Content-Disposition'] = 'attachment; filename=%s.csv' % filedate
return resp
def timeseries():
validator = ParamValidator()\
.set_optional('agg', agg_validator, 'week')\
.set_optional('data_type', make_format_validator(['json', 'csv']), 'json')\
.set_optional('dataset_name__in', list_of_datasets_validator, MetaTable.index)\
.set_optional('obs_date__ge', date_validator, datetime.now() - timedelta(days=90))\
.set_optional('obs_date__le', date_validator, datetime.now())\
.set_optional('location_geom__within', geom_validator, None)\
.set_optional('buffer', int_validator, 100)
err = validator.validate(request.args)
if err:
return bad_request(err)
geom = validator.get_geom()
table_names = validator.vals['dataset_name__in']
start_date = validator.vals['obs_date__ge']
end_date = validator.vals['obs_date__le']
agg = validator.vals['agg']
# Only examine tables that have a chance of containing records within the date and space boundaries.
try:
table_names = MetaTable.narrow_candidates(table_names, start_date, end_date, geom)
except Exception as e:
msg = 'Failed to gather candidate tables.'
return internal_error(msg, e)
try:
panel = MetaTable.timeseries_all(table_names=table_names,
agg_unit=agg,
start=start_date,
end=end_date,
geom=geom)
except Exception as e:
msg = 'Failed to construct timeseries.'
return internal_error(msg, e)
panel = MetaTable.attach_metadata(panel)
resp = json_response_base(validator, panel)
datatype = validator.vals['data_type']
if datatype == 'json':
resp = make_response(json.dumps(resp, default=dthandler), 200)
resp.headers['Content-Type'] = 'application/json'
elif datatype == 'csv':
# response format
# temporal_group,dataset_name_1,dataset_name_2
# 2014-02-24 00:00:00,235,653
# 2014-03-03 00:00:00,156,624
fields = ['temporal_group']
for o in resp['objects']:
fields.append(o['dataset_name'])
csv_resp = []
i = 0
for k,g in groupby(resp['objects'], key=itemgetter('dataset_name')):
l_g = list(g)[0]
j = 0
for row in l_g['items']:
# first iteration, populate the first column with temporal_groups
if i == 0:
csv_resp.append([row['datetime']])
csv_resp[j].append(row['count'])
j += 1
i += 1
csv_resp.insert(0, fields)
csv_resp = make_csv(csv_resp)
resp = make_response(csv_resp, 200)
resp.headers['Content-Type'] = 'text/csv'
filedate = datetime.now().strftime('%Y-%m-%d')
resp.headers['Content-Disposition'] = 'attachment; filename=%s.csv' % filedate
return resp
def _timeseries(args):
geom = args.data['geom']
dataset = args.data.get('dataset')
table_names = args.data['dataset_name__in']
start_date = args.data['obs_date__ge']
end_date = args.data['obs_date__le']
agg = args.data['agg']
# if a single dataset was provided, it's the only thing we need to consider
if dataset is not None:
table_names = [dataset.name]
# for the query's meta information, so that it doesn't show the index
del args.data['dataset_name__in']
# remove table names which wouldn't return anything for the query, given
# the time and geom constraints
try:
table_names = MetaTable.narrow_candidates(table_names, start_date, end_date, geom)
except Exception as e:
msg = 'Failed to gather candidate tables.'
return internal_error(msg, e)
# If there aren't any table names, it causes an error down the code. Better
# to return and inform them that the request wouldn't have found anything.
if not table_names:
return bad_request("Your request doesn't return any results. Try "
"adjusting your time constraint or location "
"parameters.")
try:
panel = MetaTable.timeseries_all(
table_names, agg, start_date, end_date, geom
)
except Exception as e:
msg = 'Failed to construct timeseries.'
return internal_error(msg, e)
panel = MetaTable.attach_metadata(panel)
resp = json_response_base(args, panel, args.data)
datatype = args.data['data_type']
if datatype == 'json':
resp = make_response(json.dumps(resp, default=unknown_object_json_handler), 200)
resp.headers['Content-Type'] = 'application/json'
elif datatype == 'csv':
# response format
# temporal_group,dataset_name_1,dataset_name_2
# 2014-02-24 00:00:00,235,653
# 2014-03-03 00:00:00,156,624
fields = ['temporal_group']
for o in resp['objects']:
fields.append(o['dataset_name'])
csv_resp = []
i = 0
for k, g in groupby(resp['objects'], key=itemgetter('dataset_name')):
l_g = list(g)[0]
j = 0
for row in l_g['items']:
# first iteration, populate the first column with temporal_groups
if i == 0:
csv_resp.append([row['datetime']])
csv_resp[j].append(row['count'])
j += 1
i += 1
csv_resp.insert(0, fields)
csv_resp = make_csv(csv_resp)
resp = make_response(csv_resp, 200)
resp.headers['Content-Type'] = 'text/csv'
filedate = datetime.now().strftime('%Y-%m-%d')
resp.headers['Content-Disposition'] = 'attachment; filename=%s.csv' % filedate
return resp
def timeseries():
validator = TimeseriesValidator()
deserialized_arguments = validator.load(request.args)
serialized_arguments = json.loads(validator.dumps(deserialized_arguments.data).data)
if deserialized_arguments.errors:
return make_error(deserialized_arguments.errors, 400, serialized_arguments)
qargs = deserialized_arguments.data
agg = qargs['agg']
data_type = qargs['data_type']
geom = qargs['location_geom__within']
pointset = qargs['dataset_name']
pointsets = qargs['dataset_name__in']
start_date = qargs['obs_date__ge']
end_date = qargs['obs_date__le']
ctrees = {}
raw_ctrees = {}
if has_tree_filters(request.args):
# Timeseries is a little tricky. If there aren't filters,
# it would be ridiculous to build a condition tree for every one.
for field, value in list(request.args.items()):
if 'filter' in field:
# This pattern matches the last occurrence of the '__' pattern.
# Prevents an error that is caused by dataset names with trailing
# underscores.
tablename = re.split(r'__(?!_)', field)[0]
metarecord = MetaTable.get_by_dataset_name(tablename)
pt = metarecord.point_table
ctrees[pt.name] = parse_tree(pt, json.loads(value))
raw_ctrees[pt.name] = json.loads(value)
point_set_names = [p.name for p in pointsets + [pointset] if p is not None]
if not point_set_names:
point_set_names = MetaTable.index()
results = MetaTable.timeseries_all(point_set_names, agg, start_date, end_date, geom, ctrees)
payload = {
'meta': {
'message': [],
'query': serialized_arguments,
'status': 'ok',
'total': len(results)
},
'objects': results
}
if ctrees:
payload['meta']['query']['filters'] = raw_ctrees
if data_type == 'json':
return jsonify(payload)
elif data_type == 'csv':
# response format
# temporal_group,dataset_name_1,dataset_name_2
# 2014-02-24 00:00:00,235,653
# 2014-03-03 00:00:00,156,624
fields = ['temporal_group']
for o in payload['objects']:
fields.append(o['dataset_name'])
csv_resp = []
i = 0
for k, g in groupby(payload['objects'], key=itemgetter('dataset_name')):
l_g = list(g)[0]
j = 0
for row in l_g['items']:
# first iteration, populate the first column with temporal_groups
if i == 0:
csv_resp.append([row['datetime']])
csv_resp[j].append(row['count'])
j += 1
i += 1
csv_resp.insert(0, fields)
csv_resp = make_csv(csv_resp)
resp = make_response(csv_resp, 200)
resp.headers['Content-Type'] = 'text/csv'
filedate = datetime.now().strftime('%Y-%m-%d')
resp.headers['Content-Disposition'] = 'attachment; filename=%s.csv' % filedate
return resp
