def get_data():
"""HTTP endpoint to get data.
Retrieves downsampled data, that are within the given time range, from preprocessed raw files.
HTTP Args:
name: A string representing the fill name of the file user wish to
view. (example: DMM_res.csv)
strategy: A string representing the selected downsample strategy.
start (optional): An int representing the start of time span user wish to view.
end (optional): An int representing the end of time span user wish to view.
"""
name = request.args.get('name', type=str)
strategy = request.args.get('strategy', default='avg', type=str)
start = request.args.get('start', default=None, type=int)
end = request.args.get('end', default=None, type=int)
number = request.args.get(
'number', default=NUMBER_OF_RECORDS_PER_REQUEST, type=int)
if name is None:
warning('Empty file name.')
response = make_response('Empty file name')
return response, 400
if not strategy in STRATEGIES:
warning('Incorrect Strategy: %s', strategy)
response = make_response('Incorrect Strategy: {}'.format(strategy))
return response, 400
client = storage.Client()
fetcher = DataFetcher(name, PREPROCESS_DIR,
client.bucket(PREPROCESS_BUCKET))
if not fetcher.is_preprocessed():
response = make_response('Preprocessing incomplete.')
return response, 404
data, frequency_ratio = fetcher.fetch(
strategy, number, start, end)
response_data = {
'data': data,
'frequency_ratio': frequency_ratio
}
response = app.make_response(jsonify(response_data))
response.headers['Access-Control-Allow-Credentials'] = 'true'
return response
from data_fetcher import DataFetcher
# Read list of URLs
sites = []
with open('top-dk.csv') as in_file:
for line in in_file:
sites.append('http://www.' + line.split(',')[1].strip())
# Fetch sites
fetcher = DataFetcher(sites)
fetcher.fetch(stop=10)