def get(self):
context = {
'status': "white",
'version': get_version(),
'timestamps': {
'current': utcnow().strftime(JSON_FMT),
}
}
latest = IngestionRecord.query.order_by(desc('finished')).first()
if latest is not None:
months = months_since(latest.finished)
if months < 2:
context["status"] = "green"
elif months < 7:
context["status"] = "yellow"
else:
context["status"] = "red"
tsfields = context['timestamps']
tsfields['ingestion'] = latest.finished.strftime(JSON_FMT)
tsfields['monthdelta'] = months
return context
def get(self):
context = {
'status': "white",
'version': get_version(),
'timestamps': {
'current': utcnow().strftime(JSON_FMT),
}
}
latest = IngestionRecord.query.order_by(desc('finished')).first()
if latest is not None:
months = months_since(latest.finished)
if months < 2:
context["status"] = "green"
elif months < 7:
context["status"] = "yellow"
else:
context["status"] = "red"
tsfields = context['timestamps']
tsfields['ingestion'] = latest.finished.strftime(JSON_FMT)
tsfields['monthdelta'] = months
return context
def test_heartbeat_version(self):
"""
Test the heartbeat version is correct
"""
response = self.client.get("/api/status/")
self.assertEquals(response.status_code, 200)
self.assertIn("version", response.json)
self.assertEquals(elmr.get_version(), response.json["version"])
def test_heartbeat_version(self):
"""
Test the heartbeat version is correct
"""
response = self.client.get("/api/status/")
self.assertEquals(response.status_code, 200)
self.assertIn("version", response.json)
self.assertEquals(elmr.get_version(), response.json["version"])
def ingest(**kwargs):
"""
Puts the entire ingestion package together with a single call. This
function uses the `fetch.fetch_all` and `wrangle.wrangle` methods in a
callback chain to accomplish its work.
Note: all work is logged in the database as an IngestionRecord.
:param kwargs: should be the keyword arguments to `fetch_all`
"""
startyear = int(kwargs.get('startyear', Config.STARTYEAR))
endyear = int(kwargs.get('endyear', Config.STARTYEAR))
title = kwargs.pop("title", "ELMR Ingestion Library")
## Create the log record
log = IngestionRecord(
title=title,
version=elmr.get_version(),
start_year=date(startyear, 1, 1),
end_year=date(endyear, 1, 1),
duration=0.0,
started=datetime.now(),
)
elmr.db.session.add(log)
elmr.db.session.commit()
## Initiate the callback chain
kwargs['callback'] = wrangle.wrangle
duration, num_series, num_rows = fetch.fetch_all(**kwargs)
## Update the log record
log.duration = duration
log.finished = datetime.now()
log.num_series = num_series
log.num_added = num_rows[0]
log.num_fetched = num_rows[1]
elmr.db.session.commit()
return log
def create_ingestion_record(self, days=1, duration=120):
"""
Creates an ingestion record `days` days ago with a duration of the
amount passed into the function. Used to create different ingestions.
"""
started = datetime.now() - timedelta(days=days)
finished = started + timedelta(seconds=duration)
record = IngestionRecord(title=u"ELMR Testing Fake Ingestion Record",
version=elmr.get_version(),
start_year=datetime(2000, 1, 1, 12, 0,
0).date(),
end_year=datetime(2015, 1, 1, 12, 0,
0).date(),
duration=duration,
num_series=1684,
num_added=327213,
num_fetched=333421,
started=started,
finished=finished)
elmr.db.session.add(record)
elmr.db.session.commit()
def create_ingestion_record(self, days=1, duration=120):
"""
Creates an ingestion record `days` days ago with a duration of the
amount passed into the function. Used to create different ingestions.
"""
started = datetime.now() - timedelta(days=days)
finished = started + timedelta(seconds=duration)
record = IngestionRecord(
title=u"ELMR Testing Fake Ingestion Record",
version=elmr.get_version(),
start_year=datetime(2000, 1, 1, 12, 0, 0).date(),
end_year=datetime(2015, 1, 1, 12, 0, 0).date(),
duration=duration,
num_series=1684,
num_added=327213,
num_fetched=333421,
started=started,
finished=finished
)
elmr.db.session.add(record)
elmr.db.session.commit()
def get(self, source):
"""
For a single source, return a data detail view for all time series.
"""
# Ensure that source is allowed
source = source.upper()
if source in self.FORBIDDEN_SOURCES:
context = {
'success': False,
'message': "Source '%s' is not allowed." % source,
}
return context, 400
if source not in self.ALLOWED_SOURCES:
context = {
'success': False,
'message': "Source '%s' is not found." % source,
}
return context, 404
args = self.parser.parse_args()
series = Series.query.filter_by(source=source)
start = args.start_year or int(app.config['STARTYEAR'])
finish = args.end_year or int(app.config['ENDYEAR'])
context = {
"title": "ELMR Ingested %s Data" % source,
"version": get_version(),
"period": {
"start": start,
"end": finish,
},
"descriptions": {},
"data": [],
}
data = defaultdict(dict)
for s in series.all():
context["descriptions"][s.blsid] = s.title
records = SeriesRecord.query.filter_by(series_id=s.id)
ryear = extract('year', SeriesRecord.period)
records = records.filter(ryear >= start)
records = records.filter(ryear <= finish)
for record in records.all():
data[record.period][s.blsid] = record.value
data = sorted(data.items(), key=itemgetter(0))
for (date, values) in data:
values["YEAR"] = date.year
values["MONTH"] = date.month
values["DATE"] = date.strftime("%b %Y")
context["data"].append(values)
context['period']['start'] = data[0][0].strftime("%b %Y")
context['period']['end'] = data[-1][0].strftime("%b %Y")
return context
def get(self, source):
"""
For a single source, return a data detail view for all time series.
"""
# Ensure that source is allowed
source = source.upper()
if source in self.FORBIDDEN_SOURCES:
context = {
'success': False,
'message': "Source '%s' is not allowed." % source,
}
return context, 400
if source not in self.ALLOWED_SOURCES:
context = {
'success': False,
'message': "Source '%s' is not found." % source,
}
return context, 404
args = self.parser.parse_args()
series = Series.query.filter_by(source=source)
start = args.start_year or int(app.config['STARTYEAR'])
finish = args.end_year or int(app.config['ENDYEAR'])
context = {
"title": "ELMR Ingested %s Data" % source,
"version": get_version(),
"period": {
"start": start,
"end": finish,
},
"descriptions": {},
"data": [],
}
data = defaultdict(dict)
for s in series.all():
context["descriptions"][s.blsid] = s.title
records = SeriesRecord.query.filter_by(series_id=s.id)
ryear = extract('year', SeriesRecord.period)
records = records.filter(ryear >= start)
records = records.filter(ryear <= finish)
for record in records.all():
data[record.period][s.blsid] = record.value
data = sorted(data.items(), key=itemgetter(0))
for (date, values) in data:
values["YEAR"] = date.year
values["MONTH"] = date.month
values["DATE"] = date.strftime("%b %Y")
context["data"].append(values)
context['period']['start'] = data[0][0].strftime("%b %Y")
context['period']['end'] = data[-1][0].strftime("%b %Y")
return context
sys.path.append(BASE_DIR)
## Import ELMR Libraries
import elmr
from elmr.ingest import ingest
from elmr.config import get_settings_object
from elmr.delta import deltas
##########################################################################
## Script Definition
##########################################################################
DESCRIPTION = "An administrative utility for the ELMR Project"
EPILOG = "If there are any bugs or concerns, submit an issue on Github"
VERSION = elmr.get_version()
##########################################################################
## Helper functions
##########################################################################
def get_config(mode="development"):
"""
Perform dynamic loading of Config just like Flask!
"""
name = get_settings_object(mode)
mod, cls = name.rsplit(".", 1)
module = importlib.import_module(mod)
return getattr(module, cls)
sys.path.append(BASE_DIR)
## Import ELMR Libraries
import elmr
from elmr.ingest import ingest
from elmr.config import get_settings_object
from elmr.delta import deltas
##########################################################################
## Script Definition
##########################################################################
DESCRIPTION = "An administrative utility for the ELMR Project"
EPILOG = "If there are any bugs or concerns, submit an issue on Github"
VERSION = elmr.get_version()
##########################################################################
## Helper functions
##########################################################################
def get_config(mode="development"):
"""
Perform dynamic loading of Config just like Flask!
"""
name = get_settings_object(mode)
mod, cls = name.rsplit(".", 1)
module = importlib.import_module(mod)
return getattr(module, cls)