def get(self): context = { 'status': "white", 'version': get_version(), 'timestamps': { 'current': utcnow().strftime(JSON_FMT), } } latest = IngestionRecord.query.order_by(desc('finished')).first() if latest is not None: months = months_since(latest.finished) if months < 2: context["status"] = "green" elif months < 7: context["status"] = "yellow" else: context["status"] = "red" tsfields = context['timestamps'] tsfields['ingestion'] = latest.finished.strftime(JSON_FMT) tsfields['monthdelta'] = months return context
def get(self): context = { 'status': "white", 'version': get_version(), 'timestamps': { 'current': utcnow().strftime(JSON_FMT), } } latest = IngestionRecord.query.order_by(desc('finished')).first() if latest is not None: months = months_since(latest.finished) if months < 2: context["status"] = "green" elif months < 7: context["status"] = "yellow" else: context["status"] = "red" tsfields = context['timestamps'] tsfields['ingestion'] = latest.finished.strftime(JSON_FMT) tsfields['monthdelta'] = months return context
def test_heartbeat_version(self): """ Test the heartbeat version is correct """ response = self.client.get("/api/status/") self.assertEquals(response.status_code, 200) self.assertIn("version", response.json) self.assertEquals(elmr.get_version(), response.json["version"])
def test_heartbeat_version(self): """ Test the heartbeat version is correct """ response = self.client.get("/api/status/") self.assertEquals(response.status_code, 200) self.assertIn("version", response.json) self.assertEquals(elmr.get_version(), response.json["version"])
def ingest(**kwargs): """ Puts the entire ingestion package together with a single call. This function uses the `fetch.fetch_all` and `wrangle.wrangle` methods in a callback chain to accomplish its work. Note: all work is logged in the database as an IngestionRecord. :param kwargs: should be the keyword arguments to `fetch_all` """ startyear = int(kwargs.get('startyear', Config.STARTYEAR)) endyear = int(kwargs.get('endyear', Config.STARTYEAR)) title = kwargs.pop("title", "ELMR Ingestion Library") ## Create the log record log = IngestionRecord( title=title, version=elmr.get_version(), start_year=date(startyear, 1, 1), end_year=date(endyear, 1, 1), duration=0.0, started=datetime.now(), ) elmr.db.session.add(log) elmr.db.session.commit() ## Initiate the callback chain kwargs['callback'] = wrangle.wrangle duration, num_series, num_rows = fetch.fetch_all(**kwargs) ## Update the log record log.duration = duration log.finished = datetime.now() log.num_series = num_series log.num_added = num_rows[0] log.num_fetched = num_rows[1] elmr.db.session.commit() return log
def create_ingestion_record(self, days=1, duration=120): """ Creates an ingestion record `days` days ago with a duration of the amount passed into the function. Used to create different ingestions. """ started = datetime.now() - timedelta(days=days) finished = started + timedelta(seconds=duration) record = IngestionRecord(title=u"ELMR Testing Fake Ingestion Record", version=elmr.get_version(), start_year=datetime(2000, 1, 1, 12, 0, 0).date(), end_year=datetime(2015, 1, 1, 12, 0, 0).date(), duration=duration, num_series=1684, num_added=327213, num_fetched=333421, started=started, finished=finished) elmr.db.session.add(record) elmr.db.session.commit()
def create_ingestion_record(self, days=1, duration=120): """ Creates an ingestion record `days` days ago with a duration of the amount passed into the function. Used to create different ingestions. """ started = datetime.now() - timedelta(days=days) finished = started + timedelta(seconds=duration) record = IngestionRecord( title=u"ELMR Testing Fake Ingestion Record", version=elmr.get_version(), start_year=datetime(2000, 1, 1, 12, 0, 0).date(), end_year=datetime(2015, 1, 1, 12, 0, 0).date(), duration=duration, num_series=1684, num_added=327213, num_fetched=333421, started=started, finished=finished ) elmr.db.session.add(record) elmr.db.session.commit()
def get(self, source): """ For a single source, return a data detail view for all time series. """ # Ensure that source is allowed source = source.upper() if source in self.FORBIDDEN_SOURCES: context = { 'success': False, 'message': "Source '%s' is not allowed." % source, } return context, 400 if source not in self.ALLOWED_SOURCES: context = { 'success': False, 'message': "Source '%s' is not found." % source, } return context, 404 args = self.parser.parse_args() series = Series.query.filter_by(source=source) start = args.start_year or int(app.config['STARTYEAR']) finish = args.end_year or int(app.config['ENDYEAR']) context = { "title": "ELMR Ingested %s Data" % source, "version": get_version(), "period": { "start": start, "end": finish, }, "descriptions": {}, "data": [], } data = defaultdict(dict) for s in series.all(): context["descriptions"][s.blsid] = s.title records = SeriesRecord.query.filter_by(series_id=s.id) ryear = extract('year', SeriesRecord.period) records = records.filter(ryear >= start) records = records.filter(ryear <= finish) for record in records.all(): data[record.period][s.blsid] = record.value data = sorted(data.items(), key=itemgetter(0)) for (date, values) in data: values["YEAR"] = date.year values["MONTH"] = date.month values["DATE"] = date.strftime("%b %Y") context["data"].append(values) context['period']['start'] = data[0][0].strftime("%b %Y") context['period']['end'] = data[-1][0].strftime("%b %Y") return context
def get(self, source): """ For a single source, return a data detail view for all time series. """ # Ensure that source is allowed source = source.upper() if source in self.FORBIDDEN_SOURCES: context = { 'success': False, 'message': "Source '%s' is not allowed." % source, } return context, 400 if source not in self.ALLOWED_SOURCES: context = { 'success': False, 'message': "Source '%s' is not found." % source, } return context, 404 args = self.parser.parse_args() series = Series.query.filter_by(source=source) start = args.start_year or int(app.config['STARTYEAR']) finish = args.end_year or int(app.config['ENDYEAR']) context = { "title": "ELMR Ingested %s Data" % source, "version": get_version(), "period": { "start": start, "end": finish, }, "descriptions": {}, "data": [], } data = defaultdict(dict) for s in series.all(): context["descriptions"][s.blsid] = s.title records = SeriesRecord.query.filter_by(series_id=s.id) ryear = extract('year', SeriesRecord.period) records = records.filter(ryear >= start) records = records.filter(ryear <= finish) for record in records.all(): data[record.period][s.blsid] = record.value data = sorted(data.items(), key=itemgetter(0)) for (date, values) in data: values["YEAR"] = date.year values["MONTH"] = date.month values["DATE"] = date.strftime("%b %Y") context["data"].append(values) context['period']['start'] = data[0][0].strftime("%b %Y") context['period']['end'] = data[-1][0].strftime("%b %Y") return context
sys.path.append(BASE_DIR) ## Import ELMR Libraries import elmr from elmr.ingest import ingest from elmr.config import get_settings_object from elmr.delta import deltas ########################################################################## ## Script Definition ########################################################################## DESCRIPTION = "An administrative utility for the ELMR Project" EPILOG = "If there are any bugs or concerns, submit an issue on Github" VERSION = elmr.get_version() ########################################################################## ## Helper functions ########################################################################## def get_config(mode="development"): """ Perform dynamic loading of Config just like Flask! """ name = get_settings_object(mode) mod, cls = name.rsplit(".", 1) module = importlib.import_module(mod) return getattr(module, cls)
sys.path.append(BASE_DIR) ## Import ELMR Libraries import elmr from elmr.ingest import ingest from elmr.config import get_settings_object from elmr.delta import deltas ########################################################################## ## Script Definition ########################################################################## DESCRIPTION = "An administrative utility for the ELMR Project" EPILOG = "If there are any bugs or concerns, submit an issue on Github" VERSION = elmr.get_version() ########################################################################## ## Helper functions ########################################################################## def get_config(mode="development"): """ Perform dynamic loading of Config just like Flask! """ name = get_settings_object(mode) mod, cls = name.rsplit(".", 1) module = importlib.import_module(mod) return getattr(module, cls)