def ingest_next_month():
try:
# verify that this is a cron job request
is_cron = flask.request.headers['X-Appengine-Cron']
logging.info('Received cron request {}'.format(is_cron))
# next month
bucket = CLOUD_STORAGE_BUCKET
year, month = ingest_flights.next_month(bucket)
status = 'scheduling ingest of year={} month={}'.format(year, month)
logging.info(status)
# ingest ...
gcsfile = ingest_flights.ingest(year, month, bucket)
status = 'successfully ingested={}'.format(gcsfile)
logging.info(status)
except ingest_flights.DataUnavailable:
status = 'File for {}-{} not available yet ...'.format(year, month)
logging.info(status)
except KeyError as e:
status = '<html>Sorry, this capability is accessible only by the Cron service, but I got a KeyError for {} -- try invoking it from <a href="{}"> the GCP console / AppEngine / taskqueues </a></html>'.format(
e, 'http://console.cloud.google.com/appengine/taskqueues?tab=CRON')
logging.info('Rejected non-Cron request')
return status
def ingest_next_month():
try:
# Verify that this is a cron job request
is_cron = flask.request.headers['X-Appengine-Cron']
logging.info('Received cron request {}'.format(is_cron))
# CLOUD_STORAGE_BUCKET will be defined in app.yaml
bucket = os.environ['CLOUD_STORAGE_BUCKET']
# Runtime environment variable from google app engine
project = os.environ['GOOGLE_CLOUD_PROJECT']
year, month = ingest_flights.next_month(bucket, project)
gcsfile = ingest_flights.ingest(year, month, bucket, project)
status = "Successfully ingested {}".format(gcsfile)
except KeyError as e:
logging.info('Rejected non-Cron request')
except ingest_flights.DataUnavailable as e:
status = "File for {}.{} not available yet ...".format(year, month)
logging.info(status)
return status
def ingest_next_month():
try:
# verify that this is a cron job request
is_cron = flask.request.headers['X-Appengine-Cron']
logging.info('Received cron request {}'.format(is_cron))
# next month
bucket = CLOUD_STORAGE_BUCKET
year, month = ingest_flights.next_month(bucket)
status = 'scheduling ingest of year={} month={}'.format(year, month)
logging.info(status)
# ingest ...
gcsfile = ingest_flights.ingest(year, month, bucket)
status = 'successfully ingested={}'.format(gcsfile)
logging.info(status)
except ingest_flights.DataUnavailable:
status = 'File for {}-{} not available yet ...'.format(year, month)
logging.info(status)
except KeyError as e:
status = '<html>Sorry, this capability is accessible only by the Cron service, but I got a KeyError for {} -- try invoking it from <a href="{}"> the GCP console / AppEngine / taskqueues </a></html>'.format(e, 'http://console.cloud.google.com/appengine/taskqueues?tab=CRON')
logging.info('Rejected non-Cron request')
return status
def ingest_next_month():
try:
# next month
bucket = CLOUD_STORAGE_BUCKET
year, month = ingest_flights.next_month(bucket)
logging.info('Ingesting year={} month={}'.format(year, month))
# ingest
gcsfile = ingest_flights.ingest(year, month, bucket)
# return page, and log
status = 'Successfully ingested {}'.format(gcsfile)
except DataUnavailable:
status = 'File for {}-{} not available yet ...'.format(year, month)
logging.info(status)
return status
def ingest_flights():
# noinspection PyBroadException
try:
logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO)
json = request.get_json(force=True) # https://stackoverflow.com/questions/53216177/http-triggering-cloud-function-with-cloud-scheduler/60615210#60615210
year = escape(json['year']) if 'year' in json else None
month = escape(json['month']) if 'month' in json else None
bucket = escape(json['bucket']) # required
if year is None or month is None or len(year) == 0 or len(month) == 0:
year, month = next_month(bucket)
logging.debug('Ingesting year={} month={}'.format(year, month))
tableref, numrows = ingest(year, month, bucket)
ok = 'Success ... ingested {} rows to {}'.format(numrows, tableref)
logging.info(ok)
return ok
except Exception as e:
logging.exception("Failed to ingest ... try again later?")
