Example #1
0
def ingest_next_month():
    try:
        # verify that this is a cron job request
        is_cron = flask.request.headers['X-Appengine-Cron']
        logging.info('Received cron request {}'.format(is_cron))

        # next month
        bucket = CLOUD_STORAGE_BUCKET
        year, month = ingest_flights.next_month(bucket)
        status = 'scheduling ingest of year={} month={}'.format(year, month)
        logging.info(status)

        # ingest ...
        gcsfile = ingest_flights.ingest(year, month, bucket)
        status = 'successfully ingested={}'.format(gcsfile)
        logging.info(status)

    except ingest_flights.DataUnavailable:
        status = 'File for {}-{} not available yet ...'.format(year, month)
        logging.info(status)

    except KeyError as e:
        status = '<html>Sorry, this capability is accessible only by the Cron service, but I got a KeyError for {} -- try invoking it from <a href="{}"> the GCP console / AppEngine / taskqueues </a></html>'.format(
            e, 'http://console.cloud.google.com/appengine/taskqueues?tab=CRON')
        logging.info('Rejected non-Cron request')

    return status
Example #2
0
def ingest_next_month():

    try:

        # Verify that this is a cron job request
        is_cron = flask.request.headers['X-Appengine-Cron']
        logging.info('Received cron request {}'.format(is_cron))

        # CLOUD_STORAGE_BUCKET will be defined in app.yaml
        bucket = os.environ['CLOUD_STORAGE_BUCKET']

        # Runtime environment variable from google app engine
        project = os.environ['GOOGLE_CLOUD_PROJECT']

        year, month = ingest_flights.next_month(bucket, project)

        gcsfile = ingest_flights.ingest(year, month, bucket, project)

        status = "Successfully ingested {}".format(gcsfile)
    except KeyError as e:
        logging.info('Rejected non-Cron request')

    except ingest_flights.DataUnavailable as e:
        status = "File for {}.{} not available yet ...".format(year, month)

    logging.info(status)
    return status
Example #3
0
def ingest_next_month():
    try:
         # verify that this is a cron job request
         is_cron = flask.request.headers['X-Appengine-Cron']
         logging.info('Received cron request {}'.format(is_cron))

         # next month
         bucket = CLOUD_STORAGE_BUCKET
         year, month = ingest_flights.next_month(bucket)
         status = 'scheduling ingest of year={} month={}'.format(year, month)
         logging.info(status)

         # ingest ...
         gcsfile = ingest_flights.ingest(year, month, bucket)
         status = 'successfully ingested={}'.format(gcsfile)
         logging.info(status)

    except ingest_flights.DataUnavailable:
         status = 'File for {}-{} not available yet ...'.format(year, month)
         logging.info(status)

    except KeyError as e:
         status = '<html>Sorry, this capability is accessible only by the Cron service, but I got a KeyError for {} -- try invoking it from <a href="{}"> the GCP console / AppEngine / taskqueues </a></html>'.format(e, 'http://console.cloud.google.com/appengine/taskqueues?tab=CRON')
         logging.info('Rejected non-Cron request')

    return status
def ingest_next_month():
    try:
        # next month
        bucket = CLOUD_STORAGE_BUCKET
        year, month = ingest_flights.next_month(bucket)
        logging.info('Ingesting year={} month={}'.format(year, month))

        # ingest
        gcsfile = ingest_flights.ingest(year, month, bucket)

        # return page, and log
        status = 'Successfully ingested {}'.format(gcsfile)
    except DataUnavailable:
        status = 'File for {}-{} not available yet ...'.format(year, month)
    logging.info(status)
    return status
Example #5
0
def ingest_flights():
    # noinspection PyBroadException
    try:
        logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO)
        json = request.get_json(force=True) # https://stackoverflow.com/questions/53216177/http-triggering-cloud-function-with-cloud-scheduler/60615210#60615210

        year = escape(json['year']) if 'year' in json else None
        month = escape(json['month']) if 'month' in json else None
        bucket = escape(json['bucket'])  # required

        if year is None or month is None or len(year) == 0 or len(month) == 0:
            year, month = next_month(bucket)
        logging.debug('Ingesting year={} month={}'.format(year, month))
        tableref, numrows = ingest(year, month, bucket)
        ok = 'Success ... ingested {} rows to {}'.format(numrows, tableref)
        logging.info(ok)
        return ok
    except Exception as e:
        logging.exception("Failed to ingest ... try again later?")