def log_get(recipe_id=[], timezone='America/Los_Angeles', days=1): """Returns last actionable job run for a specific recipe or all recipes. Pulls status entries from StackDriver in reverse order. A single recipe may be run multiple times for multiple tasks at different hours, do not assume a JOB_END means a recipe is complete. Only way to ensure a recipe is complete is to compare all tasks run against all tasks in recipe ( not done by log code). Args: - recipe_id ( string or list ) - Optional, if provided returns a single record for a single job. - timezone ( string ) - The local timezone to cast all record times into. Returns: - ( iterator ) - Each log entry. """ body = { 'resourceNames': [ 'projects/%s' % UI_PROJECT, ], 'filter': '\ logName="projects/%s/logs/StarThinker" \ AND labels.version="%s" \ AND labels.layer="JOB" \ ' % (UI_PROJECT, LOG_VERSION), 'orderBy': 'timestamp desc', 'pageSize': 1000 } if recipe_id: if isinstance(recipe_id, str): recipe_id = [recipe_id] body['filter'] += ' AND ( %s )' % ' OR '.join('operation.id="%s"' % r for r in recipe_id) for entry in API_StackDriver( Configuration(service=UI_SERVICE, project=UI_PROJECT), 'service', iterate=True).entries().list(body=body).execute(): yield entry
def log_put(event, severity, job=None, text=None): """Generic log writer used by helper functions. Writes to StackDriver. Creates a record that can be read using log_get function. Entire recipe is logged, worker data and stdout and stderr are added to the JOSN under worker key. Only JOB_EXCEPTION and MANAGER_EXCEPTION logs to text in case JSON is corrupt, everythng else is JSON. Do not call this directly, use helper functions instead: - log_manager_start - log_manager_error - log_manager_end - log_job_dispatch - log_job_receive - log_job_start - log_job_end - log_job_error - log_job_timeout WARNING: Do not corrupt recipe in log code, it is actively being used by workers while being logged. Args: - event ( string ): One of the JOB_* enums. - job ( json ): Recipe workflow to execute. - severity ( enum ): Stackdriver severity level. - text ( string ): Mesaging output form the task. Usual stdout and stderr. """ if VERBOSE: print "LOGGING:", event, severity, text or '' body = { "entries": [{ "logName": "projects/%s/logs/StarThinker" % UI_PROJECT, "severity": severity, "resource": { "type": "project", "labels": { "key": UI_PROJECT }, }, "labels": { "version": LOG_VERSION, "layer": event.split('_')[0], "event": event, "instance": get_instance_name(), }, #"operation": { # "id": string # "producer": string # "first": False, # "last": False, #}, # already in recipe worker logging task and instance, does this have additional value? #"sourceLocation": { # "file": string, # "line": string, # "function": string #}, }], "partialSuccess": False, "dryRun": False } if text is not None: body['entries'][0]["textPayload"] = text else: # Removing tasks from job REMOVES ALL POTENTIAL CREDENTIALS IN CODE job_buffer = json.loads( json.dumps(job, indent=2, sort_keys=True, default=str)) if 'tasks' in job_buffer['recipe']: del job_buffer['recipe']['tasks'] if 'auth' in job_buffer['recipe']['setup']: del job_buffer['recipe']['setup']['auth'] body['entries'][0]["jsonPayload"] = job_buffer project.initialize(_service=UI_SERVICE, _project=UI_PROJECT) API_StackDriver("service").entries().write(body=body).execute()
def log_put(event, severity, job=None, text=None, payload=None): """Generic log writer used by helper functions. Writes to StackDriver. Creates a record that can be read using log_get function. Entire recipe is logged, worker data and stdout and stderr are added to the JOSN under worker key. Only JOB_EXCEPTION and MANAGER_EXCEPTION logs to text in case JSON is corrupt, everythng else is JSON. Do not call this directly, use helper functions instead: - log_manager_start - log_manager_timeout - log_manager_error - log_manager_end - log_job_dispatch - log_job_receive - log_job_start - log_job_end - log_job_error - log_job_timeout WARNING: Do not corrupt recipe in log code, it is actively being used by workers while being logged. Args: - event ( string ): One of the JOB_* enums. - severity ( enum ): Stackdriver severity level. - job ( json ): Recipe workflow to execute. - text ( string ): Messaging output form the task. Usual stdout and stderr. - payload ( json ): Output from the scaler or any generic json payload. """ if VERBOSE: print('LOGGING:', event, severity, text or '') body = { 'entries': [{ 'logName': 'projects/%s/logs/StarThinker' % UI_PROJECT, 'severity': severity, 'resource': { 'type': 'project', 'labels': { 'key': UI_PROJECT }, }, 'labels': { 'version': LOG_VERSION, 'layer': event.split('_')[0], 'event': event, 'instance': get_instance_name(), }, #"operation": { # "id": string # "producer": string # "first": False, # "last": False, #}, # already in recipe worker logging task and instance, does this have additional value? #"sourceLocation": { # "file": string, # "line": string, # "function": string #}, }], 'partialSuccess': False, 'dryRun': False } if text is not None: body['entries'][0]['textPayload'] = text elif payload is not None: body['entries'][0]['jsonPayload'] = payload else: # Removing tasks from job REMOVES ALL POTENTIAL CREDENTIALS IN CODE job_buffer = json.loads( json.dumps(job, indent=2, sort_keys=True, default=str)) if 'tasks' in job_buffer['recipe']: del job_buffer['recipe']['tasks'] if 'auth' in job_buffer['recipe']['setup']: del job_buffer['recipe']['setup']['auth'] body['entries'][0]['jsonPayload'] = job_buffer try: API_StackDriver(Configuration(service=UI_SERVICE, project=UI_PROJECT), 'service').entries().write(body=body).execute() except: print('LOG EVENT ERROR')