def log_get(recipe_id=[], timezone='America/Los_Angeles', days=1):
"""Returns last actionable job run for a specific recipe or all recipes.
Pulls status entries from StackDriver in reverse order. A single recipe may
be run multiple times for multiple tasks at different hours, do not
assume a JOB_END means a recipe is complete. Only way to ensure a recipe is
complete
is to compare all tasks run against all tasks in recipe ( not done by log
code).
Args: - recipe_id ( string or list ) - Optional, if provided returns a single
record for a single job. - timezone ( string ) - The local timezone to cast
all record times into.
Returns:
- ( iterator ) - Each log entry.
"""
body = {
'resourceNames': [
'projects/%s' % UI_PROJECT,
],
'filter':
'\
logName="projects/%s/logs/StarThinker" \
AND labels.version="%s" \
AND labels.layer="JOB" \
' % (UI_PROJECT, LOG_VERSION),
'orderBy':
'timestamp desc',
'pageSize':
1000
}
if recipe_id:
if isinstance(recipe_id, str):
recipe_id = [recipe_id]
body['filter'] += ' AND ( %s )' % ' OR '.join('operation.id="%s"' % r
for r in recipe_id)
for entry in API_StackDriver(
Configuration(service=UI_SERVICE, project=UI_PROJECT),
'service',
iterate=True).entries().list(body=body).execute():
yield entry
def log_put(event, severity, job=None, text=None):
"""Generic log writer used by helper functions. Writes to StackDriver.
Creates a record that can be read using log_get function. Entire recipe is
logged, worker data and stdout and stderr are added to the JOSN under worker key.
Only JOB_EXCEPTION and MANAGER_EXCEPTION logs to text in case JSON is corrupt,
everythng else is JSON.
Do not call this directly, use helper functions instead:
- log_manager_start
- log_manager_error
- log_manager_end
- log_job_dispatch
- log_job_receive
- log_job_start
- log_job_end
- log_job_error
- log_job_timeout
WARNING: Do not corrupt recipe in log code, it is actively being used by workers while being logged.
Args:
- event ( string ): One of the JOB_* enums.
- job ( json ): Recipe workflow to execute.
- severity ( enum ): Stackdriver severity level.
- text ( string ): Mesaging output form the task. Usual stdout and stderr.
"""
if VERBOSE: print "LOGGING:", event, severity, text or ''
body = {
"entries": [{
"logName": "projects/%s/logs/StarThinker" % UI_PROJECT,
"severity": severity,
"resource": {
"type": "project",
"labels": {
"key": UI_PROJECT
},
},
"labels": {
"version": LOG_VERSION,
"layer": event.split('_')[0],
"event": event,
"instance": get_instance_name(),
},
#"operation": {
# "id": string
# "producer": string
# "first": False,
# "last": False,
#},
# already in recipe worker logging task and instance, does this have additional value?
#"sourceLocation": {
# "file": string,
# "line": string,
# "function": string
#},
}],
"partialSuccess":
False,
"dryRun":
False
}
if text is not None:
body['entries'][0]["textPayload"] = text
else:
# Removing tasks from job REMOVES ALL POTENTIAL CREDENTIALS IN CODE
job_buffer = json.loads(
json.dumps(job, indent=2, sort_keys=True, default=str))
if 'tasks' in job_buffer['recipe']: del job_buffer['recipe']['tasks']
if 'auth' in job_buffer['recipe']['setup']:
del job_buffer['recipe']['setup']['auth']
body['entries'][0]["jsonPayload"] = job_buffer
project.initialize(_service=UI_SERVICE, _project=UI_PROJECT)
API_StackDriver("service").entries().write(body=body).execute()
def log_put(event, severity, job=None, text=None, payload=None):
"""Generic log writer used by helper functions.
Writes to StackDriver.
Creates a record that can be read using log_get function. Entire recipe is
logged, worker data and stdout and stderr are added to the JOSN under worker
key.
Only JOB_EXCEPTION and MANAGER_EXCEPTION logs to text in case JSON is corrupt,
everythng else is JSON.
Do not call this directly, use helper functions instead:
- log_manager_start
- log_manager_timeout
- log_manager_error
- log_manager_end
- log_job_dispatch
- log_job_receive
- log_job_start
- log_job_end
- log_job_error
- log_job_timeout
WARNING: Do not corrupt recipe in log code, it is actively being used by
workers while being logged.
Args:
- event ( string ): One of the JOB_* enums.
- severity ( enum ): Stackdriver severity level.
- job ( json ): Recipe workflow to execute.
- text ( string ): Messaging output form the task. Usual stdout and stderr.
- payload ( json ): Output from the scaler or any generic json payload.
"""
if VERBOSE:
print('LOGGING:', event, severity, text or '')
body = {
'entries': [{
'logName': 'projects/%s/logs/StarThinker' % UI_PROJECT,
'severity': severity,
'resource': {
'type': 'project',
'labels': {
'key': UI_PROJECT
},
},
'labels': {
'version': LOG_VERSION,
'layer': event.split('_')[0],
'event': event,
'instance': get_instance_name(),
},
#"operation": {
# "id": string
# "producer": string
# "first": False,
# "last": False,
#},
# already in recipe worker logging task and instance, does this have additional value?
#"sourceLocation": {
# "file": string,
# "line": string,
# "function": string
#},
}],
'partialSuccess':
False,
'dryRun':
False
}
if text is not None:
body['entries'][0]['textPayload'] = text
elif payload is not None:
body['entries'][0]['jsonPayload'] = payload
else:
# Removing tasks from job REMOVES ALL POTENTIAL CREDENTIALS IN CODE
job_buffer = json.loads(
json.dumps(job, indent=2, sort_keys=True, default=str))
if 'tasks' in job_buffer['recipe']:
del job_buffer['recipe']['tasks']
if 'auth' in job_buffer['recipe']['setup']:
del job_buffer['recipe']['setup']['auth']
body['entries'][0]['jsonPayload'] = job_buffer
try:
API_StackDriver(Configuration(service=UI_SERVICE, project=UI_PROJECT),
'service').entries().write(body=body).execute()
except:
print('LOG EVENT ERROR')