def default(self, o):
"""."""
if isinstance(o, uuid.UUID):
return str(o)
elif isinstance(o, Decimal):
return float(o)
elif isinstance(o, Enum):
return o.name
elif isinstance(o, PagedResponse):
json_record = OrderedDict([("records", o.records)])
if o.total_records or o.total_filtered:
meta = OrderedDict()
if o.total_records is not None:
meta["records_total"] = o.total_records
if o.total_filtered is not None:
meta["records_filtered"] = o.total_filtered
json_record["_meta"] = meta
return json_record
elif isinstance(o, ErrorResponse):
json_record = OrderedDict([("code", o.code),
("message", o.message)])
if o.errors:
json_record["errors"] = [{
"field": f,
"errors": e
} for f, e in o.errors]
return json_record
elif hasattr(o, "__json__"):
return o.__json__()
elif hasattr(o, "__table__"):
unloaded = inspect(o).unloaded
_v = {
k: getattr(o, k)
for k in o.__mapper__.column_attrs.keys() if k not in unloaded
}
for k in getattr(o, "__includes__", {}):
_v[k] = getattr(o, k)
for k in getattr(o, "__excludes__", {}):
del _v[k]
return _v
return JSONEncoder.default(self, o)
def get_query_args():
g.query_args = {}
def to_int(q_arg, value):
try:
return int(value)
except ValueError as e:
log.exception("Query Argument %s = %s is not a valid int" % (q_arg, value))
e = ValueError(
"Expecting integer for argument %s, found %r" % (q_arg, str(value))
)
e.codes = ("INVALID_QUERY_ARGUMENT", 400)
raise e
def to_str(q_arg, value):
return value
def to_bool(q_arg, value):
value = value.strip().lower()
if value in set(["1", "true"]):
return True
elif value in set(["0", "false"]):
return False
else:
log.exception(
"Query Argument %s = %s is not a valid boolean" % (q_arg, value)
)
e = ValueError(
"Expecting boolean for argument %s, found %r" % (q_arg, str(value))
)
e.codes = ("INVALID_QUERY_ARGUMENT", 400)
raise e
query_args = OrderedDict(
[
("pretty-print", to_bool),
("start-index", to_int),
("max-results", to_int),
("query", to_str),
("order", to_str),
]
)
is_post = request.method == "POST"
for arg, cast in query_args.items():
if arg in request.args:
g.query_args[arg.replace("-", "_")] = cast(arg, request.args.get(arg))
# POST Query Argument overrides GET Query Argument with the same name
if is_post and arg in request.form:
g.query_args[arg.replace("-", "_")] = cast(arg, request.form[arg])
def get_query_args():
g.query_args = {}
def to_int(q_arg, value):
try:
return int(value)
except ValueError as e:
log.exception("Query Argument %s = %s is not a valid int" %
(q_arg, value))
e = ValueError("Expecting integer for argument %s, found %r" %
(q_arg, str(value)))
e.codes = ("INVALID_QUERY_ARGUMENT", 400)
raise e
def to_str(q_arg, value):
return value
def to_bool(q_arg, value):
value = value.strip().lower()
if value in set(["1", "true"]):
return True
elif value in set(["0", "false"]):
return False
else:
log.exception("Query Argument %s = %s is not a valid boolean" %
(q_arg, value))
e = ValueError("Expecting boolean for argument %s, found %r" %
(q_arg, str(value)))
e.codes = ("INVALID_QUERY_ARGUMENT", 400)
raise e
query_args = OrderedDict([
("pretty-print", to_bool),
("start-index", to_int),
("max-results", to_int),
("query", to_str),
("order", to_str),
])
is_post = request.method == "POST"
for arg, cast in query_args.items():
if arg in request.args:
g.query_args[arg.replace("-", "_")] = cast(arg,
request.args.get(arg))
# POST Query Argument overrides GET Query Argument with the same name
if is_post and arg in request.form:
g.query_args[arg.replace("-", "_")] = cast(arg, request.form[arg])
def get_query_args():
g.query_args = {}
def to_int(q_arg, value):
try:
return int(value)
except ValueError as e:
log.exception("Query Argument {} = {} is not a valid int".format(
q_arg, value))
e = ValueError(
"Expecting integer for argument {}, found {!r}".format(
q_arg, str(value)))
e.codes = ("INVALID_QUERY_ARGUMENT", 400)
raise e from None
def to_str(q_arg, value):
return value
def to_bool(q_arg, value):
value = value.strip().lower()
if value in {"1", "true"}:
return True
elif value in {"0", "false"}:
return False
else:
log.exception(
"Query Argument {} = {} is not a valid boolean".format(
q_arg, value))
e = ValueError(
"Expecting boolean for argument {}, found {!r}".format(
q_arg, str(value)))
e.codes = ("INVALID_QUERY_ARGUMENT", 400)
raise e
query_args = OrderedDict([
("pretty-print", to_bool),
("start-index", to_int),
("max-results", to_int),
("query", to_str),
("order", to_str),
])
for arg, cast in query_args.items():
if arg in request.args:
g.query_args[arg.replace("-", "_")] = cast(arg,
request.args.get(arg))
def get_query_args():
g.query_args = {}
def to_int(q_arg, value):
try:
return int(value)
except ValueError as e:
log.exception('Query Argument %s = %s is not a valid int' %
(q_arg, value))
e = ValueError('Expecting integer for argument %s, found %r' %
(q_arg, str(value)))
e.codes = ('INVALID_QUERY_ARGUMENT', 400)
raise e
def to_str(q_arg, value):
return value
def to_bool(q_arg, value):
value = value.strip().lower()
if value in set(['1', 'true']):
return True
elif value in set(['0', 'false']):
return False
else:
log.exception('Query Argument %s = %s is not a valid boolean' %
(q_arg, value))
e = ValueError('Expecting boolean for argument %s, found %r' %
(q_arg, str(value)))
e.codes = ('INVALID_QUERY_ARGUMENT', 400)
raise e
query_args = OrderedDict([('pretty-print', to_bool),
('start-index', to_int), ('max-results', to_int),
('query', to_str), ('order', to_str)])
is_post = request.method == 'POST'
for arg, cast in query_args.items():
if arg in request.args:
g.query_args[arg.replace('-', '_')] = cast(arg,
request.args.get(arg))
# POST Query Argument overrides GET Query Argument with the same name
if is_post and arg in request.form:
g.query_args[arg.replace('-', '_')] = cast(arg, request.form[arg])
def obj_to_dict(resource,
fields=None,
ignore_unloaded=False,
data=None):
data = data if data else obj
obj_dict = OrderedDict()
if ignore_unloaded:
unloaded = instance_state(data).unloaded
log.debug('ignore_unloaded is True, ignoring %s' % unloaded)
for attribute in resource.fields:
if not ignore_unloaded or (ignore_unloaded
and attribute not in unloaded):
obj_dict[attribute] = getattr(data, attribute)
if fields:
for attribute in fields:
try:
if not ignore_unloaded or (ignore_unloaded and
attribute not in unloaded):
obj_dict[attribute] = getattr(data, attribute)
except AttributeError:
pass
return obj_dict
def get_query_args():
g.query_args = {}
def to_int(q_arg, value):
try:
return int(value)
except ValueError as e:
log.exception('Query Argument %s = %s is not a valid int' % (q_arg, value))
e = ValueError('Expecting integer for argument %s, found %r' % (q_arg, str(value)))
e.codes = ('INVALID_QUERY_ARGUMENT', 400)
raise e
def to_str(q_arg, value):
return value
def to_bool(q_arg, value):
value = value.strip().lower()
if value in set(['1', 'true']):
return True
elif value in set(['0', 'false']):
return False
else:
log.exception('Query Argument %s = %s is not a valid boolean' % (q_arg, value))
e = ValueError('Expecting boolean for argument %s, found %r' % (q_arg, str(value)))
e.codes = ('INVALID_QUERY_ARGUMENT', 400)
raise e
query_args = OrderedDict([
('pretty-print', to_bool),
('start-index', to_int),
('max-results', to_int),
('query', to_str),
('order', to_str)
])
is_post = request.method == 'POST'
for arg, cast in query_args.iteritems():
if arg in request.args:
g.query_args[arg.replace('-', '_')] = cast(arg, request.args.get(arg))
# POST Query Argument overrides GET Query Argument with the same name
if is_post and arg in request.form:
g.query_args[arg.replace('-', '_')] = cast(arg, request.form[arg])
def default(self, obj):
def obj_to_dict(resource,
fields=None,
ignore_unloaded=False,
data=None):
data = data if data else obj
obj_dict = OrderedDict()
if ignore_unloaded:
unloaded = instance_state(data).unloaded
log.debug('ignore_unloaded is True, ignoring %s' % unloaded)
for attribute in resource.fields:
if not ignore_unloaded or (ignore_unloaded
and attribute not in unloaded):
obj_dict[attribute] = getattr(data, attribute)
if fields:
for attribute in fields:
try:
if not ignore_unloaded or (ignore_unloaded and
attribute not in unloaded):
obj_dict[attribute] = getattr(data, attribute)
except AttributeError:
pass
return obj_dict
if isinstance(obj, PagedResponse):
json_record = OrderedDict([('records', obj.records)])
if obj.total_records or obj.total_filtered:
meta = OrderedDict()
if obj.total_records is not None:
meta['records_total'] = obj.total_records
if obj.total_filtered is not None:
meta['records_filtered'] = obj.total_filtered
json_record['_meta'] = meta
return json_record
elif isinstance(obj, ErrorResponse):
json_record = OrderedDict([('code', obj.code),
('message', obj.message)])
if obj.errors:
json_record['errors'] = [{
'field': f,
'errors': e
} for f, e in obj.errors]
return json_record
elif isinstance(obj, DashboardWorkflow):
json_record = obj_to_dict(RootWorkflowResource())
json_record['workflow_state'] = obj.workflow_state
json_record['_links'] = OrderedDict([('workflow',
url_for('.get_workflows',
m_wf_id=obj.wf_id,
_method='GET'))])
return json_record
elif isinstance(obj, DashboardWorkflowstate):
json_record = obj_to_dict(RootWorkflowstateResource())
return json_record
elif isinstance(obj, Workflow):
json_record = obj_to_dict(WorkflowResource())
json_record['_links'] = OrderedDict([
('workflow_meta',
url_for('.get_workflow_meta', wf_id=obj.wf_id,
_method='GET')),
('workflow_state',
url_for('.get_workflow_state', wf_id=obj.wf_id,
_method='GET')),
('job',
url_for('.get_workflow_jobs', wf_id=obj.wf_id,
_method='GET')),
('task',
url_for('.get_workflow_tasks', wf_id=obj.wf_id,
_method='GET')),
('host',
url_for('.get_workflow_hosts', wf_id=obj.wf_id,
_method='GET')),
('invocation',
url_for('.get_workflow_invocations',
wf_id=obj.wf_id,
_method='GET'))
])
return json_record
elif isinstance(obj, WorkflowMeta):
json_record = obj_to_dict(WorkflowMetaResource())
json_record['_links'] = OrderedDict([('workflow',
url_for('.get_workflow',
wf_id=obj.wf_id))])
return json_record
elif isinstance(obj, WorkflowFiles):
json_record = obj_to_dict(WorkflowFilesResource())
return json_record
elif isinstance(obj, Workflowstate):
json_record = obj_to_dict(WorkflowstateResource())
json_record['_links'] = OrderedDict([('workflow',
url_for('.get_workflow',
wf_id=obj.wf_id))])
return json_record
elif isinstance(obj, Job):
json_record = obj_to_dict(JobResource())
if hasattr(obj, 'job_instance'):
json_record['job_instance'] = obj.job_instance
json_record['_links'] = OrderedDict([
('workflow', url_for('.get_workflow', wf_id=obj.wf_id)),
('task',
url_for('.get_workflow_tasks', wf_id=obj.wf_id,
_method='GET')),
('job_instance',
url_for('.get_job_instances',
wf_id=obj.wf_id,
job_id=obj.job_id,
_method='GET'))
])
return json_record
elif isinstance(obj, Host):
json_record = obj_to_dict(HostResource())
json_record['_links'] = OrderedDict([('workflow',
url_for('.get_workflows',
m_wf_id=obj.wf_id,
_method='GET'))])
return json_record
elif isinstance(obj, Jobstate):
json_record = obj_to_dict(JobstateResource())
json_record['_links'] = OrderedDict([
('job_instance',
url_for('.get_job_instance',
job_instance_id=obj.job_instance_id))
])
return json_record
elif isinstance(obj, Task):
json_record = obj_to_dict(TaskResource())
json_record['_links'] = OrderedDict([
('workflow', url_for('.get_workflow', wf_id=obj.wf_id)),
('job', url_for('.get_job', wf_id=obj.wf_id,
job_id=obj.job_id)),
('task_meta',
url_for('.get_task_meta',
wf_id=obj.wf_id,
job_id=obj.job_id,
task_id=obj.task_id))
])
return json_record
elif isinstance(obj, TaskMeta):
json_record = obj_to_dict(TaskMetaResource())
json_record['_links'] = OrderedDict([
('task', url_for('.get_task', task_id=obj.task_id))
])
return json_record
elif isinstance(obj, JobInstance):
json_record = obj_to_dict(JobInstanceResource(),
ignore_unloaded=True)
json_record['_links'] = OrderedDict([
('job', url_for('.get_job', job_id=obj.job_id)),
('state',
url_for('.get_job_instance_states',
job_id=obj.job_id,
job_instance_id=obj.job_instance_id,
_method='GET'))
])
json_record['_links']['host'] = url_for(
'.get_host', host_id=obj.host_id) if obj.host_id else None
json_record['_links']['invocation'] = url_for(
'.get_job_instance_invocations',
job_id=obj.job_id,
job_instance_id=obj.job_instance_id,
_method='GET')
return json_record
elif isinstance(obj, Invocation):
json_record = obj_to_dict(InvocationResource())
json_record['_links'] = OrderedDict([
('workflow', url_for('.get_workflow', wf_id=obj.wf_id)),
('job_instance',
url_for('.get_job_instance',
job_instance_id=obj.job_instance_id))
])
return json_record
elif isinstance(obj, RCLFN):
json_record = obj_to_dict(RCLFNResource(), fields=['pfns', 'meta'])
if hasattr(obj, 'extras'):
json_record_2 = obj_to_dict(WorkflowFilesResource(),
data=obj.extras)
json_record_2.update(json_record)
json_record = json_record_2
json_record['_links'] = OrderedDict([
('workflow',
url_for('.get_workflow', wf_id=obj.extras.wf_id)),
('task',
url_for('.get_task',
wf_id=obj.extras.wf_id,
task_id=obj.extras.task_id))
])
return json_record
elif isinstance(obj, RCPFN):
json_record = obj_to_dict(RCPFNResource())
return json_record
elif isinstance(obj, RCMeta):
json_record = obj_to_dict(RCMetaResource())
return json_record
elif isinstance(obj, OrderedSet):
json_record = [item for item in obj]
return json_record
elif isinstance(obj, Decimal):
return float(obj)
return JSONEncoder.default(self, obj)
def csv_to_json(csv, schema, index):
"""
Workflow has a 1-to-many relationship with Job
Job has a 1-to-many relationship with Task
schema:
Key: Entity to be included in output
Value: 'root' OR tuple of string:json_key, entity:parent-1, entity:parent-2, .., entity:parent-n, type
1. Must be ordered: depth-first order of the required nesting
2. First element must root element, i.e. other entities can only be nested within this entity
3. There should only be one element identified as root
type:
list, if items should be placed in a collection
set|OrderedSet, if items should be placed in a collection of unique items
None, if item should be included inline
index:
Key: Entity
Value: index of the Entity in the csv data.
Example: For sample input below
{
Workflow: 0,
Job: 1,
Task: 2,
WorkflowState: 3
}
Sample Input (csv):
Workflow | Job | Task | WORKFLOW_STATE
W1 | J1 | T1 | W1WS1
W1 | J1 | T2 | W1WS1
W1 | J1 | T3 | W1WS1
W1 | J2 | T1 | W1WS1
W2 | J1 | T1 | W2WS1
W2 | J1 | T2 | W2WS1
W2 | J2 | NULL | W2WS1
schema:
Example:
OrderedDict([
(WORKFLOW, 'root'),
(JOB, ('jobs', WORKFLOW, set)),
(TASK, ('tasks', WORKFLOW, JOBS, set)),
(WORKFLOW_STATE, ('workflow_state', WORKFLOW, None)),
])
Sample Output:
[
{
W1,
"jobs": [
{
J1,
"tasks": [
T1,
T2,
T3
]
},
{
J2,
"tasks": [
T1
]
}
],
"workflow_state": W1WS1
},
{
W2,
"jobs": [
{
J1,
"tasks": [
T1,
T2
]
},
{
J2,
"tasks": null
}
],
"workflow_state": W2WS1
}
]
schema:
OrderedDict([
(WORKFLOW, 'root'),
(JOB, ('jobs', WORKFLOW, set)),
(TASK, ('tasks', WORKFLOW, set))
])
Sample Output:
[
{
W1,
"jobs": [
J1,
J2,
],
"tasks": [
J1T1,
J1T2,
J1T3,
J2T1,
]
},
{
W2,
"jobs": [
J1,
J2,
],
"tasks": [
J1T1,
J1T2,
]
},
]
"""
# Sanity checks
if csv is None:
return None
if not schema:
raise ValueError('schema is required')
elif not isinstance(schema, dict):
raise ValueError('schema must be of type dictionary')
if not index:
raise ValueError('index is required')
elif not isinstance(index, dict):
raise ValueError('index must be of type dictionary')
# Start
root = [entity for entity in schema][0]
uniq_dict = {}
# Pass 1
for row in csv:
for entity, entity_def in schema.items():
if entity_def == 'root':
entity_dict = uniq_dict.setdefault(root, OrderedSet())
entity_dict.add(row[index[entity]])
elif isinstance(entity_def, tuple):
entity_dict = uniq_dict.setdefault(entity, {})
for parent in entity_def[1:-1]:
if parent == entity_def[-2]:
if entity_def[-1] is None:
obj = row[index[entity]]
else:
obj = OrderedSet()
else:
obj = OrderedDict()
entity_dict = entity_dict.setdefault(
row[index[parent]], obj)
if row[index[entity]] and hasattr(entity_dict, 'add'):
entity_dict.add(row[index[entity]])
# Pass 2
for row in csv:
for entity, entity_def in schema.items():
if entity_def == 'root':
continue
elif isinstance(entity_def, tuple):
# Immediate parent
parent_entity = entity_def[-2]
p = uniq_dict[parent_entity]
data = uniq_dict[entity]
for parent in entity_def[1:-1]:
if parent != parent_entity:
p = p[row[index[parent]]]
data = data[row[index[parent]]]
if not data:
data = None
row[index[parent_entity]].__setattr__(entity_def[0], data)
return uniq_dict[root]