def save(self, *args, **kwargs):
if not self.name:
self.name = self.job_name.split('.')[-1]
# [TODO] not too efficient... consider do the invalidation in db?
try:
old = WorkflowJob.objects.get(pk=self.pk)
except WorkflowJob.DoesNotExist:
old = WorkflowJob() # empty
cond1 = self.workflow_id != old.workflow_id
cond2 = self.job_id != old.job_id
cond3 = not deep_eq(self.job_settings, old.job_settings)
super(WorkflowJob, self).save(*args, **kwargs)
if cond1 or cond2 or cond3:
wf_id = self.workflow_id
Workflow.objects.filter(pk__in=list(set([wf_id, old.workflow_id]))).update(valid=False)
def save(self, *args, **kwargs):
if not self.name:
self.name = self.job_name.split('.')[-1]
# [TODO] not too efficient... consider do the invalidation in db?
try:
old = WorkflowJob.objects.get(pk=self.pk)
except WorkflowJob.DoesNotExist:
old = WorkflowJob() # empty
cond1 = self.workflow_id != old.workflow_id
cond2 = self.job_id != old.job_id
cond3 = not deep_eq(self.job_settings, old.job_settings)
super(WorkflowJob, self).save(*args, **kwargs)
if cond1 or cond2 or cond3:
wf_id = self.workflow_id
Workflow.objects.filter(
pk__in=list(set([wf_id, old.workflow_id]))).update(valid=False)
def __init__(cls, clsname, bases, attrs):
super(RodanTaskType, cls).__init__(clsname, bases, attrs)
# check the number of arguments of implemented function
if 'run_my_task' in attrs:
argspec = inspect.getargspec(attrs['run_my_task'])
assert len(argspec.args) == 4, 'run_my_task'
if 'get_my_interface' in attrs:
argspec = inspect.getargspec(attrs['get_my_interface'])
assert len(argspec.args) == 3, 'get_my_interface'
if 'validate_my_user_input' in attrs:
argspec = inspect.getargspec(attrs['validate_my_user_input'])
assert len(argspec.args) == 4, 'validate_my_user_input'
if 'test_my_task' in attrs:
argspec = inspect.getargspec(attrs['test_my_task'])
assert len(argspec.args) == 2, 'test_my_task'
if attrs.get('_abstract') == True: # not the abstract class
return
else:
if not Job.objects.filter(name=attrs['name']).exists():
if not getattr(settings, '_update_rodan_jobs', None) and not settings.TEST:
raise ImproperlyConfigured('The catalogue of local jobs does not match the ones in database: local job `{0}` has not been registered. Please run `manage.py migrate` on Rodan server to update the database.')
try:
# verify the schema
jsonschema.Draft4Validator.check_schema(attrs['settings'])
except jsonschema.exceptions.SchemaError as e:
raise e
schema = attrs['settings'] or {'type': 'object'}
j = Job(name=attrs['name'],
author=attrs['author'],
description=attrs['description'],
settings=schema,
enabled=attrs['enabled'],
category=attrs['category'],
interactive=attrs['interactive'])
j.save()
try:
for ipt in attrs['input_port_types']:
i = InputPortType(job=j,
name=ipt['name'],
minimum=ipt['minimum'],
maximum=ipt['maximum'],
is_list=ipt.get('is_list', False))
i.save()
resource_types = RodanTaskType._resolve_resource_types(ipt['resource_types'])
if len(resource_types) == 0:
raise ValueError('No available resource types found for this InputPortType: {0}'.format(ipt['resource_types']))
i.resource_types.add(*resource_types)
for opt in attrs['output_port_types']:
o = OutputPortType(job=j,
name=opt['name'],
minimum=opt['minimum'],
maximum=opt['maximum'],
is_list=opt.get('is_list', False))
o.save()
resource_types = RodanTaskType._resolve_resource_types(opt['resource_types'])
if len(resource_types) == 0:
raise ValueError('No available resource types found for this OutputPortType: {0}'.format(opt['resource_types']))
o.resource_types.add(*resource_types)
except Exception as e:
j.delete() # clean the job
raise e
if not settings.TEST:
print "Added: {0}".format(j.name)
else:
UPDATE_JOBS = getattr(rodan_settings, "_update_rodan_jobs", False)
# perform an integrity check, and update jobs if demanded.
j = Job.objects.get(name=attrs['name'])
def check_field(field_name, original_value, new_value, compare_fn=lambda x, y: x == y):
if not compare_fn(original_value, new_value):
if not UPDATE_JOBS:
raise ImproperlyConfigured("The field `{0}` of Job `{1}` seems to be updated: {2} --> {3}. Try to run `manage.py migrate` to confirm this update.".format(field_name, j.name, convert_to_unicode(original_value), convert_to_unicode(new_value)))
else:
confirm_update = raw_input("The field `{0}` of Job `{1}` seems to be updated: \n{2}\n -->\n{3}\n\nConfirm (y/N)? ".format(field_name, j.name, convert_to_unicode(original_value), convert_to_unicode(new_value)))
if confirm_update.lower() == 'y':
setattr(j, field_name, new_value)
j.save()
print " ..updated.\n\n"
else:
print " ..not updated.\n\n"
check_field("author", j.author, attrs['author'])
check_field("description", j.description, attrs['description'])
check_field("settings", j.settings, attrs['settings'] or {'type': 'object'}, compare_fn=lambda x, y: deep_eq(x, y))
check_field("enabled", j.enabled, attrs['enabled'])
check_field("category", j.category, attrs['category'])
check_field("interactive", j.interactive, attrs['interactive'])
# Input Port Types
def check_port_types(which):
"which == 'in' or 'out'"
if which == 'in':
attrs_pts = list(copy.deepcopy(attrs['input_port_types']))
db_pts = list(j.input_port_types.all())
msg = 'Input'
elif which == 'out':
attrs_pts = list(copy.deepcopy(attrs['output_port_types']))
db_pts = list(j.output_port_types.all())
msg = 'Output'
for pt in db_pts:
pt_name = pt.name
idx = next((i for (i, this_pt) in enumerate(attrs_pts) if this_pt['name'] == pt_name), None)
if idx is not None: # pt exists in database and in code. Check values
attrs_pt = attrs_pts[idx]
# Compare values
if attrs_pt['minimum'] != pt.minimum:
if not UPDATE_JOBS:
raise ImproperlyConfigured("The field `{0}` of {5} Port Type `{1}` of Job `{2}` seems to be updated: {3} --> {4}. Try to run `manage.py migrate` to confirm this update.".format('minimum', pt_name, j.name, pt.minimum, attrs_pt['minimum'], msg))
else:
confirm_update = raw_input("The field `{0}` of {5} Port Type `{1}` of Job `{2}` seems to be updated: \n{3}\n -->\n{4}\n\nConfirm (y/N)? ".format('minimum', pt_name, j.name, pt.minimum, attrs_pt['minimum'], msg))
if confirm_update.lower() == 'y':
pt.minimum = attrs_pt['minimum']
pt.save()
print " ..updated.\n\n"
else:
print " ..not updated.\n\n"
if attrs_pt['maximum'] != pt.maximum:
if not UPDATE_JOBS:
raise ImproperlyConfigured("The field `{0}` of {5} Port Type `{1}` of Job `{2}` seems to be updated: {3} --> {4}. Try to run `manage.py migrate` to confirm this update.".format('maximum', pt_name, j.name, pt.maximum, attrs_pt['maximum'], msg))
else:
confirm_update = raw_input("The field `{0}` of {5} Port Type `{1}` of Job `{2}` seems to be updated: \n{3}\n -->\n{4}\n\nConfirm (y/N)? ".format('maximum', pt_name, j.name, pt.maximum, attrs_pt['maximum'], msg))
if confirm_update.lower() == 'y':
pt.maximum = attrs_pt['maximum']
pt.save()
print " ..updated.\n\n"
else:
print " ..not updated.\n\n"
attrs_is_list = bool(attrs_pt.get('is_list', False))
if attrs_is_list != pt.is_list:
if not UPDATE_JOBS:
raise ImproperlyConfigured("The field `{0}` of {5} Port Type `{1}` of Job `{2}` seems to be updated: {3} --> {4}. Try to run `manage.py migrate` to confirm this update.".format('is_list', pt_name, j.name, pt.is_list, attrs_is_list, msg))
else:
confirm_update = raw_input("The field `{0}` of {5} Port Type `{1}` of Job `{2}` seems to be updated: \n{3}\n -->\n{4}\n\nConfirm (y/N)? ".format('is_list', pt_name, j.name, pt.is_list, attrs_is_list, msg))
if confirm_update.lower() == 'y':
pt.is_list = attrs_is_list
pt.save()
print " ..updated.\n\n"
else:
print " ..not updated.\n\n"
resource_types = RodanTaskType._resolve_resource_types(attrs_pt['resource_types'])
rt_code = set(map(lambda rt: rt.mimetype, resource_types))
rt_db = set(map(lambda rt: rt.mimetype, pt.resource_types.all()))
if rt_code != rt_db:
if not UPDATE_JOBS:
raise ImproperlyConfigured("The field `{0}` of {5} Port Type `{1}` of Job `{2}` seems to be updated: {3} --> {4}. Try to run `manage.py migrate` to confirm this update.".format('resource_types', pt_name, j.name, rt_db, rt_code, msg))
else:
confirm_update = raw_input("The field `{0}` of {5} Port Type `{1}` of Job `{2}` seems to be updated: \n{3}\n -->\n{4}\n\nConfirm (y/N)? ".format('resource_types', pt_name, j.name, rt_db, rt_code, msg))
if confirm_update.lower() == 'y':
pt.resource_types.clear()
pt.resource_types.add(*resource_types)
print " ..updated.\n\n"
else:
print " ..not updated.\n\n"
del attrs_pts[idx]
else: # pt exists in database but not in code. Should be deleted.
if not UPDATE_JOBS:
raise ImproperlyConfigured("The {2} Port Type `{0}` of Job `{1}` seems to be deleted. Try to run `manage.py migrate` to confirm this deletion.".format(pt_name, j.name, msg))
else:
confirm_delete = raw_input("The {2} Port Type `{0}` of Job `{1}` seems to be deleted. Confirm (y/N)? ".format(pt_name, j.name, msg))
if confirm_delete.lower() == 'y':
try:
pt.delete()
print " ..deleted.\n\n"
except Exception as e:
print " ..not deleted because of an exception: {0}. Please fix it manually.\n\n".format(str(e))
else:
print " ..not deleted.\n\n"
if attrs_pts: # ipt exists in code but not in database. Should be added to the database.
for pt in attrs_pts:
if not UPDATE_JOBS:
raise ImproperlyConfigured("The {2} Port Type `{0}` of Job `{1}` seems to be newly added. Try to run `manage.py migrate` to confirm this update.".format(pt['name'], j.name, msg))
else:
confirm_update = raw_input("The {2} Port Type `{0}` of Job `{1}` seems to be newly added. Confirm (y/N)? ".format(pt['name'], j.name, msg))
if confirm_update.lower() == 'y':
if which == 'in':
Model = InputPortType
elif which == 'out':
Model = OutputPortType
i = Model(job=j,
name=pt['name'],
minimum=pt['minimum'],
maximum=pt['maximum'],
is_list=bool(pt.get('is_list', False)))
i.save()
resource_types = RodanTaskType._resolve_resource_types(pt['resource_types'])
if len(resource_types) == 0:
raise ValueError('No available resource types found for this {1}PortType: {0}'.format(pt['resource_types'], msg))
i.resource_types.add(*resource_types)
print " ..updated.\n\n"
else:
print " ..not updated.\n\n"
check_port_types('in')
check_port_types('out')
# Process done
from rodan.jobs.load import job_list
if attrs['name'] in job_list:
job_list.remove(attrs['name'])
def __init__(cls, clsname, bases, attrs):
super(RodanTaskType, cls).__init__(clsname, bases, attrs)
# check the number of arguments of implemented function
if 'run_my_task' in attrs:
argspec = inspect.getargspec(attrs['run_my_task'])
assert len(argspec.args) == 4, 'run_my_task'
if 'get_my_interface' in attrs:
argspec = inspect.getargspec(attrs['get_my_interface'])
assert len(argspec.args) == 3, 'get_my_interface'
if 'validate_my_user_input' in attrs:
argspec = inspect.getargspec(attrs['validate_my_user_input'])
assert len(argspec.args) == 4, 'validate_my_user_input'
if 'test_my_task' in attrs:
argspec = inspect.getargspec(attrs['test_my_task'])
assert len(argspec.args) == 2, 'test_my_task'
if attrs.get('_abstract') == True: # not the abstract class
return
else:
# Set base settings schema if they do not already exist in the job.
schema = attrs.get('settings', {
'job_queue': 'celery',
'type': 'object'
})
if not Job.objects.filter(name=attrs['name']).exists():
if not getattr(settings, '_update_rodan_jobs',
None) and not settings.TEST:
raise ImproperlyConfigured(
'The catalogue of local jobs does not match the ones in database: local job `{0}` has not been registered. Please run `manage.py migrate` on Rodan server to update the database.'
)
try:
# verify the schema
jsonschema.Draft4Validator.check_schema(attrs['settings'])
except jsonschema.exceptions.SchemaError as e:
raise e
j = Job(
name=attrs['name'],
author=attrs['author'],
description=attrs['description'],
settings=schema,
enabled=attrs['enabled'],
category=attrs['category'],
interactive=attrs['interactive'],
# Check for the presence of job_queue in the rodan job's settings, if not use the default 'celery'
job_queue=schema.get('job_queue', 'celery'))
j.save()
try:
for ipt in attrs['input_port_types']:
i = InputPortType(job=j,
name=ipt['name'],
minimum=ipt['minimum'],
maximum=ipt['maximum'],
is_list=ipt.get('is_list', False))
i.save()
resource_types = RodanTaskType._resolve_resource_types(
ipt['resource_types'])
if len(resource_types) == 0:
raise ValueError(
'No available resource types found for this InputPortType: {0}'
.format(ipt['resource_types']))
i.resource_types.add(*resource_types)
for opt in attrs['output_port_types']:
o = OutputPortType(job=j,
name=opt['name'],
minimum=opt['minimum'],
maximum=opt['maximum'],
is_list=opt.get('is_list', False))
o.save()
resource_types = RodanTaskType._resolve_resource_types(
opt['resource_types'])
if len(resource_types) == 0:
raise ValueError(
'No available resource types found for this OutputPortType: {0}'
.format(opt['resource_types']))
o.resource_types.add(*resource_types)
except Exception as e:
j.delete() # clean the job
raise e
if not settings.TEST:
print("Added: {0}".format(j.name))
else:
UPDATE_JOBS = getattr(rodan_settings, "_update_rodan_jobs",
False)
# perform an integrity check, and update jobs if demanded.
j = Job.objects.get(name=attrs['name'])
def check_field(field_name,
original_value,
new_value,
compare_fn=lambda x, y: x == y):
if not compare_fn(original_value, new_value):
if not UPDATE_JOBS:
raise ImproperlyConfigured(
"The field `{0}` of Job `{1}` seems to be updated: {2} --> {3}. Try to run `manage.py migrate` to confirm this update."
.format(field_name, j.name,
convert_to_unicode(original_value),
convert_to_unicode(new_value)))
else:
confirm_update = confirm(
"The field `{0}` of Job `{1}` seems to be updated: \n{2}\n -->\n{3}\n\nConfirm (y/N)? "
.format(field_name, j.name,
convert_to_unicode(original_value),
convert_to_unicode(new_value)))
if confirm_update:
setattr(j, field_name, new_value)
j.save()
print(" ..updated.\n\n")
else:
print(" ..not updated.\n\n")
check_field("author", j.author, attrs['author'])
check_field("description", j.description, attrs['description'])
check_field("settings",
j.settings,
schema,
compare_fn=lambda x, y: deep_eq(x, y))
check_field("enabled", j.enabled, attrs['enabled'])
check_field("category", j.category, attrs['category'])
check_field("interactive", j.interactive, attrs['interactive'])
check_field("job_queue", j.job_queue,
schema.get('job_queue', 'celery'))
# Input Port Types
def check_port_types(which):
"which == 'in' or 'out'"
if which == 'in':
attrs_pts = list(
copy.deepcopy(attrs['input_port_types']))
db_pts = list(j.input_port_types.all())
msg = 'Input'
elif which == 'out':
attrs_pts = list(
copy.deepcopy(attrs['output_port_types']))
db_pts = list(j.output_port_types.all())
msg = 'Output'
for pt in db_pts:
pt_name = pt.name
idx = next((i for (i, this_pt) in enumerate(attrs_pts)
if this_pt['name'] == pt_name), None)
if idx is not None: # pt exists in database and in code. Check values
attrs_pt = attrs_pts[idx]
# Compare values
if attrs_pt['minimum'] != pt.minimum:
if not UPDATE_JOBS:
raise ImproperlyConfigured(
"The field `{0}` of {5} Port Type `{1}` of Job `{2}` seems to be updated: {3} --> {4}. Try to run `manage.py migrate` to confirm this update."
.format('minimum', pt_name, j.name,
pt.minimum,
attrs_pt['minimum'], msg))
else:
confirm_update = confirm(
"The field `{0}` of {5} Port Type `{1}` of Job `{2}` seems to be updated: \n{3}\n -->\n{4}\n\nConfirm (y/N)? "
.format('minimum', pt_name, j.name,
pt.minimum,
attrs_pt['minimum'], msg))
if confirm_update:
pt.minimum = attrs_pt['minimum']
pt.save()
print(" ..updated.\n\n")
else:
print(" ..not updated.\n\n")
if attrs_pt['maximum'] != pt.maximum:
if not UPDATE_JOBS:
raise ImproperlyConfigured(
"The field `{0}` of {5} Port Type `{1}` of Job `{2}` seems to be updated: {3} --> {4}. Try to run `manage.py migrate` to confirm this update."
.format('maximum', pt_name, j.name,
pt.maximum,
attrs_pt['maximum'], msg))
else:
confirm_update = confirm(
"The field `{0}` of {5} Port Type `{1}` of Job `{2}` seems to be updated: \n{3}\n -->\n{4}\n\nConfirm (y/N)? "
.format('maximum', pt_name, j.name,
pt.maximum,
attrs_pt['maximum'], msg))
if confirm_update:
pt.maximum = attrs_pt['maximum']
pt.save()
print(" ..updated.\n\n")
else:
print(" ..not updated.\n\n")
attrs_is_list = bool(attrs_pt.get(
'is_list', False))
if attrs_is_list != pt.is_list:
if not UPDATE_JOBS:
raise ImproperlyConfigured(
"The field `{0}` of {5} Port Type `{1}` of Job `{2}` seems to be updated: {3} --> {4}. Try to run `manage.py migrate` to confirm this update."
.format('is_list', pt_name, j.name,
pt.is_list, attrs_is_list,
msg))
else:
confirm_update = confirm(
"The field `{0}` of {5} Port Type `{1}` of Job `{2}` seems to be updated: \n{3}\n -->\n{4}\n\nConfirm (y/N)? "
.format('is_list', pt_name, j.name,
pt.is_list, attrs_is_list,
msg))
if confirm_update:
pt.is_list = attrs_is_list
pt.save()
print(" ..updated.\n\n")
else:
print(" ..not updated.\n\n")
resource_types = RodanTaskType._resolve_resource_types(
attrs_pt['resource_types'])
rt_code = set(
map(lambda rt: rt.mimetype, resource_types))
rt_db = set(
map(lambda rt: rt.mimetype,
pt.resource_types.all()))
if rt_code != rt_db:
if not UPDATE_JOBS:
raise ImproperlyConfigured(
"The field `{0}` of {5} Port Type `{1}` of Job `{2}` seems to be updated: {3} --> {4}. Try to run `manage.py migrate` to confirm this update."
.format('resource_types', pt_name,
j.name, rt_db, rt_code, msg))
else:
confirm_update = confirm(
"The field `{0}` of {5} Port Type `{1}` of Job `{2}` seems to be updated: \n{3}\n -->\n{4}\n\nConfirm (y/N)? "
.format('resource_types', pt_name,
j.name, rt_db, rt_code, msg))
if confirm_update:
pt.resource_types.clear()
pt.resource_types.add(*resource_types)
print(" ..updated.\n\n")
else:
print(" ..not updated.\n\n")
del attrs_pts[idx]
else: # pt exists in database but not in code. Should be deleted.
if not UPDATE_JOBS:
raise ImproperlyConfigured(
"The {2} Port Type `{0}` of Job `{1}` seems to be deleted. Try to run `manage.py migrate` to confirm this deletion."
.format(pt_name, j.name, msg))
else:
confirm_delete = confirm(
"The {2} Port Type `{0}` of Job `{1}` seems to be deleted. Confirm (y/N)? "
.format(pt_name, j.name, msg))
if confirm_delete:
try:
pt.delete()
print(" ..deleted.\n\n")
except Exception as e:
print(
" ..not deleted because of an exception: {0}. Please fix it manually.\n\n"
.format(str(e)))
else:
print(" ..not deleted.\n\n")
if attrs_pts: # ipt exists in code but not in database. Should be added to the database.
for pt in attrs_pts:
if not UPDATE_JOBS:
raise ImproperlyConfigured(
"The {2} Port Type `{0}` of Job `{1}` seems to be newly added. Try to run `manage.py migrate` to confirm this update."
.format(pt['name'], j.name, msg))
else:
confirm_update = confirm(
"The {2} Port Type `{0}` of Job `{1}` seems to be newly added. Confirm (y/N)? "
.format(pt['name'], j.name, msg))
if confirm_update:
if which == 'in':
Model = InputPortType
elif which == 'out':
Model = OutputPortType
i = Model(job=j,
name=pt['name'],
minimum=pt['minimum'],
maximum=pt['maximum'],
is_list=bool(
pt.get('is_list', False)))
i.save()
resource_types = RodanTaskType._resolve_resource_types(
pt['resource_types'])
if len(resource_types) == 0:
raise ValueError(
'No available resource types found for this {1}PortType: {0}'
.format(pt['resource_types'], msg))
i.resource_types.add(*resource_types)
print(" ..updated.\n\n")
else:
print(" ..not updated.\n\n")
check_port_types('in')
check_port_types('out')
# Process done
from rodan.jobs.load import job_list
if attrs['name'] in job_list:
job_list.remove(attrs['name'])