def test_dictionary_field_from_db_value(self, mock_tp):
ret = DictionaryField()
ret.from_db_value(value='foo',
expression='exp',
connection='con',
context='ctx')
mock_tp.assert_called_with('foo')
def test_dictionary_field_clean(self, mock_sc, mock_gpv):
ret = DictionaryField()
input_value = 'foo'
input_model_instance = mock.MagicMock()
output = mock_sc.return_value
ret.clean(value=input_value, model_instance=input_model_instance)
mock_sc.assert_called_with(input_value, input_model_instance)
mock_gpv.assert_called_with(output)
def test_dictionary_field_value_to_string(self, mock_gpvo, mock_gpv):
ret = DictionaryField()
output = mock_gpvo.return_value
mock_obj = mock.MagicMock()
ret.value_to_string(obj=mock_obj)
mock_gpvo.assert_called_with(mock_obj)
mock_gpv.assert_called_with(output)
def test_dictionary_field_clean(self, mock_sc, mock_gpv):
ret = DictionaryField()
input_value = 'foo'
input_model_instance = mock.MagicMock()
output = mock_sc.return_value
ret.clean(value=input_value, model_instance=input_model_instance)
mock_sc.assert_called_with(input_value, input_model_instance)
mock_gpv.assert_called_with(output)
def test_dictionary_field_value_to_string(self, mock_gpvo, mock_gpv):
ret = DictionaryField()
output = mock_gpvo.return_value
mock_obj = mock.MagicMock()
ret.value_to_string(obj=mock_obj)
mock_gpvo.assert_called_with(mock_obj)
mock_gpv.assert_called_with(output)
def test_dictionary_field_to_python_none(self):
ret = DictionaryField()
self.assertIsNone(ret.to_python(value=None))
class TethysJob(models.Model):
"""
Base class for all job types. This is intended to be an abstract class that is not directly instantiated.
"""
class Meta:
verbose_name = 'Job'
objects = InheritanceManager()
STATUSES = (
('PEN', 'Pending'),
('SUB', 'Submitted'),
('RUN', 'Running'),
('COM', 'Complete'),
('ERR', 'Error'),
('ABT', 'Aborted'),
('VAR', 'Various'),
('VCP', 'Various-Complete'),
('RES', 'Results-Ready'),
)
STATUS_DICT = {k: v for v, k in STATUSES}
VALID_STATUSES = [v for v, _ in STATUSES]
name = models.CharField(max_length=1024)
description = models.CharField(max_length=2048, blank=True, default='')
user = models.ForeignKey(User, on_delete=models.CASCADE)
label = models.CharField(max_length=1024)
creation_time = models.DateTimeField(auto_now_add=True)
execute_time = models.DateTimeField(blank=True, null=True)
start_time = models.DateTimeField(blank=True, null=True)
completion_time = models.DateTimeField(blank=True, null=True)
workspace = models.CharField(max_length=1024, default='')
extended_properties = DictionaryField(default='', blank=True)
_process_results_function = models.CharField(max_length=1024,
blank=True,
null=True)
_status = models.CharField(max_length=3,
choices=STATUSES,
default=STATUSES[0][0])
@property
def type(self):
"""
Returns the name of Tethys Job type.
"""
return self.__class__.__name__
@property
def update_status_interval(self):
if not hasattr(self, '_update_status_interval'):
self._update_status_interval = datetime.timedelta(seconds=10)
return self._update_status_interval
@property
def last_status_update(self):
if not getattr(self, '_last_status_update', None):
self._last_status_update = self.execute_time or timezone.now(
) - self.update_status_interval
return self._last_status_update
@property
def status(self):
self.update_status()
field = self._meta.get_field('_status')
status = self._get_FIELD_display(field)
return status
@status.setter
def status(self, value):
self.update_status(status=value)
@property
def run_time(self):
start_time = self.start_time or self.execute_time
if start_time:
end_time = self.completion_time or datetime.datetime.now(
start_time.tzinfo)
run_time = end_time - start_time
else:
return ''
return run_time
def execute(self, *args, **kwargs):
"""
executes the job
"""
self._execute(*args, **kwargs)
self.execute_time = timezone.now()
self._status = 'SUB'
self.save()
def update_status(self, status=None, *args, **kwargs):
"""
Update status of job.
"""
old_status = self._status
# Set status from status given
if status:
if status not in self.VALID_STATUSES:
log.error('Invalid status given: {}'.format(status))
return
self._status = status
self.save()
# Update status if status not given and still pending/running
elif old_status in ['PEN', 'SUB', 'RUN', 'VAR'
] and self.is_time_to_update():
self._update_status(*args, **kwargs)
self._last_status_update = timezone.now()
# Post-process status after update if old status was pending/running
if old_status in ['PEN', 'SUB', 'RUN', 'VAR']:
if self._status == 'RUN' and (old_status == 'PEN'
or old_status == 'SUB'):
self.start_time = timezone.now()
if self._status in ["COM", "VCP", "RES"]:
self.process_results()
elif self._status == 'ERR' or self._status == 'ABT':
self.completion_time = timezone.now()
self.save()
def is_time_to_update(self):
"""
Check if it is time to update again.
Returns:
bool: True if update_status_interval or longer has elapsed since our last update, else False.
"""
time_since_last_update = timezone.now() - self.last_status_update
is_time_to_update = time_since_last_update > self.update_status_interval
return is_time_to_update
@property
def process_results_function(self):
"""
Returns:
A function handle or None if function cannot be resolved.
"""
if self._process_results_function:
function_extractor = TethysFunctionExtractor(
self._process_results_function, None)
if function_extractor.valid:
return function_extractor.function
@process_results_function.setter
def process_results_function(self, function):
if isinstance(function, str):
self._process_results_function = function
return
module_path = inspect.getmodule(function).__name__.split('.')
module_path.append(function.__name__)
self._process_results_function = '.'.join(module_path)
def process_results(self, *args, **kwargs):
"""
Process the results.
"""
log.debug('Started processing results for job: {}'.format(self))
self._process_results(*args, **kwargs)
self.completion_time = timezone.now()
self._status = 'COM'
self.save()
log.debug('Finished processing results for job: {}'.format(self))
@abstractmethod
def _execute(self, *args, **kwargs):
pass
@abstractmethod
def _update_status(self, *args, **kwargs):
pass
@abstractmethod
def _process_results(self, *args, **kwargs):
pass
@abstractmethod
def stop(self):
"""
Stops job from executing
"""
raise NotImplementedError()
@abstractmethod
def pause(self):
"""
Pauses job during execution
"""
raise NotImplementedError()
@abstractmethod
def resume(self):
"""
Resumes a job that has been paused
"""
raise NotImplementedError()
class CondorWorkflowNode(models.Model):
"""
Base class for CondorWorkflow Nodes
"""
TYPES = (('JOB', 'JOB'),
('DAT', 'DATA'),
('SUB', 'SUBDAG'),
('SPL', 'SPLICE'),
('FIN', 'FINAL'),
)
TYPE_DICT = {k: v for v, k in TYPES}
objects = InheritanceManager()
name = models.CharField(max_length=1024)
workflow = models.ForeignKey(CondorPyWorkflow, on_delete=models.CASCADE, related_name='node_set')
parent_nodes = models.ManyToManyField('self', related_name='children_nodes', symmetrical=False)
pre_script = models.CharField(max_length=1024, null=True, blank=True)
pre_script_args = models.CharField(max_length=1024, null=True, blank=True)
post_script = models.CharField(max_length=1024, null=True, blank=True)
post_script_args = models.CharField(max_length=1024, null=True, blank=True)
variables = DictionaryField(default='', blank=True)
priority = models.IntegerField(null=True, blank=True)
category = models.CharField(max_length=128, null=True, blank=True)
retry = models.PositiveSmallIntegerField(null=True, blank=True)
retry_unless_exit_value = models.IntegerField(null=True, blank=True)
pre_skip = models.IntegerField(null=True, blank=True)
abort_dag_on = models.IntegerField(null=True, blank=True)
abort_dag_on_return_value = models.IntegerField(null=True, blank=True)
dir = models.CharField(max_length=1024, null=True, blank=True)
noop = models.BooleanField(default=False)
done = models.BooleanField(default=False)
@abstractmethod
def type(self):
pass
@abstractmethod
def job(self):
pass
@property
def condorpy_node(self):
if not hasattr(self, '_condorpy_node'):
condorpy_node = Node(job=self.job,
pre_script=self.pre_script,
pre_script_args=self.pre_script_args,
post_script=self.post_script,
post_script_args=self.post_script_args,
variables=self.variables,
priority=self.priority,
category=self.category,
retry=self.retry,
pre_skip=self.pre_skip,
abort_dag_on=self.abort_dag_on,
abort_dag_on_return_value=self.abort_dag_on_return_value,
dir=self.dir,
noop=self.noop,
done=self.done
)
self._condorpy_node = condorpy_node
return self._condorpy_node
@property
def parents(self):
return self.parent_nodes.select_subclasses()
def add_parent(self, parent):
self.parent_nodes.add(parent)
def update_database_fields(self):
pass
def test_dictionary_field_get_prep_value_list(self, mock_jd):
ret = DictionaryField()
input_value = ['foo', 'bar']
ret.get_prep_value(value=input_value)
mock_jd.assert_called_with(input_value)
def test_dictionary_field_to_python_empty_str(self):
ret = DictionaryField()
self.assertDictEqual({}, ret.to_python(value=""))
def test_dictionary_field_to_python_dict(self):
ret = DictionaryField()
input_dict = {'name': 'foo', 'extra': 'bar'}
res = ret.to_python(value=input_dict)
self.assertDictEqual(input_dict, res)
def test_dictionary_field_to_python_str(self, mock_jl):
ret = DictionaryField()
ret.to_python(value='foo')
mock_jl.assert_called_with('foo')
def test_dictionary_field_get_prep_value_list(self, mock_jd):
ret = DictionaryField()
input_value = ['foo', 'bar']
ret.get_prep_value(value=input_value)
mock_jd.assert_called_with(input_value)
def test_dictionary_field_get_prep_value_str(self):
ret = DictionaryField()
self.assertEqual('foo', ret.get_prep_value(value='foo'))
def test_dictionary_field_from_db_value(self, mock_tp):
ret = DictionaryField()
ret.from_db_value(value='foo', expression='exp', connection='con', context='ctx')
mock_tp.assert_called_with('foo')
def test_dictionary_field_to_python_empty_dict(self):
ret = DictionaryField()
input_value = ['test1', 'test2']
res = ret.to_python(value=input_value)
self.assertDictEqual({}, res)
def test_dictionary_field_to_python_none(self):
ret = DictionaryField()
self.assertIsNone(ret.to_python(value=None))
def test_dictionary_field_to_python_dict(self):
ret = DictionaryField()
input_dict = {'name': 'foo', 'extra': 'bar'}
res = ret.to_python(value=input_dict)
self.assertDictEqual(input_dict, res)
def test_dictionary_field_to_python_empty_str(self):
ret = DictionaryField()
self.assertDictEqual({}, ret.to_python(value=""))
def test_dictionary_field_to_python_str(self, mock_jl):
ret = DictionaryField()
ret.to_python(value='foo')
mock_jl.assert_called_with('foo')
def test_dictionary_field_to_python_str_value_error(self, mock_jl):
ret = DictionaryField()
mock_jl.side_effect = ValueError
self.assertRaises(ValueError, ret.to_python, value='foo')
def test_dictionary_field_formfield(self, mock_ff):
ret = DictionaryField()
ret.formfield(additional='test2')
mock_ff.assert_called_once()
def test_dictionary_field_to_python_empty_dict(self):
ret = DictionaryField()
input_value = ['test1', 'test2']
res = ret.to_python(value=input_value)
self.assertDictEqual({}, res)
class TethysJob(models.Model):
"""
Base class for all job types. This is intended to be an abstract class that is not directly instantiated.
"""
class Meta:
verbose_name = 'Job'
STATUSES = (
('PEN', 'Pending'),
('SUB', 'Submitted'),
('RUN', 'Running'),
('COM', 'Complete'),
('ERR', 'Error'),
('ABT', 'Aborted'),
('VAR', 'Various'),
('VCP', 'Various-Complete'),
)
STATUS_DICT = {k: v for v, k in STATUSES}
name = models.CharField(max_length=1024)
description = models.CharField(max_length=2048, blank=True, default='')
user = models.ForeignKey(User)
label = models.CharField(max_length=1024)
creation_time = models.DateTimeField(auto_now_add=True)
execute_time = models.DateTimeField(blank=True, null=True)
completion_time = models.DateTimeField(blank=True, null=True)
workspace = models.CharField(max_length=1024, default='')
extended_properties = DictionaryField(default='', blank=True)
_subclass = models.CharField(max_length=30, default='basicjob')
_status = models.CharField(max_length=3,
choices=STATUSES,
default=STATUSES[0][0])
@property
def status(self):
self.update_status()
field = self.child._meta.get_field('_status')
status = self._get_FIELD_display(field)
return status
@property
def child(self):
return getattr(self, self._subclass)
def execute(self, *args, **kwargs):
"""
"""
self.execute_time = timezone.now()
self._status = 'PEN'
self.save()
self.child._execute(*args, **kwargs)
def update_status(self, *args, **kwargs):
if self._status in ['PEN', 'SUB', 'RUN', 'VAR']:
self.child._update_status(*args, **kwargs)
self._status = self.child._status
if self._status == "COM" or self._status == "VCP":
self.process_results()
elif self._status == 'ERR' or self._status == 'ABT':
self.completion_time = timezone.now()
self.save()
def process_results(self, *args, **kwargs):
"""
"""
self.completion_time = timezone.now()
self.save()
self.child._process_results(*args, **kwargs)
@abstractmethod
def _execute(self, *args, **kwargs):
pass
@abstractmethod
def _update_status(self, *args, **kwargs):
pass
@abstractmethod
def _process_results(self, *args, **kwargs):
pass
@abstractmethod
def stop(self):
"""
"""
raise NotImplementedError()
def pause(self):
"""
"""
raise NotImplementedError()
def resume(self):
"""
"""
raise NotImplementedError()
def test_dictionary_field_get_prep_value_str(self):
ret = DictionaryField()
self.assertEqual('foo', ret.get_prep_value(value='foo'))
class CondorJob(TethysJob):
"""
Condor job type
"""
executable = models.CharField(max_length=1024)
condorpy_template_name = models.CharField(max_length=1024,
blank=True,
null=True)
attributes = DictionaryField(default='')
remote_input_files = ListField(default='')
cluster_id = models.IntegerField(blank=True, default=0)
num_jobs = models.IntegerField(default=1)
remote_id = models.CharField(max_length=32, blank=True, null=True)
tethys_job = models.OneToOneField(TethysJob)
scheduler = models.ForeignKey(Scheduler, blank=True, null=True)
STATUS_MAP = {
'Unexpanded': 'PEN',
'Idle': 'SUB',
'Running': 'RUN',
'Removed': 'ABT',
'Completed': 'COM',
'Held': 'ERR',
'Submission_err': 'ERR',
'Various': 'VAR',
'Various-Complete': 'VCP',
}
def __init__(self, *args, **kwargs):
kwargs.update({'_subclass': self.__class__.__name__.lower()})
super(self.__class__, self).__init__(*args, **kwargs)
@property
def condorpy_template(self):
if self.condorpy_template_name:
template = getattr(Templates, self.condorpy_template_name)
else:
template = Templates.base
return template
@property
def condorpy_job(self):
if not hasattr(self, '_condorpy_job'):
if 'executable' in self.attributes.keys():
del self.attributes['executable']
if self.scheduler:
host = self.scheduler.host
username = self.scheduler.username
password = self.scheduler.password
private_key = self.scheduler.private_key_path
private_key_pass = self.scheduler.private_key_pass
else:
host = None
username = None
password = None
private_key = None
private_key_pass = None
attributes = dict()
attributes.update(self.attributes)
attributes.pop('remote_input_files', None)
job = Job(name=self.name.replace(' ', '_'),
attributes=self.condorpy_template,
executable=self.executable,
host=host,
username=username,
password=password,
private_key=private_key,
private_key_pass=private_key_pass,
remote_input_files=self.remote_input_files,
working_directory=self.workspace,
**attributes)
job._cluster_id = self.cluster_id
job._num_jobs = self.num_jobs
if self.remote_id:
job._remote_id = self.remote_id
else:
self.remote_id = job._remote_id
self._condorpy_job = job
return self._condorpy_job
@property
def statuses(self):
return self.condorpy_job.statuses
@property
def initial_dir(self):
return os.path.join(self.workspace, self.condorpy_job.initial_dir)
def _update_status(self):
if not self.execute_time:
return 'PEN'
try:
condor_status = self.condorpy_job.status
if condor_status == 'Various':
statuses = self.condorpy_job.statuses
running_statuses = statuses['Unexpanded'] + statuses[
'Idle'] + statuses['Running']
if not running_statuses:
condor_status = 'Various-Complete'
except Exception, e:
# raise e
condor_status = 'Submission_err'
self._status = self.STATUS_MAP[condor_status]
self.save()
class TethysJob(models.Model):
"""
Base class for all job types. This is intended to be an abstract class that is not directly instantiated.
"""
class Meta:
verbose_name = 'Job'
objects = InheritanceManager()
STATUSES = (
('PEN', 'Pending'),
('SUB', 'Submitted'),
('RUN', 'Running'),
('COM', 'Complete'),
('ERR', 'Error'),
('ABT', 'Aborted'),
('VAR', 'Various'),
('VCP', 'Various-Complete'),
)
STATUS_DICT = {k: v for v, k in STATUSES}
name = models.CharField(max_length=1024)
description = models.CharField(max_length=2048, blank=True, default='')
user = models.ForeignKey(User, on_delete=models.CASCADE)
label = models.CharField(max_length=1024)
creation_time = models.DateTimeField(auto_now_add=True)
execute_time = models.DateTimeField(blank=True, null=True)
start_time = models.DateTimeField(blank=True, null=True)
completion_time = models.DateTimeField(blank=True, null=True)
workspace = models.CharField(max_length=1024, default='')
extended_properties = DictionaryField(default='', blank=True)
_process_results_function = models.CharField(max_length=1024,
blank=True,
null=True)
_status = models.CharField(max_length=3,
choices=STATUSES,
default=STATUSES[0][0])
@property
def update_status_interval(self):
if not hasattr(self, '_update_status_interval'):
self._update_status_interval = datetime.timedelta(seconds=10)
return self._update_status_interval
@property
def last_status_update(self):
return self._last_status_update
@property
def status(self):
self.update_status()
field = self._meta.get_field('_status')
status = self._get_FIELD_display(field)
return status
@property
def run_time(self):
# start_time = self.start_time
start_time = self.execute_time
if start_time:
end_time = self.completion_time or datetime.datetime.now(
start_time.tzinfo)
run_time = end_time - start_time
else:
if self.completion_time and self.execute_time:
run_time = self.completion_time - self.execute_time
else:
return ''
return run_time
def execute(self, *args, **kwargs):
"""
executes the job
"""
self.execute_time = timezone.now()
self._status = 'PEN'
self.save()
self._execute(*args, **kwargs)
def update_status(self, *args, **kwargs):
old_status = self._status
if self._status in ['PEN', 'SUB', 'RUN', 'VAR']:
if not hasattr(self, '_last_status_update') \
or datetime.datetime.now()-self.last_status_update > self.update_status_interval:
self._update_status(*args, **kwargs)
self._last_status_update = datetime.datetime.now()
if self._status == 'RUN' and (old_status == 'PEN'
or old_status == 'SUB'):
self.start_time = timezone.now()
if self._status == "COM" or self._status == "VCP":
self.process_results()
elif self._status == 'ERR' or self._status == 'ABT':
self.completion_time = timezone.now()
self.save()
@property
def process_results_function(self):
"""
Returns:
A function handle or None if function cannot be resolved.
"""
if self._process_results_function:
function_extractor = TethysFunctionExtractor(
self._process_results_function, None)
if function_extractor.valid:
return function_extractor.function
@process_results_function.setter
def process_results_function(self, function):
module_path = inspect.getmodule(function).__name__.split('.')[2:]
module_path.append(function.__name__)
self._process_results_function = '.'.join(module_path)
def process_results(self, *args, **kwargs):
"""
"""
self.completion_time = timezone.now()
self.save()
self._process_results(*args, **kwargs)
@abstractmethod
def _execute(self, *args, **kwargs):
pass
@abstractmethod
def _update_status(self, *args, **kwargs):
pass
@abstractmethod
def _process_results(self, *args, **kwargs):
pass
@abstractmethod
def stop(self):
"""
Stops job from executing
"""
raise NotImplementedError()
@abstractmethod
def pause(self):
"""
Pauses job during execution
"""
raise NotImplementedError()
@abstractmethod
def resume(self):
"""
Resumes a job that has been paused
"""
raise NotImplementedError()
class CondorPyJob(models.Model):
"""
Database model for condorpy jobs
"""
condorpyjob_id = models.AutoField(primary_key=True)
_attributes = DictionaryField(default='')
_num_jobs = models.IntegerField(default=1)
_remote_input_files = ListField(default='')
@classmethod
def get_condorpy_template(cls, template_name):
template_name = template_name or 'base'
template = getattr(Templates, template_name)
if not template:
template = Templates.base
return template
@property
def condorpy_job(self):
if not hasattr(self, '_condorpy_job'):
job = Job(name=self.name.replace(' ', '_'),
attributes=self.attributes,
num_jobs=self.num_jobs,
remote_input_files=self.remote_input_files,
working_directory=self.workspace)
self._condorpy_job = job
return self._condorpy_job
@property
def attributes(self):
return self._attributes
# @attributes.setter
# def attributes(self, attributes):
# assert isinstance(attributes, dict)
# self.condorpy_job._attributes = attributes
# self._attributes = attributes
@property
def num_jobs(self):
return self._num_jobs
@num_jobs.setter
def num_jobs(self, num_jobs):
num_jobs = int(num_jobs)
self.condorpy_job.num_jobs = num_jobs
self._num_jobs = num_jobs
@property
def remote_input_files(self):
return self._remote_input_files
@remote_input_files.setter
def remote_input_files(self, remote_input_files):
self.condorpy_job.remote_input_files = remote_input_files
self._remote_input_files = remote_input_files
@property
def initial_dir(self):
return os.path.join(self.workspace, self.condorpy_job.initial_dir)
def get_attribute(self, attribute):
self.condorpy_job.get(attribute)
def set_attribute(self, attribute, value):
setattr(self.condorpy_job, attribute, value)
def update_database_fields(self):
self._attributes = self.condorpy_job.attributes
self.num_jobs = self.condorpy_job.num_jobs
self.remote_input_files = self.condorpy_job.remote_input_files
def test_dictionary_field_formfield(self, mock_ff):
ret = DictionaryField()
ret.formfield(additional='test2')
mock_ff.assert_called_once()
def test_DictionaryField(self):
ret = DictionaryField()
self.assertEqual('Dictionary object', ret.description)
class CondorPyWorkflow(models.Model):
"""
Database model for condorpy workflows
"""
condorpyworkflow_id = models.AutoField(primary_key=True)
_max_jobs = DictionaryField(default='', blank=True)
_config = models.CharField(max_length=1024, null=True, blank=True)
@property
def condorpy_workflow(self):
"""
Returns: an instance of a condorpy Workflow
"""
if not hasattr(self, '_condorpy_workflow'):
workflow = Workflow(name=self.name.replace(' ', '_'),
max_jobs=self.max_jobs,
config=self.config,
working_directory=self.workspace
)
self._condorpy_workflow = workflow
self.load_nodes()
return self._condorpy_workflow
@property
def max_jobs(self):
return self._max_jobs
@max_jobs.setter
def max_jobs(self, max_jobs):
self.condorpy_workflow._max_jobs = max_jobs
self._max_jobs = max_jobs
@property
def config(self):
return self._config
@config.setter
def config(self, config):
self.condorpy_workflow.config = config
self._config = config
@property
def nodes(self):
return self.node_set.select_subclasses()
@property
def num_jobs(self):
return self.condorpy_workflow.num_jobs
def load_nodes(self):
workflow = self.condorpy_workflow
node_dict = dict()
def add_node_to_dict(node):
if node not in node_dict:
node_dict[node] = node.condorpy_node
for node in self.nodes:
add_node_to_dict(node)
condorpy_node = node_dict[node]
parents = node.parents
for parent in parents:
add_node_to_dict(parent)
condorpy_node.add_parent(node_dict[parent])
workflow.add_node(condorpy_node)
def add_max_jobs_throttle(self, category, max_jobs):
"""
Adds a max_jobs attribute to the workflow to throttle the number of jobs in a category
Args:
category (str): The category to throttle.
max_jobs (int): The maximum number of jobs that submit at one time
"""
self.max_jobs[category] = max_jobs
self.condorpy_workflow.add_max_jobs_throttle(category, max_jobs)
def update_database_fields(self):
# self.max_jobs = self.condorpy_workflow.max_jobs
# self.config = self.condorpy_workflow.config
for node in self.nodes:
node.update_database_fields()
def test_dictionary_field_get_internal_type(self):
ret = DictionaryField()
self.assertEqual('TextField', ret.get_internal_type())
class CondorPyJob(models.Model):
"""
Database model for condorpy jobs
"""
condorpyjob_id = models.AutoField(primary_key=True)
_attributes = DictionaryField(default='')
_num_jobs = models.IntegerField(default=1)
_remote_input_files = ListField(default='')
def __init__(self, *args, **kwargs):
# if condorpy_template_name or attributes is passed in then get the template and add it to the _attributes
attributes = kwargs.pop('attributes', dict())
_attributes = kwargs.get('_attributes', dict())
attributes.update(_attributes)
condorpy_template_name = kwargs.pop('condorpy_template_name', None)
if condorpy_template_name is not None:
template = self.get_condorpy_template(condorpy_template_name)
template.update(attributes)
attributes = template
kwargs['_attributes'] = attributes
super(CondorPyJob, self).__init__(*args, **kwargs)
@classmethod
def get_condorpy_template(cls, template_name):
template_name = template_name or 'base'
template = getattr(Templates, template_name, None)
if not template:
template = Templates.base
return template
@property
def condorpy_job(self):
if not hasattr(self, '_condorpy_job'):
job = Job(name=self.name.replace(' ', '_'),
attributes=self.attributes,
num_jobs=self.num_jobs,
remote_input_files=self.remote_input_files,
working_directory=self.workspace)
self._condorpy_job = job
return self._condorpy_job
@property
def attributes(self):
return self._attributes
@attributes.setter
def attributes(self, attributes):
assert isinstance(attributes, dict)
self._attributes = attributes
self.condorpy_job._attributes = attributes
@property
def num_jobs(self):
return self._num_jobs
@num_jobs.setter
def num_jobs(self, num_jobs):
num_jobs = int(num_jobs)
self.condorpy_job.num_jobs = num_jobs
self._num_jobs = num_jobs
@property
def remote_input_files(self):
return self._remote_input_files
@remote_input_files.setter
def remote_input_files(self, remote_input_files):
self.condorpy_job.remote_input_files = remote_input_files
self._remote_input_files = remote_input_files
@property
def initial_dir(self):
return os.path.join(self.workspace, self.condorpy_job.initial_dir)
def get_attribute(self, attribute):
return self.condorpy_job.get(attribute)
def set_attribute(self, attribute, value):
setattr(self.condorpy_job, attribute, value)
def update_database_fields(self):
self._attributes = self.condorpy_job.attributes
self.num_jobs = self.condorpy_job.num_jobs
self.remote_input_files = self.condorpy_job.remote_input_files
def test_dictionary_field_get_internal_type(self):
ret = DictionaryField()
self.assertEqual('TextField', ret.get_internal_type())
