Example #1
0
def has_local_job_refs(io_hash):
    '''
    :param io_hash: input/output hash
    :type io_hash: dict
    :returns: boolean indicating whether any job-based object references are found in *io_hash*
    '''
    q = []

    for field in io_hash:
        if is_job_ref(io_hash[field]):
            if get_job_from_jbor(io_hash[field]).startswith('localjob'):
                return True
        elif isinstance(io_hash[field], list) or isinstance(io_hash[field], dict):
            q.append(io_hash[field])

    while len(q) > 0:
        thing = q.pop()
        if isinstance(thing, list):
            for i in range(len(thing)):
                if is_job_ref(thing[i]):
                    if get_job_from_jbor(thing[i]).startswith('localjob'):
                        return True
                elif isinstance(thing[i], list) or isinstance(thing[i], dict):
                    q.append(thing[i])
        else:
            for field in thing:
                if is_job_ref(thing[field]):
                    if get_job_from_jbor(thing[field]).startswith('localjob'):
                        return True
                elif isinstance(thing[field], list) or isinstance(thing[field], dict):
                    q.append(thing[field])

    return False
Example #2
0
def has_local_job_refs(io_hash):
    '''
    :param io_hash: input/output hash
    :type io_hash: dict
    :returns: boolean indicating whether any job-based object references are found in *io_hash*
    '''
    q = []

    for field in io_hash:
        if is_job_ref(io_hash[field]):
            if get_job_from_jbor(io_hash[field]).startswith('localjob'):
                return True
        elif isinstance(io_hash[field], list) or isinstance(
                io_hash[field], dict):
            q.append(io_hash[field])

    while len(q) > 0:
        thing = q.pop()
        if isinstance(thing, list):
            for i in range(len(thing)):
                if is_job_ref(thing[i]):
                    if get_job_from_jbor(thing[i]).startswith('localjob'):
                        return True
                elif isinstance(thing[i], list) or isinstance(thing[i], dict):
                    q.append(thing[i])
        else:
            for field in thing:
                if is_job_ref(thing[field]):
                    if get_job_from_jbor(thing[field]).startswith('localjob'):
                        return True
                elif isinstance(thing[field], list) or isinstance(
                        thing[field], dict):
                    q.append(thing[field])

    return False
Example #3
0
def resolve_job_ref(jbor, job_outputs={}, should_resolve=True):
    '''
    :param jbor: a dict that is a valid job-based object reference
    :type jbor: dict
    :param job_outputs: a dict of finished local jobs to their output hashes
    :type job_outputs: :class:`collections.OrderedDict`
    :returns: the referenced value if present
    :raises: :exc:`Exception` if the job-based object reference cannot be resolved

    TODO: Support metadata references
    '''
    ref_job_id = get_job_from_jbor(jbor)
    ref_job_field = get_field_from_jbor(jbor)
    if is_localjob_id(ref_job_id):
        if job_outputs.get(ref_job_id) is None:
            if should_resolve:
                raise Exception('Job ' + ref_job_id + ' not found in local finished jobs')
            else:
                return jbor
        if ref_job_field not in job_outputs[ref_job_id]:
            raise Exception('Cannot resolve a JBOR with job ID ' + ref_job_id + ' because field "' + ref_job_field + '" was not found in its output')
        return job_outputs[ref_job_id][ref_job_field]
    else:
        dxjob = dxpy.DXJob(ref_job_id)
        try:
            dxjob.wait_on_done()
        except Exception as e:
            raise Exception('Could not wait for ' + ref_job_id + ' to finish: ' + str(e))
        job_desc = dxjob.describe()
        if ref_job_field not in job_desc['output']:
            raise Exception('Cannot resolve a JBOR with job ID ' + ref_job_id + ' because field "' + ref_job_field + '" was not found in its output')
        return job_desc['output'][ref_job_field]
Example #4
0
 def test_run_workflow(self):
     launched_jobs = self.workflow.run({"0.record": dxpy.dxlink(self.closedrecord)})
     self.assertEqual(len(launched_jobs), 2)
     job_descs = [dxjob.describe() for dxjob in launched_jobs]
     self.assertEqual(job_descs[0]['name'], 'identity-record - workflowname')
     self.assertEqual(get_dxlink_ids(job_descs[0]['input']['record'])[0], self.closedrecord.get_id())
     self.assertTrue(is_job_ref(job_descs[1]['runInput']['record']))
     self.assertEqual(get_job_from_jbor(job_descs[1]['runInput']['record']), job_descs[0]['id'])
     self.assertEqual(get_field_from_jbor(job_descs[1]['runInput']['record']), 'record')
     launched_jobs[1].wait_on_done()
Example #5
0
def resolve_job_ref(jbor, job_outputs={}, should_resolve=True):
    '''
    :param jbor: a dict that is a valid job-based object reference
    :type jbor: dict
    :param job_outputs: a dict of finished local jobs to their output hashes
    :type job_outputs: :class:`collections.OrderedDict`
    :returns: the referenced value if present
    :raises: :exc:`Exception` if the job-based object reference cannot be resolved

    TODO: Support metadata references
    '''
    ref_job_id = get_job_from_jbor(jbor)
    ref_job_field = get_field_from_jbor(jbor)
    ref_job_index = get_index_from_jbor(jbor)

    def resolve_from_hash(output_hash):
        if ref_job_index is None:
            return output_hash[ref_job_field]
        else:
            return output_hash[ref_job_field][ref_job_index]

    if is_localjob_id(ref_job_id):
        if job_outputs.get(ref_job_id) is None:
            if should_resolve:
                raise Exception('Job ' + ref_job_id +
                                ' not found in local finished jobs')
            else:
                return jbor
        if ref_job_field not in job_outputs[ref_job_id]:
            raise Exception('Cannot resolve a JBOR with job ID ' + ref_job_id +
                            ' because field "' + ref_job_field +
                            '" was not found in its output')
        return resolve_from_hash(job_outputs[ref_job_id])
    else:
        dxjob = dxpy.DXJob(ref_job_id)
        try:
            dxjob.wait_on_done()
        except Exception as e:
            raise Exception('Could not wait for ' + ref_job_id +
                            ' to finish: ' + str(e))
        job_desc = dxjob.describe()
        if ref_job_field not in job_desc['output']:
            raise Exception('Cannot resolve a JBOR with job ID ' + ref_job_id +
                            ' because field "' + ref_job_field +
                            '" was not found in its output')
        return resolve_from_hash(job_desc['output'])