def from_yaml(data):
        try:
            name = data['challenge']
            title = data['title']
            description = data['description']
            protocol = data['protocol']
            date_open = data['date-open']
            date_close = data['date-close']

            roles = data['roles']
            transitions = data['transitions']
            steps = data['steps']
            Steps = {}
            for k, v in steps.items():
                Steps[k] = ChallengeStep.from_yaml(name, v)

            return ChallengeDescription(name,
                                        title,
                                        description,
                                        protocol,
                                        date_open,
                                        date_close,
                                        Steps,
                                        roles=roles,
                                        transitions=transitions)
        except KeyError as e:
            msg = 'Missing config %s' % e
            raise_wrapped(InvalidChallengeDescription, e, msg)
Пример #2
0
def substitute_dependencies(a, db):
    from compmake import Promise

    # XXX: this is a workaround
    if type(a).__name__ in ['ObjectSpec']:
        return deepcopy(a)
    
    if isinstance(a, dict):
        ca = type(a)
        rest = [(k, substitute_dependencies(v, db=db)) for k, v in a.items()]
        try:
            res = ca(rest)
            #print('%s->%s' % (a, str(res)))
            return res
        except (BaseException, TypeError) as e:
            msg = 'Could not instance something looking like a dict.',
            raise_wrapped(Exception, e, msg, ca=ca)

    elif isinstance(a, list):
        # XXX: This fails for subclasses of list
        return type(a)([substitute_dependencies(x, db=db) for x in a])
    elif isinstance(a, tuple):
        # XXX: This fails for subclasses of tuples
        return type(a)([substitute_dependencies(x, db=db) for x in a])
    elif isinstance(a, Promise):
        s = get_job_userobject(a.job_id, db=db)
        return substitute_dependencies(s, db=db)
    else:
        return deepcopy(a)
Пример #3
0
 def bel(self, x):
     from .space import NotBelongs
     try:
         f(self, x)
     except NotBelongs as e:
         msg = 'Point does not belong to space.'
         raise_wrapped(NotBelongs, e, msg, space=self, x=x, compact=True)
     return f
Пример #4
0
 def bel(self, x):
     from .space import NotBelongs
     try:
         f(self, x)
     except NotBelongs as e:
         msg = 'Point does not belong to space.'
         raise_wrapped(NotBelongs, e, msg, space=self, x=x, compact=True)
     return f
Пример #5
0
    def __init__(self, dp, fnames, rnames, icon=None):
        assert isinstance(dp, PrimitiveDP), type(dp)
        self.dp = dp

        F = self.dp.get_fun_space()
        R = self.dp.get_res_space()

        try:
            # assume that dp has product spaces of given length

            if isinstance(rnames, list):
                if not len(set(rnames)) == len(rnames):
                    raise ValueError('Repeated rnames.')

            if isinstance(fnames, list):
                if not len(set(fnames)) == len(fnames):
                    raise ValueError('Repeated fnames.')

            if isinstance(fnames, str):
                self.F_single = True
                self.Fname = fnames
            else:

                if isinstance(F, PosetProduct):
                    if not isinstance(fnames, list) or not len(F) == len(fnames):
                        raise ValueError("F incompatible")
                    self.F_single = False
                    self.Fnames = fnames
                else:
                    if not isinstance(fnames, str):
                        msg = "F and fnames incompatible: not a string"
                        raise_desc(ValueError, msg, F=F, fnames=fnames)
                    self.F_single = True
                    self.Fname = fnames


            if isinstance(rnames, str):
                self.R_single = True
                self.Rname = rnames
            else:
                if isinstance(R, PosetProduct):
                    if not isinstance(rnames, list) or not len(R) == len(rnames):
                        raise ValueError("R incompatible")
                    self.R_single = False
                    self.Rnames = rnames
                else:
                    if not isinstance(rnames, str):
                        msg = "R and rnames incompatible: not a string"
                        raise_desc(ValueError, msg, R=R, rnames=rnames)
                    self.R_single = True
                    self.Rname = rnames
            self.icon = icon
        except Exception as e:
            msg = 'Cannot wrap primitive DP.'
            raise_wrapped(ValueError, e, msg, dp=self.dp, F=F, R=R,
                          fnames=fnames, rnames=rnames, exc=sys.exc_info())
Пример #6
0
def poset_check_chain(poset, chain):
    """ Raises an exception if the chain is not a chain. """
    chain = list(chain)
    for i in range(len(chain) - 1):
        try:
            poset.check_leq(chain[i], chain[i + 1])
        except NotLeq as e:
            msg = ('Fails for i = %s: %s ≰ %s' % (i, chain[i], chain[i + 1]))
            raise_wrapped(ValueError, e, msg, compact=True, chain=chain, poset=poset)

    return True
Пример #7
0
    def from_dict(res):
        from compmake.jobs.result_dict import result_dict_check

        result_dict_check(res)
        try:
            res['abort']
            e = HostFailed(host=res['host'],
                           job_id=res['job_id'],
                           bt=res['bt'],
                           reason=res['reason'])
        except KeyError as e:
            raise_wrapped(CompmakeBug, e, 'Incomplete dict', res=res,
                          keys=list(res.keys()))

        return e
Пример #8
0
Файл: utils.py Проект: rusi/mcdp
def poset_check_chain(poset, chain):
    """ Raises an exception if the chain is not a chain. """
    chain = list(chain)
    for i in range(len(chain) - 1):
        try:
            poset.check_leq(chain[i], chain[i + 1])
        except NotLeq as e:
            msg = ('Fails for i = %s: %s ≰ %s' % (i, chain[i], chain[i + 1]))
            raise_wrapped(ValueError,
                          e,
                          msg,
                          compact=True,
                          chain=chain,
                          poset=poset)

    return True
Пример #9
0
def make_server_request(token, endpoint, data=None, method='GET', timeout=3):
    """
        Raise RequestFailed or ConnectionError.

        Returns the result in 'result'.
    """
    server = get_duckietown_server_url()
    url = server + endpoint

    headers = {'X-Messaging-Token': token}
    if data is not None:
        data = json.dumps(data)
    req = urllib2.Request(url, headers=headers, data=data)
    req.get_method = lambda: method
    try:
        res = urllib2.urlopen(req, timeout=timeout)
        data = res.read()
    except urllib2.URLError as e:
        msg = 'Cannot connect to server %s' % url
        raise_wrapped(ConnectionError, e, msg)
        raise

    try:
        result = json.loads(data)
    except ValueError as e:
        msg = 'Cannot read answer from server.'
        msg += '\n\n' + indent(data, '  > ')
        raise_wrapped(ConnectionError, e, msg)
        raise

    if not isinstance(result, dict) or 'ok' not in result:
        msg = 'Server provided invalid JSON response. Expected a dict with "ok" in it.'
        msg += '\n\n' + indent(data, '  > ')
        raise ConnectionError(msg)

    if result['ok']:
        if 'result' not in result:
            msg = 'Server provided invalid JSON response. Expected a field "result".'
            msg += '\n\n' + indent(result, '  > ')
            raise ConnectionError(msg)
        return result['result']
    else:
        msg = 'Failed request for %s:\n%s' % (url, result.get('error', result))
        raise RequestFailed(msg)
Пример #10
0
def substitute_dependencies(a, db):
    from compmake import Promise

    # XXX: this is a workaround
    if leave_it_alone(a):
        return deepcopy(a)
    
    if isinstance(a, dict):
        ca = type(a)
        rest = [(k, substitute_dependencies(v, db=db)) for k, v in a.items()]
        try:
            res = ca(rest)
            #print('%s->%s' % (a, str(res)))
            return res
        except (BaseException, TypeError) as e:
            msg = 'Could not instance something looking like a dict.',
            raise_wrapped(Exception, e, msg, ca=ca)

    elif isinstance(a, list):
        # XXX: This fails for subclasses of list
        return type(a)([substitute_dependencies(x, db=db) for x in a])
    elif isinstance(a, tuple):
        # First, check that there are dependencies
        deps_in_tuple = collect_dependencies(a)
        if not deps_in_tuple:
            # if not, just return the tuple
            return a
        # XXX: This fails for subclasses of tuples
        assert not isnamedtupleinstance(a), a

        ta = type(a)
        contents = ([substitute_dependencies(x, db=db) for x in a])
        try:
            return ta(contents)
        except TypeError as e:
            msg = 'Cannot reconstruct complex tuple.'
            raise_wrapped(ValueError, e, msg, ta=ta, contents=contents)
    elif isinstance(a, Promise):
        s = get_job_userobject(a.job_id, db=db)
        return substitute_dependencies(s, db=db)
    else:
        return a
Пример #11
0
def substitute_dependencies(a, db):
    from compmake import Promise

    # XXX: this is a workaround
    if leave_it_alone(a):
        return deepcopy(a)

    if isinstance(a, dict):
        ca = type(a)
        rest = [(k, substitute_dependencies(v, db=db)) for k, v in a.items()]
        try:
            res = ca(rest)
            #print('%s->%s' % (a, str(res)))
            return res
        except (BaseException, TypeError) as e:
            msg = 'Could not instance something looking like a dict.',
            raise_wrapped(Exception, e, msg, ca=ca)

    elif isinstance(a, list):
        # XXX: This fails for subclasses of list
        return type(a)([substitute_dependencies(x, db=db) for x in a])
    elif isinstance(a, tuple):
        # First, check that there are dependencies
        deps_in_tuple = collect_dependencies(a)
        if not deps_in_tuple:
            # if not, just return the tuple
            return a
        # XXX: This fails for subclasses of tuples
        assert not isnamedtupleinstance(a), a

        ta = type(a)
        contents = ([substitute_dependencies(x, db=db) for x in a])
        try:
            return ta(contents)
        except TypeError as e:
            msg = 'Cannot reconstruct complex tuple.'
            raise_wrapped(ValueError, e, msg, ta=ta, contents=contents)
    elif isinstance(a, Promise):
        s = get_job_userobject(a.job_id, db=db)
        return substitute_dependencies(s, db=db)
    else:
        return a
Пример #12
0
    def __call__(self, x):
        if do_extra_checks():
            D = self.get_domain()
            try:
                D.belongs(x)
            except NotBelongs as e:
                msg = 'Point does not belong to domain.'
                raise_wrapped(NotBelongs, e, msg, map=self, x=x, domain=D)

        y = self._call(x)

        if do_extra_checks():
            C = self.get_codomain()
            try:
                C.belongs(y)
            except NotBelongs as e:
                msg = 'Point does not belong to codomain.'
                raise_wrapped(NotBelongs, e, msg, map=self, y=y, codomain=C)

        return y
Пример #13
0
Файл: space.py Проект: rusi/mcdp
    def __call__(self, x):
        if do_extra_checks():
            D = self.get_domain()
            try:
                D.belongs(x)
            except NotBelongs as e:
                msg = 'Point does not belong to domain.'
                raise_wrapped(NotBelongs, e, msg, map=self, x=x, domain=D)

        y = self._call(x)

        if do_extra_checks():
            C = self.get_codomain()
            try:
                C.belongs(y)
            except NotBelongs as e:
                msg = 'Point does not belong to codomain.'
                raise_wrapped(NotBelongs, e, msg, map=self, y=y, codomain=C)

        return y
Пример #14
0
def mplayer_identify(filename, intolerant=True):
    """
        Returns a dictionary, with fields:
        
            width, height
            fps *
            length * 
            
            extra_mencoder_info
            
        Mplayer might fail to identify FPS and length. If intolerant=True,
        an error is raised, otherwise they are set to None.
    
    """
 
    # need at least 1 frame otherwise sometimes the video aspect is not known
    args = ('mplayer -identify -vo null -ao null -frames 1'.split()
            + [filename])
    
    try:
        
        try:
            res = system_cmd_result('.', args,
                      display_stdout=False,
                      display_stderr=False,
                      raise_on_error=True,
                      capture_keyboard_interrupt=False)
        except CmdException:
            raise 
        
    except Exception as e:
        raise_wrapped(Exception, e, "Could not identify movie", 
                      filename=filename)

    try:
        output = res.stdout
        return parse_mplayer_info_output(output,intolerant=intolerant)
    except Exception as e:
        raise_wrapped(Exception, e, "Could not identify movie",
                      cmd=" ".join(args), filename=filename,
                      output=output)
Пример #15
0
def mplayer_identify(filename, intolerant=True):
    """
        Returns a dictionary, with fields:
        
            width, height
            fps *
            length * 
            
            extra_mencoder_info
            
        Mplayer might fail to identify FPS and length. If intolerant=True,
        an error is raised, otherwise they are set to None.
    
    """
 
    # need at least 1 frame otherwise sometimes the video aspect is not known
    args = ('mplayer -identify -vo null -ao null -frames 1'.split()
            + [filename])
    
    try:
        
        try:
            res = system_cmd_result('.', args,
                      display_stdout=False,
                      display_stderr=False,
                      raise_on_error=True,
                      capture_keyboard_interrupt=False)
        except CmdException:
            raise 
        
    except Exception as e:
        raise_wrapped(Exception, e, "Could not identify movie", 
                      filename=filename)

    try:
        output = res.stdout
        return parse_mplayer_info_output(output,intolerant=intolerant)
    except Exception as e:
        raise_wrapped(Exception, e, "Could not identify movie",
                      cmd=" ".join(args), filename=filename,
                      output=output)
Пример #16
0
def interpret_commands_wrap(commands, context, cq):
    """ 
        Returns None or raises CommandFailed, ShellExitRequested, 
            CompmakeBug, KeyboardInterrupt.
    """
    assert context is not None
    publish(context, 'command-line-starting', command=commands)

    try:
        interpret_commands(commands, context=context, cq=cq)
        publish(context, 'command-line-succeeded', command=commands)
    except CompmakeBug:
        raise
    except UserError as e:
        publish(context, 'command-line-failed', command=commands, reason=e)
        raise CommandFailed(str(e))
    except CommandFailed as e:
        publish(context, 'command-line-failed', command=commands, reason=e)
        raise
    except (KeyboardInterrupt, JobInterrupted) as e:
        publish(context,
                'command-line-interrupted',
                command=commands,
                reason='KeyboardInterrupt')
        # If debugging
        # tb = traceback.format_exc()
        # print tb  # XXX
        raise CommandFailed(str(e))
        # raise CommandFailed('Execution of %r interrupted.' % commands)
    except ShellExitRequested:
        raise
    except Exception as e:
        tb = traceback.format_exc()
        msg0 = ('Warning, I got this exception, while it should '
                'have been filtered out already. '
                'This is a compmake BUG that should be reported '
                'at http://github.com/AndreaCensi/compmake/issues')
        msg = msg0 + "\n" + indent(tb, 'bug| ')
        publish(context, 'compmake-bug', user_msg=msg, dev_msg="")  # XXX
        raise_wrapped(CompmakeBug, e, msg)
Пример #17
0
def interpret_commands_wrap(commands, context, cq):
    """ 
        Returns None or raises CommandFailed, ShellExitRequested, 
            CompmakeBug, KeyboardInterrupt.
    """
    assert context is not None
    publish(context, 'command-line-starting', command=commands)

    try:
        interpret_commands(commands, context=context, cq=cq)
        publish(context, 'command-line-succeeded', command=commands)
    except CompmakeBug:
        raise
    except UserError as e:
        publish(context, 'command-line-failed', command=commands, reason=e)
        raise CommandFailed(str(e))
    except CommandFailed as e:
        publish(context, 'command-line-failed', command=commands, reason=e)
        raise
    except (KeyboardInterrupt, JobInterrupted) as e:
        publish(context, 'command-line-interrupted',
                command=commands, reason='KeyboardInterrupt')
        # If debugging
        # tb = traceback.format_exc()
        # print tb  # XXX 
        raise CommandFailed(str(e))
        # raise CommandFailed('Execution of %r interrupted.' % commands)
    except ShellExitRequested:
        raise
    except Exception as e:
        tb = traceback.format_exc()
        msg0 = ('Warning, I got this exception, while it should '
                'have been filtered out already. '
                'This is a compmake BUG that should be reported '
                'at http://github.com/AndreaCensi/compmake/issues')
        msg = msg0 + "\n" + indent(tb, 'bug| ')
        publish(context, 'compmake-bug', user_msg=msg, dev_msg="")  # XXX
        raise_wrapped(CompmakeBug, e, msg0)
Пример #18
0
def comp_(context, command_, *args, **kwargs):
    """
        Main method to define a computation step.

        Extra arguments:

        :arg:job_id:   sets the job id (respects job_prefix)
        :arg:extra_dep: extra dependencies (not passed as arguments)
        :arg:command_name: used to define job name if job_id not provided.
        If not given, command_.__name__ is used.

        :arg:needs_context: if this is a dynamic job

        Raises UserError if command is not pickable.
    """

    db = context.get_compmake_db()

    command = command_

    if hasattr(command, '__module__') and command.__module__ == '__main__':
        if not command in WarningStorage.warned:
            if WarningStorage.warned:
                # already warned for another function
                msg = ('(Same warning for function %r.)' % command.__name__)
            else:
                msg = ("A warning about the function %r: " % command.__name__)
                msg += (
                    "This function is defined directly in the __main__ "
                    "module, "
                    "which means that it cannot be pickled correctly due to "
                    "a limitation of Python and 'make new_process=1' will "
                    "fail. "
                    "For best results, please define functions in external "
                    "modules. "
                    'For more info, read '
                    'http://stefaanlippens.net/pickleproblem '
                    'and the bug report http://bugs.python.org/issue5509.')
            warning(msg)
            WarningStorage.warned.add(command)

    if get_compmake_status() == CompmakeConstants.compmake_status_slave:
        return None

    # Check that this is a pickable function
    try:
        try_pickling(command)
    except Exception as e:
        msg = ('Cannot pickle function. Make sure it is not a lambda '
               'function or a nested function. (This is a limitation of '
               'Python)')
        raise_wrapped(UserError, e, msg, command=command)

    if CompmakeConstants.command_name_key in kwargs:
        command_desc = kwargs.pop(CompmakeConstants.command_name_key)
    elif hasattr(command, '__name__'):
        command_desc = command.__name__
    else:
        command_desc = type(command).__name__

    args = list(args)  # args is a non iterable tuple

    # Get job id from arguments
    if CompmakeConstants.job_id_key in kwargs:
        # make sure that command does not have itself a job_id key
        try:
            argspec = inspect.getargspec(command)
        except TypeError:
            # Assume Cython function
            # XXX: write test
            pass
        else:
            if CompmakeConstants.job_id_key in argspec.args:
                msg = ("You cannot define the job id in this way because %r "
                       "is already a parameter of this function." %
                       CompmakeConstants.job_id_key)
                raise UserError(msg)

        job_id = kwargs[CompmakeConstants.job_id_key]
        check_isinstance(job_id, six.string_types)
        if ' ' in job_id:
            msg = 'Invalid job id: %r' % job_id
            raise UserError(msg)

        job_prefix = context.get_comp_prefix()
        if job_prefix:
            job_id = '%s-%s' % (job_prefix, job_id)

        del kwargs[CompmakeConstants.job_id_key]

        if context.was_job_defined_in_this_session(job_id):
            # unless it is dynamically geneterated
            if not job_exists(job_id, db=db):
                msg = 'The job %r was defined but not found in DB. I will let it slide.' % job_id
                print(msg)
            else:
                msg = 'The job %r was already defined in this session.' % job_id
                old_job = get_job(job_id, db=db)
                msg += '\n  old_job.defined_by: %s ' % old_job.defined_by
                msg += '\n context.currently_executing: %s ' % context.currently_executing
                msg += ' others defined in session: %s' % context.get_jobs_defined_in_this_session(
                )
                print(msg)
                #                 warnings.warn('I know something is more complicated here')
                #             if old_job.defined_by is not None and
                # old_job.defined_by == context.currently_executing:
                #                 # exception, it's ok
                #                 pass
                #             else:

                msg = 'Job %r already defined.' % job_id
                raise UserError(msg)
        else:
            if job_exists(job_id, db=db):
                # ok, you gave us a job_id, but we still need to check whether
                # it is the same job
                stack = context.currently_executing
                defined_by = get_job(job_id, db=db).defined_by
                if defined_by == stack:
                    # this is the same job-redefining
                    pass
                else:

                    for i in range(1000):  # XXX
                        n = '%s-%d' % (job_id, i)
                        if not job_exists(n, db=db):
                            job_id = n
                            break

                    if False:
                        print('The job_id %r was given explicitly but already '
                              'defined.' % job_id)
                        print('current stack: %s' % stack)
                        print('    its stack: %s' % defined_by)
                        print('New job_id is %s' % job_id)

    else:
        job_id = generate_job_id(command_desc, context=context)

    context.add_job_defined_in_this_session(job_id)

    # could be done better
    if 'needs_context' in kwargs:
        needs_context = True
        del kwargs['needs_context']
    else:
        needs_context = False

    if CompmakeConstants.extra_dep_key in kwargs:
        extra_dep = kwargs[CompmakeConstants.extra_dep_key]
        del kwargs[CompmakeConstants.extra_dep_key]

        if not isinstance(extra_dep, (list, Promise)):
            msg = ('The "extra_dep" argument must be a list of promises; '
                   'got: %s' % describe_value(extra_dep))
            raise ValueError(msg)
        if isinstance(extra_dep, Promise):
            extra_dep = [extra_dep]
        assert isinstance(extra_dep, list)
        for ed in extra_dep:
            if not isinstance(ed, Promise):
                msg = ('The "extra_dep" argument must be a list of promises; '
                       'got: %s' % describe_value(extra_dep))
                raise ValueError(msg)
        extra_dep = collect_dependencies(extra_dep)

    else:
        extra_dep = set()

    children = collect_dependencies([args, kwargs])
    children.update(extra_dep)

    for c in children:
        if not job_exists(c, db):
            msg = "Job %r references a job %r that doesnt exist." % (job_id, c)
            raise ValueError(msg)

    all_args = (command, args, kwargs)

    assert len(context.currently_executing) >= 1
    assert context.currently_executing[0] == 'root'

    c = Job(job_id=job_id,
            children=children,
            command_desc=command_desc,
            needs_context=needs_context,
            defined_by=context.currently_executing)

    # Need to inherit the pickle
    if context.currently_executing[-1] != 'root':
        parent_job = get_job(context.currently_executing[-1], db)
        c.pickle_main_context = parent_job.pickle_main_context

    if job_exists(job_id, db):
        old_job = get_job(job_id, db)

        if old_job.defined_by != c.defined_by:
            warning('Redefinition of %s: ' % job_id)
            warning(' cur defined_by: %s' % c.defined_by)
            warning(' old defined_by: %s' % old_job.defined_by)

        if old_job.children != c.children:
            #warning('Redefinition problem:')
            #warning(' old children: %s' % (old_job.children))
            #warning(' old dyn children: %s' % old_job.dynamic_children)
            #warning(' new children: %s' % (c.children))

            # fixing this
            for x, deps in old_job.dynamic_children.items():
                if not x in c.children:
                    # not a child any more
                    # FIXME: ok but note it might be a dependence of a child
                    # continue
                    pass
                c.dynamic_children[x] = deps
                for j in deps:
                    if not j in c.children:
                        c.children.add(j)

        if old_job.parents != c.parents:
            # warning('Redefinition of %s: ' % job_id)
            #  warning(' cur parents: %s' % (c.parents))
            # warning(' old parents: %s' % old_job.parents)
            for p in old_job.parents:
                c.parents.add(p)

                # TODO: preserve defines
                #     from compmake.ui.visualization import info
                #     info('defining job %r with children %r' % (job_id,
                # c.children))

                #     if True or c.defined_by == ['root']:

    for child in children:
        db_job_add_parent_relation(child=child, parent=job_id, db=db)

    if get_compmake_config('check_params') and job_exists(job_id, db):
        # OK, this is going to be black magic.
        # We want to load the previous job definition,
        # however, by unpickling(), it will start
        # __import__()ing the modules, perhaps
        # even the one that is calling us.
        # What happens, then is that it will try to
        # add another time this computation recursively.
        # What we do, is that we temporarely switch to
        # slave mode, so that recursive calls to comp()
        # are disabled.
        #             old_status = get_compmake_status()
        #             set_compmake_status(
        # CompmakeConstants.compmake_status_slave)
        all_args_old = get_job_args(job_id, db=db)
        #             set_compmake_status(old_status)
        same, reason = same_computation(all_args, all_args_old)

        if not same:
            #print('different job, cleaning cache:\n%s  ' % reason)
            from compmake.jobs.actions import clean_targets
            clean_targets([job_id], db)
            #             if job_cache_exists(job_id, db):
            #                 delete_job_cache(job_id, db)
            publish(context, 'job-redefined', job_id=job_id, reason=reason)
        else:
            # print('ok, same job')
            pass
            # XXX TODO clean the cache
            #             else:
            #                 publish(context, 'job-already-defined',
            # job_id=job_id)

    set_job_args(job_id, all_args, db=db)
    set_job(job_id, c, db=db)
    publish(context, 'job-defined', job_id=job_id)

    return Promise(job_id)
Пример #19
0
def comp_(context, command_, *args, **kwargs):
    """
        Main method to define a computation step.

        Extra arguments:

        :arg:job_id:   sets the job id (respects job_prefix)
        :arg:extra_dep: extra dependencies (not passed as arguments)
        :arg:command_name: used to define job name if job_id not provided.
        If not given, command_.__name__ is used.

        :arg:needs_context: if this is a dynamic job

        Raises UserError if command is not pickable.
    """

    db = context.get_compmake_db()

    command = command_

    if hasattr(command, '__module__') and command.__module__ == '__main__':
        if not command in WarningStorage.warned:
            if WarningStorage.warned:
                # already warned for another function
                msg = ('(Same warning for function %r.)' % command.__name__)
            else:
                msg = ("A warning about the function %r: " % command.__name__)
                msg += (
                    "This function is defined directly in the __main__ "
                    "module, "
                    "which means that it cannot be pickled correctly due to "
                    "a limitation of Python and 'make new_process=1' will "
                    "fail. "
                    "For best results, please define functions in external "
                    "modules. "
                    'For more info, read '
                    'http://stefaanlippens.net/pickleproblem '
                    'and the bug report http://bugs.python.org/issue5509.')
            warning(msg)
            WarningStorage.warned.add(command)

    if get_compmake_status() == CompmakeConstants.compmake_status_slave:
        return None

    # Check that this is a pickable function
    try:
        try_pickling(command)
    except Exception as e:
        msg = ('Cannot pickle function. Make sure it is not a lambda '
               'function or a nested function. (This is a limitation of '
               'Python)')
        raise_wrapped(UserError, e, msg, command=command)

    if CompmakeConstants.command_name_key in kwargs:
        command_desc = kwargs.pop(CompmakeConstants.command_name_key)
    elif hasattr(command, '__name__'):
        command_desc = command.__name__
    else:
        command_desc = type(command).__name__

    args = list(args)  # args is a non iterable tuple

    # Get job id from arguments
    if CompmakeConstants.job_id_key in kwargs:
        # make sure that command does not have itself a job_id key
        try:
            argspec = inspect.getargspec(command)
        except TypeError:
            # Assume Cython function
            # XXX: write test
            pass
        else:
            if CompmakeConstants.job_id_key in argspec.args:
                msg = ("You cannot define the job id in this way because %r "
                       "is already a parameter of this function." %
                       CompmakeConstants.job_id_key)
                raise UserError(msg)

        job_id = kwargs[CompmakeConstants.job_id_key]
        check_isinstance(job_id, six.string_types)
        if ' ' in job_id:
            msg = 'Invalid job id: %r' % job_id
            raise UserError(msg)

        job_prefix = context.get_comp_prefix()
        if job_prefix:
            job_id = '%s-%s' % (job_prefix, job_id)

        del kwargs[CompmakeConstants.job_id_key]

        if context.was_job_defined_in_this_session(job_id):
            # unless it is dynamically geneterated
            if not job_exists(job_id, db=db):
                msg = 'The job %r was defined but not found in DB. I will let it slide.' % job_id
                print(msg)
            else:
                msg = 'The job %r was already defined in this session.' % job_id
                old_job = get_job(job_id, db=db)
                msg += '\n  old_job.defined_by: %s ' % old_job.defined_by
                msg += '\n context.currently_executing: %s ' % context.currently_executing
                msg += ' others defined in session: %s' % context.get_jobs_defined_in_this_session()
                print(msg)
#                 warnings.warn('I know something is more complicated here')
                #             if old_job.defined_by is not None and
                # old_job.defined_by == context.currently_executing:
                #                 # exception, it's ok
                #                 pass
                #             else:

                msg = 'Job %r already defined.' % job_id
                raise UserError(msg)
        else:
            if job_exists(job_id, db=db):
                # ok, you gave us a job_id, but we still need to check whether
                # it is the same job
                stack = context.currently_executing
                defined_by = get_job(job_id, db=db).defined_by
                if defined_by == stack:
                    # this is the same job-redefining
                    pass
                else:

                    for i in range(1000):  # XXX
                        n = '%s-%d' % (job_id, i)
                        if not job_exists(n, db=db):
                            job_id = n
                            break
                        
                    if False:
                        print(
                            'The job_id %r was given explicitly but already '
                            'defined.' % job_id)
                        print('current stack: %s' % stack)
                        print('    its stack: %s' % defined_by)
                        print('New job_id is %s' % job_id)

    else:
        job_id = generate_job_id(command_desc, context=context)

    context.add_job_defined_in_this_session(job_id)

    # could be done better
    if 'needs_context' in kwargs:
        needs_context = True
        del kwargs['needs_context']
    else:
        needs_context = False

    if CompmakeConstants.extra_dep_key in kwargs:
        extra_dep = kwargs[CompmakeConstants.extra_dep_key]
        del kwargs[CompmakeConstants.extra_dep_key]

        if not isinstance(extra_dep, (list, Promise)):
            msg = ('The "extra_dep" argument must be a list of promises; '
                   'got: %s' % describe_value(extra_dep))
            raise ValueError(msg)
        if isinstance(extra_dep, Promise):
            extra_dep = [extra_dep]
        assert isinstance(extra_dep, list)
        for ed in extra_dep:
            if not isinstance(ed, Promise):
                msg = ('The "extra_dep" argument must be a list of promises; '
                       'got: %s' % describe_value(extra_dep))
                raise ValueError(msg)
        extra_dep = collect_dependencies(extra_dep)

    else:
        extra_dep = set()

    children = collect_dependencies([args, kwargs])
    children.update(extra_dep)

    for c in children:
        if not job_exists(c, db):
            msg = "Job %r references a job %r that doesnt exist." % (job_id, c)
            raise ValueError(msg)

    all_args = (command, args, kwargs)

    assert len(context.currently_executing) >= 1
    assert context.currently_executing[0] == 'root'
    
    c = Job(job_id=job_id,
            children=children,
            command_desc=command_desc,
            needs_context=needs_context,
            defined_by=context.currently_executing)
    
    # Need to inherit the pickle
    if context.currently_executing[-1] != 'root':
        parent_job = get_job(context.currently_executing[-1], db)
        c.pickle_main_context = parent_job.pickle_main_context

    if job_exists(job_id, db):
        old_job = get_job(job_id, db)

        if old_job.defined_by != c.defined_by:
            warning('Redefinition of %s: ' % job_id)
            warning(' cur defined_by: %s' % c.defined_by)
            warning(' old defined_by: %s' % old_job.defined_by)

        if old_job.children != c.children:
            #warning('Redefinition problem:')
            #warning(' old children: %s' % (old_job.children))
            #warning(' old dyn children: %s' % old_job.dynamic_children)
            #warning(' new children: %s' % (c.children))

            # fixing this
            for x, deps in old_job.dynamic_children.items():
                if not x in c.children:
                    # not a child any more
                    # FIXME: ok but note it might be a dependence of a child
                    # continue
                    pass
                c.dynamic_children[x] = deps
                for j in deps:
                    if not j in c.children:
                        c.children.add(j)

        if old_job.parents != c.parents:
            # warning('Redefinition of %s: ' % job_id)
            #  warning(' cur parents: %s' % (c.parents))
            # warning(' old parents: %s' % old_job.parents)
            for p in old_job.parents:
                c.parents.add(p)

                # TODO: preserve defines
                #     from compmake.ui.visualization import info
                #     info('defining job %r with children %r' % (job_id,
                # c.children))

                #     if True or c.defined_by == ['root']:

    for child in children:
        db_job_add_parent_relation(child=child, parent=job_id, db=db)

    if get_compmake_config('check_params') and job_exists(job_id, db):
        # OK, this is going to be black magic.
        # We want to load the previous job definition,
        # however, by unpickling(), it will start
        # __import__()ing the modules, perhaps
        # even the one that is calling us.
        # What happens, then is that it will try to
        # add another time this computation recursively.
        # What we do, is that we temporarely switch to
        # slave mode, so that recursive calls to comp()
        # are disabled.
        #             old_status = get_compmake_status()
        #             set_compmake_status(
        # CompmakeConstants.compmake_status_slave)
        all_args_old = get_job_args(job_id, db=db)
        #             set_compmake_status(old_status)
        same, reason = same_computation(all_args, all_args_old)

        if not same:
            #print('different job, cleaning cache:\n%s  ' % reason)
            from compmake.jobs.actions import clean_targets
            clean_targets([job_id], db)
#             if job_cache_exists(job_id, db):
#                 delete_job_cache(job_id, db)
            publish(context, 'job-redefined', job_id=job_id, reason=reason)
        else:
            # print('ok, same job')
            pass
            # XXX TODO clean the cache
            #             else:
            #                 publish(context, 'job-already-defined',
            # job_id=job_id)

    set_job_args(job_id, all_args, db=db)
    set_job(job_id, c, db=db)
    publish(context, 'job-defined', job_id=job_id)

    return Promise(job_id)
Пример #20
0
Файл: wrap.py Проект: rusi/mcdp
    def __init__(self, dp, fnames, rnames, icon=None):
        assert isinstance(dp, PrimitiveDP), type(dp)
        self.dp = dp

        F = self.dp.get_fun_space()
        R = self.dp.get_res_space()

        try:
            # assume that dp has product spaces of given length

            if isinstance(rnames, list):
                if not len(set(rnames)) == len(rnames):
                    raise ValueError('Repeated rnames.')

            if isinstance(fnames, list):
                if not len(set(fnames)) == len(fnames):
                    raise ValueError('Repeated fnames.')

            if isinstance(fnames, str):
                self.F_single = True
                self.Fname = fnames
            else:

                if isinstance(F, PosetProduct):
                    if not isinstance(fnames,
                                      list) or not len(F) == len(fnames):
                        raise ValueError("F incompatible")
                    self.F_single = False
                    self.Fnames = fnames
                else:
                    if not isinstance(fnames, str):
                        msg = "F and fnames incompatible: not a string"
                        raise_desc(ValueError, msg, F=F, fnames=fnames)
                    self.F_single = True
                    self.Fname = fnames

            if isinstance(rnames, str):
                self.R_single = True
                self.Rname = rnames
            else:
                if isinstance(R, PosetProduct):
                    if not isinstance(rnames,
                                      list) or not len(R) == len(rnames):
                        raise ValueError("R incompatible")
                    self.R_single = False
                    self.Rnames = rnames
                else:
                    if not isinstance(rnames, str):
                        msg = "R and rnames incompatible: not a string"
                        raise_desc(ValueError, msg, R=R, rnames=rnames)
                    self.R_single = True
                    self.Rname = rnames
            self.icon = icon
        except Exception as e:
            msg = 'Cannot wrap primitive DP.'
            raise_wrapped(ValueError,
                          e,
                          msg,
                          dp=self.dp,
                          F=F,
                          R=R,
                          fnames=fnames,
                          rnames=rnames,
                          exc=sys.exc_info())