Exemplo n.º 1
0
    def _create_future(self, func, args, kwargs, filtered_args_dict, should_submit):
        if not func.version_info['ignore_deps']:
            # TODO: Check a map of possible dependencies executed for
            # this run here, in case a depdency just got changed. This
            # should be done by consulting a pickled on-file database
            # AFAICT.
            raise NotImplementedError('Please use ignore_deps for now')

        # Make job_path containing hashes
        h = NumpyHasher('sha1')
        h.hash((func.version_info['digest'], filtered_args_dict))
        job_name = '%s-%s' % (func.__name__,
                              self._encode_digest(h._hash.digest()))
        # Construct job dir if not existing. Remember that we may
        # race for this; if we got to create the directory, we have
        # the lock.
        we_made_it = self._ensure_job_dir(job_name, func, args, kwargs)
        is_generator = inspect.isgenerator(func) or inspect.isgeneratorfunction(func)
        future = self._create_future_from_job_dir(job_name, is_generator)
        if we_made_it:
            self.logger.info('Created job: %s' % job_name)
            if should_submit:
                jobid = future.submit()
                self.logger.info('Submitted job as %s: %s' % (jobid, job_name))
        else:
            self.logger.info('Job already exists: %s' % job_name)
        return future
Exemplo n.º 2
0
    def _create_future(self, func, args, kwargs, filtered_args_dict,
                       should_submit):
        if not func.version_info['ignore_deps']:
            # TODO: Check a map of possible dependencies executed for
            # this run here, in case a depdency just got changed. This
            # should be done by consulting a pickled on-file database
            # AFAICT.
            raise NotImplementedError('Please use ignore_deps for now')

        # Make job_path containing hashes
        h = NumpyHasher('sha1')
        h.hash((func.version_info['digest'], filtered_args_dict))
        job_name = '%s-%s' % (func.__name__,
                              self._encode_digest(h._hash.digest()))
        # Construct job dir if not existing. Remember that we may
        # race for this; if we got to create the directory, we have
        # the lock.
        we_made_it = self._ensure_job_dir(job_name, func, args, kwargs)
        is_generator = inspect.isgenerator(
            func) or inspect.isgeneratorfunction(func)
        future = self._create_future_from_job_dir(job_name, is_generator)
        if we_made_it:
            self.logger.info('Created job: %s' % job_name)
            if should_submit:
                jobid = future.submit()
                self.logger.info('Submitted job as %s: %s' % (jobid, job_name))
        else:
            self.logger.info('Job already exists: %s' % job_name)
        return future
Exemplo n.º 3
0
def test_getbuffer_not_called_on_const_ndarray_in_hash(mocker):
    arr = np.array([0, 1, 2])
    arr = const_ndarray(arr)

    hasher = NumpyHasher(hash_name='md5', coerce_mmap=False)
    mock__getbuffer = mocker.patch.object(hasher, '_getbuffer')
    hasher.hash(arr)
    assert len(mock__getbuffer.call_args_list) == 0
Exemplo n.º 4
0
    def get_job(self, func, args_dict):
        if not hasattr(func, "version_info"):
            raise ValueError("func does not have @versioned decorator")

        func_hash = encode_digest(func.version_info["digest"])

        fplst, name = get_func_name(func)
        fplst.append(name)
        func_path = pjoin(*fplst)

        h = NumpyHasher("sha1")
        h.hash((func.version_info["digest"], args_dict))
        job_hash = encode_digest(h._hash.digest())

        return ClusterJob(self, func, func_path, func_hash, job_hash)
Exemplo n.º 5
0
    def get_job(self, func, args_dict):
        if not hasattr(func, 'version_info'):
            raise ValueError('func does not have @versioned decorator')

        func_hash = encode_digest(func.version_info['digest'])

        fplst, name = get_func_name(func)
        fplst.append(name)
        func_path = pjoin(*fplst)

        h = NumpyHasher('sha1')
        h.hash((func.version_info['digest'], args_dict))
        job_hash = encode_digest(h._hash.digest())

        return ClusterJob(self, func, func_path, func_hash, job_hash)