def _create_future(self, func, args, kwargs, filtered_args_dict, should_submit): if not func.version_info['ignore_deps']: # TODO: Check a map of possible dependencies executed for # this run here, in case a depdency just got changed. This # should be done by consulting a pickled on-file database # AFAICT. raise NotImplementedError('Please use ignore_deps for now') # Make job_path containing hashes h = NumpyHasher('sha1') h.hash((func.version_info['digest'], filtered_args_dict)) job_name = '%s-%s' % (func.__name__, self._encode_digest(h._hash.digest())) # Construct job dir if not existing. Remember that we may # race for this; if we got to create the directory, we have # the lock. we_made_it = self._ensure_job_dir(job_name, func, args, kwargs) is_generator = inspect.isgenerator(func) or inspect.isgeneratorfunction(func) future = self._create_future_from_job_dir(job_name, is_generator) if we_made_it: self.logger.info('Created job: %s' % job_name) if should_submit: jobid = future.submit() self.logger.info('Submitted job as %s: %s' % (jobid, job_name)) else: self.logger.info('Job already exists: %s' % job_name) return future
def _create_future(self, func, args, kwargs, filtered_args_dict, should_submit): if not func.version_info['ignore_deps']: # TODO: Check a map of possible dependencies executed for # this run here, in case a depdency just got changed. This # should be done by consulting a pickled on-file database # AFAICT. raise NotImplementedError('Please use ignore_deps for now') # Make job_path containing hashes h = NumpyHasher('sha1') h.hash((func.version_info['digest'], filtered_args_dict)) job_name = '%s-%s' % (func.__name__, self._encode_digest(h._hash.digest())) # Construct job dir if not existing. Remember that we may # race for this; if we got to create the directory, we have # the lock. we_made_it = self._ensure_job_dir(job_name, func, args, kwargs) is_generator = inspect.isgenerator( func) or inspect.isgeneratorfunction(func) future = self._create_future_from_job_dir(job_name, is_generator) if we_made_it: self.logger.info('Created job: %s' % job_name) if should_submit: jobid = future.submit() self.logger.info('Submitted job as %s: %s' % (jobid, job_name)) else: self.logger.info('Job already exists: %s' % job_name) return future
def test_getbuffer_not_called_on_const_ndarray_in_hash(mocker): arr = np.array([0, 1, 2]) arr = const_ndarray(arr) hasher = NumpyHasher(hash_name='md5', coerce_mmap=False) mock__getbuffer = mocker.patch.object(hasher, '_getbuffer') hasher.hash(arr) assert len(mock__getbuffer.call_args_list) == 0
def get_job(self, func, args_dict): if not hasattr(func, "version_info"): raise ValueError("func does not have @versioned decorator") func_hash = encode_digest(func.version_info["digest"]) fplst, name = get_func_name(func) fplst.append(name) func_path = pjoin(*fplst) h = NumpyHasher("sha1") h.hash((func.version_info["digest"], args_dict)) job_hash = encode_digest(h._hash.digest()) return ClusterJob(self, func, func_path, func_hash, job_hash)
def get_job(self, func, args_dict): if not hasattr(func, 'version_info'): raise ValueError('func does not have @versioned decorator') func_hash = encode_digest(func.version_info['digest']) fplst, name = get_func_name(func) fplst.append(name) func_path = pjoin(*fplst) h = NumpyHasher('sha1') h.hash((func.version_info['digest'], args_dict)) job_hash = encode_digest(h._hash.digest()) return ClusterJob(self, func, func_path, func_hash, job_hash)