def _create_future(self, func, args, kwargs, filtered_args_dict, should_submit):
if not func.version_info['ignore_deps']:
# TODO: Check a map of possible dependencies executed for
# this run here, in case a depdency just got changed. This
# should be done by consulting a pickled on-file database
# AFAICT.
raise NotImplementedError('Please use ignore_deps for now')
# Make job_path containing hashes
h = NumpyHasher('sha1')
h.hash((func.version_info['digest'], filtered_args_dict))
job_name = '%s-%s' % (func.__name__,
self._encode_digest(h._hash.digest()))
# Construct job dir if not existing. Remember that we may
# race for this; if we got to create the directory, we have
# the lock.
we_made_it = self._ensure_job_dir(job_name, func, args, kwargs)
is_generator = inspect.isgenerator(func) or inspect.isgeneratorfunction(func)
future = self._create_future_from_job_dir(job_name, is_generator)
if we_made_it:
self.logger.info('Created job: %s' % job_name)
if should_submit:
jobid = future.submit()
self.logger.info('Submitted job as %s: %s' % (jobid, job_name))
else:
self.logger.info('Job already exists: %s' % job_name)
return future
def _create_future(self, func, args, kwargs, filtered_args_dict,
should_submit):
if not func.version_info['ignore_deps']:
# TODO: Check a map of possible dependencies executed for
# this run here, in case a depdency just got changed. This
# should be done by consulting a pickled on-file database
# AFAICT.
raise NotImplementedError('Please use ignore_deps for now')
# Make job_path containing hashes
h = NumpyHasher('sha1')
h.hash((func.version_info['digest'], filtered_args_dict))
job_name = '%s-%s' % (func.__name__,
self._encode_digest(h._hash.digest()))
# Construct job dir if not existing. Remember that we may
# race for this; if we got to create the directory, we have
# the lock.
we_made_it = self._ensure_job_dir(job_name, func, args, kwargs)
is_generator = inspect.isgenerator(
func) or inspect.isgeneratorfunction(func)
future = self._create_future_from_job_dir(job_name, is_generator)
if we_made_it:
self.logger.info('Created job: %s' % job_name)
if should_submit:
jobid = future.submit()
self.logger.info('Submitted job as %s: %s' % (jobid, job_name))
else:
self.logger.info('Job already exists: %s' % job_name)
return future
def test_getbuffer_not_called_on_const_ndarray_in_hash(mocker):
arr = np.array([0, 1, 2])
arr = const_ndarray(arr)
hasher = NumpyHasher(hash_name='md5', coerce_mmap=False)
mock__getbuffer = mocker.patch.object(hasher, '_getbuffer')
hasher.hash(arr)
assert len(mock__getbuffer.call_args_list) == 0
def get_job(self, func, args_dict):
if not hasattr(func, "version_info"):
raise ValueError("func does not have @versioned decorator")
func_hash = encode_digest(func.version_info["digest"])
fplst, name = get_func_name(func)
fplst.append(name)
func_path = pjoin(*fplst)
h = NumpyHasher("sha1")
h.hash((func.version_info["digest"], args_dict))
job_hash = encode_digest(h._hash.digest())
return ClusterJob(self, func, func_path, func_hash, job_hash)
def get_job(self, func, args_dict):
if not hasattr(func, 'version_info'):
raise ValueError('func does not have @versioned decorator')
func_hash = encode_digest(func.version_info['digest'])
fplst, name = get_func_name(func)
fplst.append(name)
func_path = pjoin(*fplst)
h = NumpyHasher('sha1')
h.hash((func.version_info['digest'], args_dict))
job_hash = encode_digest(h._hash.digest())
return ClusterJob(self, func, func_path, func_hash, job_hash)