def process_images(random_line):
conflict = 0
tic = time.clock()
filepath = str(random_line).replace(
'/home/gaobing/projects/rpp-ycoady/spectral', '/spectral', 1)
#filename = line.rsplit('/',1)[1].strip()
filename = filepath.strip()
# level2 filename determined from level1 name, if outdir is not provided it will go to the same folder as level1
# for fn in glob(filepath+pattern ,recursive=True):
print("new file name is ", filename)
lock = SoftFileLock(prcessing_lockname, timeout=45)
with lock:
try:
with lock.acquire(timeout=35):
with open(prcessing_fn, 'a') as outfile:
outfile.write(random_line)
except Timeout:
print("timed out - prcessing_fn")
time.sleep(5)
try:
user_algorithm(filename)
except OutputExists:
print("found this one exists in destination")
conflict = conflict + 1
return
toc = time.clock()
print('Processing time for images', random_line, ': (seconds)')
print(toc - tic)
print('conflict number ', conflict)
def add(self, dose_file):
lockfile = dose_file + ".lock"
dose = None
n_primaries = 0
t0 = datetime.now()
logger.debug("lockfile exists" if os.path.
exists(lockfile) else "lockfile does not exist")
lock = SoftFileLock(lockfile)
try:
# TODO: the length of the timeout should maybe be configured in the system configuration
with lock.acquire(timeout=3):
t1 = datetime.now()
logger.debug("acquiring lock file took {} seconds".format(
(t1 - t0).total_seconds()))
##########################
n_primaries = self.get_nprimaries(dose_file)
if n_primaries < 1:
logger.warn(
f"dose file seems to be based on too few primaries ({n_primaries})"
)
elif bool(self.mass) and bool(self.mask):
simdose = itk.imread(dose_file)
logger.debug(
"resampling dose with size {} using mass file of size {} to target size {}"
.format(itk.size(simdose), itk.size(self.mass),
itk.size(self.mask)))
dose = mass_weighted_resampling(simdose, self.mass,
self.mask)
del simdose
else:
dose = itk.imread(dose_file)
logger.debug("read dose with size {}".format(
itk.size(dose)))
t2 = datetime.now()
logger.debug(
"acquiring dose data {} file took {} seconds".format(
os.path.basename(dose_file),
(t2 - t1).total_seconds()))
if self.wmin > n_primaries:
self.wmin = n_primaries
if self.wmax < n_primaries:
self.wmax = n_primaries
except Timeout:
logger.warn(
"failed to acquire lock for {} for 3 seconds, giving up for now"
.format(dose_file))
return
if not bool(dose):
logger.warn("skipping {}".format(dose_file))
return
adose = itk.array_from_image(dose)
if adose.shape != self.dosesum.shape:
raise RuntimeError(
"PROGRAMMING ERROR: dose shape {} differs from expected shape {}"
.format(adose.shape, self.dosesum.shape))
self.dosesum += adose # n_primaries * (adose / n_primaries)
self.dose2sum += adose**2 / n_primaries # n_primaries * (adose / n_primaries)**2
self.weightsum += n_primaries
self.n += 1
def __init__(self, path, acquire_timeout=60*60, lock_timeout=15, expire=120):
self.path = path
self.mlock_path = self.path + '.2'
self.expire = expire
self.lock_timeout = lock_timeout
self.acquire_timeout = acquire_timeout
self.primary_locker = SoftFileLock(self.path, self.lock_timeout)
def clean_up(random_line):
lock = SoftFileLock(completed_lockname, timeout=35)
with lock:
try:
with lock.acquire(timeout=30):
print("lock acquired - complete_file")
with open(completed_fn, 'a') as outfile:
outfile.write(random_line)
except Timeout:
print("timed out - complete_file")
time.sleep(5)
def update_archival(outpath):
#add information to "package slip" meta meta data
infofile = os.path.join(outpath, "package_slip.json")
infofile_lock = os.path.join(outpath, "package_slip.json.lock")
lock_timeout = 5
lock = SoftFileLock(infofile_lock)
try:
with lock.acquire(timeout=lock_timeout):
if os.path.isfile(infofile):
# file exists, needs update
help.status_note(['Going to update ', infofile], d=is_debug)
with open(infofile, encoding='utf-8') as data_file:
data = json.load(data_file)
found = False
for key in data['standards_used']:
if key == archival_info['standards_used'][0]:
found = True
if not found:
data['standards_used'].append(
archival_info['standards_used'][0])
else:
# file does not exist, needs to be created
data = archival_info
# in any case: write current data back to info file
help.status_note(['Write info to ', infofile], d=is_debug)
with open(infofile, 'w', encoding='utf-8') as outfile:
# for json:
output_data = json.dumps(data,
sort_keys=True,
indent=4,
separators=(',', ': '))
outfile.write(str(output_data))
help.status_note([
str(os.stat(infofile).st_size), ' bytes written to ',
os.path.abspath(infofile)
],
d=is_debug)
except Timeout:
help.status_note([
'Cannot acquire lock within ', lock_timeout,
' seconds, raising exception!'
],
d=is_debug)
raise
except Exception as exc:
help.status_note(str(exc), d=is_debug)
raise
finally:
lock.release()
async def __aenter__(self):
lock = SoftFileLock(self.lockfile)
acquired_lock = False
while not acquired_lock:
try:
lock.acquire(timeout=0)
acquired_lock = True
except Timeout:
await asyncio.sleep(self.timeout)
if self.timeout <= 2:
self.timeout = self.timeout * 2
self.lock = lock
return self
def save_data(self, key, value):
"""Store config flie"""
datadict = {}
if os.path.exists(self.data_file):
with SoftFileLock("%s.lock" % self.data_file):
with open(self.data_file, "rt") as file:
datadict = load(file)
else:
dirname = os.path.dirname(self.data_file)
if not os.path.exists(dirname):
mkdir_p(dirname)
with SoftFileLock("%s.lock" % self.data_file):
with open(self.data_file, "wt") as file:
datadict[key] = value
dump(datadict, file)
def __init__(self,
path,
acquire_timeout=4 * 60 * 60,
lock_timeout=15,
expire=120,
keep_alive_handler=None):
self.path = path
self.mlock_path = self.path + '.2'
self.expire = expire
self.lock_timeout = lock_timeout
self.acquire_timeout = acquire_timeout
self.primary_locker = SoftFileLock(self.path, self.lock_timeout)
self.keep_alive_handler = keep_alive_handler
def get_lock(path: str, timeout: int = 600):
os.makedirs(os.path.dirname(path), exist_ok=True)
lock_path = path + '.lock'
logger.info('trying to acquire lock "%s"', lock_path)
return SoftFileLock(lock_path, timeout=timeout)
def write_dsn(self, wdmpath, dsn, data):
"""Write to self.wdmfp/dsn the time-series data."""
dsn_desc = self.describe_dsn(wdmpath, dsn)
tcode = dsn_desc["tcode"]
tstep = dsn_desc["tstep"]
tsfill = dsn_desc["tsfill"]
data.fillna(tsfill, inplace=True)
start_date = data.index[0]
dstart_date = start_date.timetuple()[:6]
llsdat = self._tcode_date(tcode, dstart_date)
if dsn_desc["base_year"] > llsdat[0]:
raise ValueError(
tsutils.error_wrapper("""
The base year for this DSN is {0}. All data to insert must be after the
base year. Instead the first year of the series is {1}.
""".format(dsn_desc["base_year"], llsdat[0])))
nval = len(data)
lock = SoftFileLock(wdmpath + ".lock", timeout=30)
with lock:
wdmfp = self._open(wdmpath, 58)
retcode = self.wdtput(wdmfp, dsn, tstep, llsdat, nval, 1, 0, tcode,
data)
self._close(wdmpath)
self._retcode_check(retcode,
additional_info="wdtput file={0} DSN={1}".format(
wdmpath, dsn))
def _run_task(task_class, config, logger, metadata):
config.set_logger(logger.get_with_module('config'))
config.iteration_cursor = metadata["finished_iterations"]
task = task_class(config, logger.get_with_module('task'), metadata)
metadata_lock = SoftFileLock(metadata["task_dir"] / "metadata.json.lock")
def save_func(finished_iterations):
with metadata_lock:
task.save(metadata["task_dir"])
with open(str(metadata["task_dir"] / Path("metadata.json")), 'r') as handle:
data = json.load(handle)
with open(str(metadata["task_dir"] / Path("metadata.json")), 'w') as handle:
data['saved_time'] = time.mktime(datetime.datetime.now().timetuple())
data['finished_iterations'] = finished_iterations
json.dump(data, handle)
metadata["pipe"].send(PipeMsg.SAVED_FINISHED_ITERATIONS, {"saved_finished_iterations": finished_iterations, "saved_time": data['saved_time']})
def checkpoint_func(finished_iterations):
save_func(finished_iterations)
checkpoint = TaskWrapper._create_checkpoint(metadata_lock, metadata["task_dir"], finished_iterations)
return checkpoint
if metadata["finished_iterations"] > 0:
task.load(metadata["task_dir"])
task.run(save_func, checkpoint_func)
save_func(task.finished_iterations)
def save(task_path: Path, result=None, task=None, name_prefix=None):
"""
Save a :class:`~pydra.engine.core.TaskBase` object and/or results.
Parameters
----------
task_path : :obj:`Path`
Write directory
result : :obj:`Result`
Result to pickle and write
task : :class:`~pydra.engine.core.TaskBase`
Task to pickle and write
"""
if task is None and result is None:
raise ValueError("Nothing to be saved")
if not isinstance(task_path, Path):
task_path = Path(task_path)
task_path.mkdir(parents=True, exist_ok=True)
if name_prefix is None:
name_prefix = ""
lockfile = task_path.parent / (task_path.name + "_save.lock")
with SoftFileLock(lockfile):
if result:
if task_path.name.startswith(
"Workflow") and result.output is not None:
# copy files to the workflow directory
result = copyfile_workflow(wf_path=task_path, result=result)
with (task_path / f"{name_prefix}_result.pklz").open("wb") as fp:
cp.dump(result, fp)
if task:
with (task_path / f"{name_prefix}_task.pklz").open("wb") as fp:
cp.dump(task, fp)
async def _run(self, submitter=None, rerun=False, **kwargs):
# self.inputs = dc.replace(self.inputs, **kwargs) don't need it?
# output_spec needs to be set using set_output or at workflow initialization
if self.output_spec is None:
raise ValueError(
"Workflow output cannot be None, use set_output to define output(s)"
)
checksum = self.checksum
# Eagerly retrieve cached
if not (rerun or self.task_rerun):
result = self.result()
if result is not None:
return result
# creating connections that were defined after adding tasks to the wf
for task in self.graph.nodes:
# if workflow has task_rerun=True and propagate_rerun=True,
# it should be passed to the tasks
if self.task_rerun and self.propagate_rerun:
task.task_rerun = self.task_rerun
# if the task is a wf, than the propagate_rerun should be also set
if is_workflow(task):
task.propagate_rerun = self.propagate_rerun
task.cache_locations = task._cache_locations + self.cache_locations
self.create_connections(task)
# TODO add signal handler for processes killed after lock acquisition
# adding info file with the checksum in case the task was cancelled
# and the lockfile has to be removed
with open(self.cache_dir / f"{self.uid}_info.json", "w") as jsonfile:
json.dump({"checksum": checksum}, jsonfile)
lockfile = self.cache_dir / (checksum + ".lock")
self.hooks.pre_run(self)
with SoftFileLock(lockfile):
# # Let only one equivalent process run
odir = self.output_dir
if not self.can_resume and odir.exists():
shutil.rmtree(odir)
cwd = os.getcwd()
odir.mkdir(parents=False, exist_ok=True if self.can_resume else False)
self.audit.start_audit(odir=odir)
result = Result(output=None, runtime=None, errored=False)
self.hooks.pre_run_task(self)
try:
self.audit.monitor()
await self._run_task(submitter, rerun=rerun)
result.output = self._collect_outputs()
except Exception as e:
record_error(self.output_dir, e)
result.errored = True
self._errored = True
raise
finally:
self.hooks.post_run_task(self, result)
self.audit.finalize_audit(result=result)
save(odir, result=result, task=self)
# removing the additional file with the chcksum
(self.cache_dir / f"{self.uid}_info.json").unlink()
os.chdir(cwd)
self.hooks.post_run(self, result)
return result
def test_cleanup_soft_lock(tmp_path: Path) -> None:
# tests if the lock file is removed after use
lock_path = tmp_path / "a"
lock = SoftFileLock(str(lock_path))
with lock:
assert lock_path.exists()
assert not lock_path.exists()
def _run(self, rerun=False, **kwargs):
self.inputs = attr.evolve(self.inputs, **kwargs)
self.inputs.check_fields_input_spec()
checksum = self.checksum
lockfile = self.cache_dir / (checksum + ".lock")
# Eagerly retrieve cached - see scenarios in __init__()
self.hooks.pre_run(self)
with SoftFileLock(lockfile):
if not (rerun or self.task_rerun):
result = self.result()
if result is not None:
return result
# adding info file with the checksum in case the task was cancelled
# and the lockfile has to be removed
with open(self.cache_dir / f"{self.uid}_info.json",
"w") as jsonfile:
json.dump({"checksum": self.checksum}, jsonfile)
# Let only one equivalent process run
odir = self.output_dir
if not self.can_resume and odir.exists():
shutil.rmtree(odir)
cwd = os.getcwd()
odir.mkdir(parents=False,
exist_ok=True if self.can_resume else False)
orig_inputs = attr.asdict(self.inputs)
map_copyfiles = copyfile_input(self.inputs, self.output_dir)
modified_inputs = template_update(self.inputs,
self.output_dir,
map_copyfiles=map_copyfiles)
if modified_inputs:
self.inputs = attr.evolve(self.inputs, **modified_inputs)
self.audit.start_audit(odir)
result = Result(output=None, runtime=None, errored=False)
self.hooks.pre_run_task(self)
try:
self.audit.monitor()
self._run_task()
result.output = self._collect_outputs(output_dir=odir)
except Exception as e:
etype, eval, etr = sys.exc_info()
traceback = format_exception(etype, eval, etr)
record_error(self.output_dir, error=traceback)
result.errored = True
raise
finally:
self.hooks.post_run_task(self, result)
self.audit.finalize_audit(result)
save(odir, result=result, task=self)
self.output_ = None
# removing the additional file with the chcksum
(self.cache_dir / f"{self.uid}_info.json").unlink()
# # function etc. shouldn't change anyway, so removing
orig_inputs = dict((k, v) for (k, v) in orig_inputs.items()
if not k.startswith("_"))
self.inputs = attr.evolve(self.inputs, **orig_inputs)
os.chdir(cwd)
self.hooks.post_run(self, result)
return result
def __init__(self, storage_path):
self.storage_path = storage_path
# check path existante
assert os.path.exists(
self.full_path(COUNTER_DAT)
), "Storage path access error. Missing %s !" % self.full_path(
COUNTER_DAT)
# Lock files
self.lock = SoftFileLock(self.full_path(LOCK_FILE))
def get_data(self, key):
"""Read options file"""
if not os.path.exists(self.data_file):
return None
with SoftFileLock("%s.lock" % self.data_file):
with open(self.data_file, "rt") as file:
datadict = load(file)
if key in datadict:
return datadict[key]
return None
def _run_run(run):
"""Actually run a run."""
res = _execute(run.command)
# check return codes
if (res.returncode not in run.allowed_return_codes
and run.allowed_return_codes != []):
_print_warning("unexpected return code for command: " + res.args +
"\n" + res.stderr.strip() + "")
return
if run.stdout_file is None:
return
filename = run.stdout_file
lock = filename + ".lock"
os.makedirs(os.path.dirname(filename), exist_ok=True)
# create new file with header
if not os.path.isfile(filename) and (run.header_command is not None
or run.header_string is not None):
header = run.header_string
if run.header_command is not None:
header = _execute(run.header_command).stdout.strip()
header = run.header_mod(header)
with SoftFileLock(lock):
if not os.path.isfile(filename):
with open(filename, "w") as out:
print(header, file=out, flush=True)
# write to stdout
stdout = res.stdout.strip()
mod = run.stdout_mod
output = mod(stdout) if len(signature(mod).parameters) == 1 else mod(
stdout, res)
if run.stdout_res is not None:
run.args["stdout"] = output
output = deblob(run.stdout_res, run.args)
with SoftFileLock(run.stdout_file + ".lock"):
with open(run.stdout_file, "a") as out:
print(output, file=out, flush=True)
def _do_action_under_lock_internal(lock_file,
func,
*args,
timeout=LOCK_TIMEOUT,
**kwargs):
# Using this instead of FileLock because it will be used across nodes
# on the Lustre filesystem.
lock_acquisition_seconds = None
lock = SoftFileLock(lock_file, timeout=timeout)
try:
start = time.time()
lock.acquire(timeout=timeout)
lock_acquisition_seconds = time.time() - start
except Timeout:
# Picked a default value such that this should not trip. If it does
# trip under normal circumstances then we need to reconsider this.
logger.error(
"Failed to acquire file lock %s within %s seconds. hostname=%s",
lock_file,
timeout,
socket.gethostname(),
)
raise
if lock_acquisition_seconds > 5:
logger.warning(
"cluster lock acquisition for %s took %s seconds",
func.__name__,
lock_acquisition_seconds,
)
try:
val = func(*args, **kwargs)
lock.release()
return val
except Exception as exc:
lock.release()
# SoftFileLock always deletes the file, so create it again.
# There is a small window where this won't work and another node takes over, so
# callers need to handle the possibility.
logger.exception(
"An exception occurred while holding the Cluster lock. The state of the cluster "
"is unknown.")
try:
fd = os.open(lock_file,
os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_TRUNC)
os.close(fd)
logger.error("A deadlock will occur.")
except (IOError, OSError):
logger.exception(
"Unable to cause deadlock. Another node was promoted to submitter"
)
raise exc
def _lock():
"""Return a context manager for acquiring ``_lock_filename()``.
For example::
with BatteryController._lock():
status = BatteryController._read_state()
# --- Mutate status here ---
BatteryController._poll(status)
"""
return SoftFileLock(BatteryController._lock_filename(), timeout=30)
def loadpkl(infile):
"""Load a zipped or plain cPickled file."""
infile = Path(infile)
fmlogger.debug('Loading pkl: %s', infile)
pklopen = gzip.open if infile.suffix == '.pklz' else open
with SoftFileLock('%s.lock' % infile):
with pklopen(str(infile), 'rb') as pkl_file:
pkl_contents = pkl_file.read()
pkl_metadata = None
# Look if pkl file contains version metadata
idx = pkl_contents.find(b'\n')
if idx >= 0:
try:
pkl_metadata = json.loads(pkl_contents[:idx])
except (UnicodeDecodeError, json.JSONDecodeError):
# Could not get version info
pass
else:
# On success, skip JSON metadata
pkl_contents = pkl_contents[idx + 1:]
# Pickle files may contain relative paths that must be resolved relative
# to the working directory, so use indirectory while attempting to load
unpkl = None
try:
with indirectory(infile.parent):
unpkl = pickle.loads(pkl_contents)
except UnicodeDecodeError:
# Was this pickle created with Python 2.x?
with indirectory(infile.parent):
unpkl = pickle.loads(pkl_contents, fix_imports=True, encoding='utf-8')
fmlogger.info('Successfully loaded pkl in compatibility mode.')
# Unpickling problems
except Exception as e:
if pkl_metadata and 'version' in pkl_metadata:
from nipype import __version__ as version
if pkl_metadata['version'] != version:
fmlogger.error("""\
Attempted to open a results file generated by Nipype version %s, \
with an incompatible Nipype version (%s)""", pkl_metadata['version'], version)
raise e
fmlogger.warning("""\
No metadata was found in the pkl file. Make sure you are currently using \
the same Nipype version from the generated pkl.""")
raise e
if unpkl is None:
raise ValueError('Loading %s resulted in None.' % infile)
return unpkl
def savepkl(filename, record, versioning=False):
pklopen = gzip.open if filename.endswith('.pklz') else open
with SoftFileLock('%s.lock' % filename):
with pklopen(filename, 'wb') as pkl_file:
if versioning:
from nipype import __version__ as version
metadata = json.dumps({'version': version})
pkl_file.write(metadata.encode('utf-8'))
pkl_file.write('\n'.encode('utf-8'))
pickle.dump(record, pkl_file)
def update_archival(outpath):
#add information to "package slip" meta meta data
infofile = os.path.join(outpath, "package_slip.json")
infofile_lock = os.path.join(outpath, "package_slip.json.lock")
lock_timeout = 5
lock = SoftFileLock(infofile_lock)
try:
with lock.acquire(timeout=lock_timeout):
if os.path.isfile(infofile):
# file exists, needs update
status_note(['Going to update ', infofile], d=is_debug)
with open(infofile, encoding='utf-8') as data_file:
data = json.load(data_file)
found = False
for key in data['standards_used']:
if key == archival_info['standards_used'][0]:
found = True
if not found:
data['standards_used'].append(archival_info['standards_used'][0])
else:
# file does not exist, needs to be created
data = archival_info
# in any case: write current data back to info file
status_note(['Write info to ', infofile], d=is_debug)
with open(infofile, 'w', encoding='utf-8') as outfile:
# for json:
output_data = json.dumps(data, sort_keys=True, indent=4, separators=(',', ': '))
outfile.write(str(output_data))
status_note([str(os.stat(infofile).st_size), ' bytes written to ', os.path.abspath(infofile)], d=is_debug)
except Timeout:
status_note(['Cannot acquire lock within ', lock_timeout, ' seconds, raising exception!'], d=is_debug)
raise
except Exception as exc:
status_note(str(exc), d=is_debug)
raise
finally:
lock.release()
def spool_lock(spoolpath):
lock = SoftFileLock(join(spoolpath, 'lock'))
lock.acquire()
try:
yield lock
finally:
lock.release()
def renumber_dsn(self, wdmpath, odsn, ndsn):
"""Will renumber the odsn to the ndsn."""
odsn = int(odsn)
ndsn = int(ndsn)
lock = SoftFileLock(wdmpath + ".lock", timeout=30)
with lock:
wdmfp = self._open(wdmpath, 51)
retcode = self.wddsrn(wdmfp, odsn, ndsn)
self._close(wdmpath)
self._retcode_check(retcode,
additional_info="wddsrn file={0} DSN={1}".format(
wdmpath, odsn))
def locked_call(callable: Callable[[], Any], lock_file: str, lock_timeout: int) -> Any:
"""Locks a callable execution with a given timout and a specified lock file.
Parameters
----------
callable : Callable[[], Any]
Whatever should be executed thread safe in a multi host environment.
Returns
-------
Any
The result of the callable
Raises
------
Timeout
If the locking times out.
"""
lock = SoftFileLock(lock_file, timeout=lock_timeout)
with lock.acquire(timeout=lock_timeout):
return callable()
def _optimistic_lock():
"""Return a context manager for acquiring ``_lock_filename()``.
If the file is locked, we raise an exception rather than
waiting.
For example::
with BatteryController._optimistic_lock():
status = BatteryController._read_state()
# --- Mutate status here ---
BatteryController._poll(status)
"""
return SoftFileLock(BatteryController._lock_filename())
async def _run(self, submitter=None, **kwargs):
# self.inputs = dc.replace(self.inputs, **kwargs) don't need it?
checksum = self.checksum
lockfile = self.cache_dir / (checksum + ".lock")
# Eagerly retrieve cached
result = self.result()
if result is not None:
return result
# creating connections that were defined after adding tasks to the wf
for task in self.graph.nodes:
self.create_connections(task)
"""
Concurrent execution scenarios
1. prior cache exists -> return result
2. other process running -> wait
a. finishes (with or without exception) -> return result
b. gets killed -> restart
3. no cache or other process -> start
4. two or more concurrent new processes get to start
"""
# TODO add signal handler for processes killed after lock acquisition
self.hooks.pre_run(self)
with SoftFileLock(lockfile):
# # Let only one equivalent process run
odir = self.output_dir
if not self.can_resume and odir.exists():
shutil.rmtree(odir)
cwd = os.getcwd()
odir.mkdir(parents=False,
exist_ok=True if self.can_resume else False)
self.audit.start_audit(odir=odir)
result = Result(output=None, runtime=None, errored=False)
self.hooks.pre_run_task(self)
try:
self.audit.monitor()
await self._run_task(submitter)
result.output = self._collect_outputs()
except Exception as e:
record_error(self.output_dir, e)
result.errored = True
raise
finally:
self.hooks.post_run_task(self, result)
self.audit.finalize_audit(result=result)
save(odir, result=result, task=self)
os.chdir(cwd)
self.hooks.post_run(self, result)
return result
def _run(self, rerun=False, **kwargs):
self.inputs = attr.evolve(self.inputs, **kwargs)
self.inputs.check_fields_input_spec()
checksum = self.checksum
lockfile = self.cache_dir / (checksum + ".lock")
# Eagerly retrieve cached - see scenarios in __init__()
self.hooks.pre_run(self)
# TODO add signal handler for processes killed after lock acquisition
with SoftFileLock(lockfile):
if not (rerun or self.task_rerun):
result = self.result()
if result is not None:
return result
# Let only one equivalent process run
odir = self.output_dir
if not self.can_resume and odir.exists():
shutil.rmtree(odir)
cwd = os.getcwd()
odir.mkdir(parents=False,
exist_ok=True if self.can_resume else False)
orig_inputs = attr.asdict(self.inputs)
map_copyfiles = copyfile_input(self.inputs, self.output_dir)
modified_inputs = template_update(self.inputs,
self.output_dir,
map_copyfiles=map_copyfiles)
if modified_inputs:
self.inputs = attr.evolve(self.inputs, **modified_inputs)
self.audit.start_audit(odir)
result = Result(output=None, runtime=None, errored=False)
self.hooks.pre_run_task(self)
try:
self.audit.monitor()
self._run_task()
result.output = self._collect_outputs()
except Exception as e:
record_error(self.output_dir, e)
result.errored = True
raise
finally:
self.hooks.post_run_task(self, result)
self.audit.finalize_audit(result)
save(odir, result=result, task=self)
# # function etc. shouldn't change anyway, so removing
orig_inputs = dict((k, v) for (k, v) in orig_inputs.items()
if not k.startswith("_"))
self.inputs = attr.evolve(self.inputs, **orig_inputs)
os.chdir(cwd)
self.hooks.post_run(self, result)
return result
def _create_lock(path):
"""Create lock based on file system capabilities
Determine if we can rely on the fcntl module for locking files.
Otherwise, fallback on using the directory creation atomicity as a locking mechanism.
"""
file_system = _get_fs(path)
if _fs_support_globalflock(file_system):
log.debug("Using flock.")
return FileLock(path)
else:
log.debug(
"Cluster does not support flock. Falling back to softfilelock.")
return SoftFileLock(path)
def lock(self, key):
lockdir = join(self.base, 'locks')
lockfname = join(lockdir, self.hash(key) + '.lock')
self.pdebug('LOCK - ' + key + '(' + lockfname + ')')
if not exists(lockdir):
try:
makedirs(lockdir)
except:
pass
print(f'locking {key}', flush=True)
return SoftFileLock(lockfname)