def wrapper():
if LOGIN_ACTION_NAME in interaction:
globals.login_failed = False
if globals.login_failed:
pytest.skip(f"login is failed")
start = time.time()
error_msg = 'Success'
full_exception = ''
try:
func()
success = True
except Exception:
success = False
# https://docs.python.org/2/library/sys.html#sys.exc_info
exc_type, full_exception = sys.exc_info()[:2]
error_msg = f"Failed measure: {interaction} - {exc_type.__name__}"
end = time.time()
timing = str(int((end - start) * 1000))
lockfile = f'{selenium_results_file}.lock'
with filelock.SoftFileLock(lockfile):
with open(selenium_results_file, "a+") as jtl_file:
timestamp = round(time.time() * 1000)
jtl_file.write(
f"{timestamp},{timing},{interaction},,{error_msg},,{success},0,0,0,0,,0\n"
)
print(f"{timestamp},{timing},{interaction},{error_msg},{success}")
if not success:
if LOGIN_ACTION_NAME in interaction:
globals.login_failed = True
raise Exception(error_msg, full_exception)
def merge_tree(src, dst, symlinks=False, timeout=90):
"""
Merge src into dst recursively by copying all files from src into dst.
Return a list of all files copied.
Like copytree(src, dst), but raises an error if merging the two trees
would overwrite any files.
"""
assert src not in dst, ("Can't merge/copy source into subdirectory of itself. Please create "
"separate spaces for these things.")
new_files = copytree(src, dst, symlinks=symlinks, dry_run=True)
# do not copy lock files
new_files = [f for f in new_files if not f.endswith('.conda_lock')]
existing = [f for f in new_files if isfile(f)]
if existing:
raise IOError("Can't merge {0} into {1}: file exists: "
"{2}".format(src, dst, existing[0]))
lock = filelock.SoftFileLock(join(src, ".conda_lock"))
lock.acquire(timeout=timeout)
try:
copytree(src, dst, symlinks=symlinks)
except:
raise
finally:
lock.release()
rm_rf(os.path.join(dst, '.conda_lock'))
def create_new_location(source, upload, move=False):
base_dir = current_app.config['BOOKS_BASE_DIR']
if isinstance(upload, model.Upload):
new_file = os.path.join(current_app.config['UPLOAD_DIR'], upload.file)
else:
new_file = upload
new_location = os.path.join(source.ebook.base_dir,
os.path.basename(norm_file_name(source)))
#if source.ebook.base_dir else norm_file_name(source) #TODO: Remove this WA
ebook_dir = os.path.join(base_dir, os.path.split(new_location)[0])
if not os.path.exists(ebook_dir):
os.makedirs(ebook_dir, exist_ok=True)
lock_file = os.path.join(ebook_dir, '.lock_this_dir')
index = 1
with filelock.SoftFileLock(lock_file, timeout=5):
while os.path.exists(os.path.join(base_dir, new_location)):
name, ext = os.path.splitext(new_location)
new_location = name + '(%d)' % index + ext
index += 1
if move:
shutil.move(new_file, os.path.join(base_dir, new_location))
else:
shutil.copy(new_file, os.path.join(base_dir, new_location))
return new_location
def get_download_lock(lock_dir):
mkdir(lock_dir)
lockfile = os.path.join(lock_dir, 'download.lock')
try:
return filelock.FileLock(lockfile).acquire()
except:
return filelock.SoftFileLock(lockfile).acquire()
def copy_into(src, dst, timeout=90, symlinks=False):
"Copy all the files and directories in src to the directory dst"
lock = None
if isdir(dst):
lock = filelock.SoftFileLock(join(dst, ".conda_lock"))
lock.acquire(timeout=timeout)
try:
if isdir(src):
merge_tree(src, dst, symlinks)
else:
if isdir(dst):
dst_fn = os.path.join(dst, os.path.basename(src))
else:
dst_fn = dst
try:
if os.path.sep in dst_fn and not os.path.isdir(
os.path.dirname(dst_fn)):
os.makedirs(os.path.dirname(dst_fn))
shutil.copy2(src, dst_fn)
except shutil.Error:
log.debug("skipping %s - already exists in %s",
os.path.basename(src), dst)
finally:
if lock:
lock.release()
def copy_into(src, dst, timeout=90, symlinks=False):
"Copy all the files and directories in src to the directory dst"
if isdir(src):
merge_tree(src, dst, symlinks, timeout=timeout)
else:
if isdir(dst):
dst_fn = os.path.join(dst, os.path.basename(src))
else:
dst_fn = dst
lock = None
if os.path.isabs(src):
src_folder = os.path.dirname(src)
lock = filelock.SoftFileLock(join(src_folder, ".conda_lock"))
try:
if os.path.sep in dst_fn and not os.path.isdir(os.path.dirname(dst_fn)):
os.makedirs(os.path.dirname(dst_fn))
if lock:
lock.acquire(timeout=timeout)
# with each of these, we are copying less metadata. This seems to be necessary
# to cope with some shared filesystems with some virtual machine setups.
# See https://github.com/conda/conda-build/issues/1426
try:
shutil.copy2(src, dst_fn)
except OSError:
try:
shutil.copy(src, dst_fn)
except OSError:
shutil.copyfile(src, dst_fn)
except shutil.Error:
log.debug("skipping %s - already exists in %s", os.path.basename(src), dst)
finally:
if lock:
lock.release()
def get_lock(folder, timeout=90, filename=".conda_lock"):
global _locations
location = os.path.abspath(os.path.normpath(folder))
if not os.path.isdir(location):
os.makedirs(location)
if location not in _locations:
_locations[location] = filelock.SoftFileLock(os.path.join(location, filename),
timeout)
return _locations[location]
def clean_pkg_cache(dist, timeout):
cc.pkgs_dirs = cc.pkgs_dirs[:1]
locks = []
for folder in cc.pkgs_dirs:
locks.append(filelock.SoftFileLock(join(folder, ".conda_lock")))
for lock in locks:
lock.acquire(timeout=timeout)
try:
rmplan = [
'RM_EXTRACTED {0} local::{0}'.format(dist),
'RM_FETCHED {0} local::{0}'.format(dist),
]
plan.execute_plan(rmplan)
# Conda does not seem to do a complete cleanup sometimes. This is supplemental.
# Conda's cleanup is still necessary - it keeps track of its own in-memory
# list of downloaded things.
for folder in cc.pkgs_dirs:
try:
assert not os.path.exists(os.path.join(folder, dist))
assert not os.path.exists(
os.path.join(folder, dist + '.tar.bz2'))
for pkg_id in [dist, 'local::' + dist]:
assert pkg_id not in package_cache()
except AssertionError:
log.debug(
"Conda caching error: %s package remains in cache after removal",
dist)
log.debug("Clearing package cache to compensate")
cache = package_cache()
keys = [key for key in cache.keys() if dist in key]
for pkg_id in keys:
if pkg_id in cache:
del cache[pkg_id]
for entry in glob(os.path.join(folder, dist + '*')):
rm_rf(entry)
except:
raise
finally:
for lock in locks:
lock.release()
if os.path.isfile(lock._lock_file):
os.remove(lock._lock_file)
def __init__(self, name: str, path: str = None, logger=None):
if logger is not None:
self._logger = logger
else:
self._logger = logs.get_logger('LocalQueue')
self._logger.setLevel(get_storage_verbose_level())
self.name = name
if path is None:
self.path = self._get_queue_directory()
else:
self.path = path
self.path = os.path.join(self.path, name)
os.makedirs(self.path, exist_ok=True)
# Local queue is considered active, iff its directory exists.
self._lock_path = os.path.join(self.path, LOCK_FILE_NAME)
self._lock = filelock.SoftFileLock(self._lock_path)
def write_repodata(repodata, dir_path, config=None, lock=None):
""" Write updated repodata.json and repodata.json.bz2 """
if not config:
import conda_build.config
config = conda_build.config.config
if not lock:
lock = filelock.SoftFileLock(join(dir_path, ".conda_lock"))
lock.acquire(timeout=config.timeout)
data = json.dumps(repodata, indent=2, sort_keys=True)
# strip trailing whitespace
data = '\n'.join(line.rstrip() for line in data.splitlines())
# make sure we have newline at the end
if not data.endswith('\n'):
data += '\n'
with open(join(dir_path, 'repodata.json'), 'w') as fo:
fo.write(data)
with open(join(dir_path, 'repodata.json.bz2'), 'wb') as fo:
fo.write(bz2.compress(data.encode('utf-8')))
lock.release()
def read_index_tar(tar_path, config, lock=None):
""" Returns the index.json dict inside the given package tarball. """
if not lock:
lock = filelock.SoftFileLock(join(os.path.dirname(tar_path), ".conda_lock"))
lock.acquire(timeout=config.timeout)
try:
with tarfile.open(tar_path) as t:
try:
return json.loads(t.extractfile('info/index.json').read().decode('utf-8'))
except EOFError:
raise RuntimeError("Could not extract %s. File probably corrupt."
% tar_path)
except OSError as e:
raise RuntimeError("Could not extract %s (%s)" % (tar_path, e))
except tarfile.ReadError:
raise RuntimeError("Could not extract metadata from %s. "
"File probably corrupt." % tar_path)
finally:
lock.release()
def test(m, config, move_broken=True):
'''
Execute any test scripts for the given package.
:param m: Package's metadata.
:type m: Metadata
'''
if not os.path.isdir(config.build_folder):
os.makedirs(config.build_folder)
clean_pkg_cache(m.dist(), config.timeout)
with filelock.SoftFileLock(join(config.build_folder, ".conda_lock"),
timeout=config.timeout):
tmp_dir = config.test_dir
if not isdir(tmp_dir):
os.makedirs(tmp_dir)
create_files(tmp_dir, m, config)
# Make Perl or Python-specific test files
if m.name().startswith('perl-'):
pl_files = create_pl_files(tmp_dir, m)
py_files = False
lua_files = False
else:
py_files = create_py_files(tmp_dir, m)
pl_files = False
lua_files = False
shell_files = create_shell_files(tmp_dir, m, config)
if not (py_files or shell_files or pl_files or lua_files):
print("Nothing to test for:", m.dist())
return
print("TEST START:", m.dist())
get_build_metadata(m, config=config)
specs = ['%s %s %s' % (m.name(), m.version(), m.build_id())]
# add packages listed in the run environment and test/requires
specs.extend(ms.spec for ms in m.ms_depends('run'))
specs += m.get_value('test/requires', [])
if py_files:
# as the tests are run by python, ensure that python is installed.
# (If they already provided python as a run or test requirement,
# this won't hurt anything.)
specs += ['python %s*' % environ.get_py_ver(config)]
if pl_files:
# as the tests are run by perl, we need to specify it
specs += ['perl %s*' % environ.get_perl_ver(config)]
if lua_files:
# not sure how this shakes out
specs += ['lua %s*' % environ.get_lua_ver(config)]
create_env(config.test_prefix, specs, config=config)
with path_prepended(config.test_prefix):
env = dict(os.environ.copy())
env.update(
environ.get_dict(config=config, m=m,
prefix=config.test_prefix))
env["CONDA_BUILD_STATE"] = "TEST"
if not config.activate:
# prepend bin (or Scripts) directory
env = prepend_bin_path(env,
config.test_prefix,
prepend_prefix=True)
if on_win:
env['PATH'] = config.test_prefix + os.pathsep + env['PATH']
for varname in 'CONDA_PY', 'CONDA_NPY', 'CONDA_PERL', 'CONDA_LUA':
env[varname] = str(getattr(config, varname) or '')
# Python 2 Windows requires that envs variables be string, not unicode
env = {str(key): str(value) for key, value in env.items()}
suffix = "bat" if on_win else "sh"
test_script = join(tmp_dir,
"conda_test_runner.{suffix}".format(suffix=suffix))
with open(test_script, 'w') as tf:
if config.activate:
ext = ".bat" if on_win else ""
tf.write(
"{source} {conda_root}activate{ext} {test_env} {squelch}\n"
.format(conda_root=root_script_dir + os.path.sep,
source="call" if on_win else "source",
ext=ext,
test_env=config.test_prefix,
squelch=">nul 2>&1" if on_win else "&> /dev/null"))
if on_win:
tf.write("if errorlevel 1 exit 1\n")
if py_files:
tf.write("{python} -s {test_file}\n".format(
python=config.test_python,
test_file=join(tmp_dir, 'run_test.py')))
if on_win:
tf.write("if errorlevel 1 exit 1\n")
if pl_files:
tf.write("{perl} {test_file}\n".format(perl=config.test_perl,
test_file=join(
tmp_dir,
'run_test.pl')))
if on_win:
tf.write("if errorlevel 1 exit 1\n")
if lua_files:
tf.write("{lua} {test_file}\n".format(lua=config.test_lua,
test_file=join(
tmp_dir,
'run_test.lua')))
if on_win:
tf.write("if errorlevel 1 exit 1\n")
if shell_files:
test_file = join(tmp_dir, 'run_test.' + suffix)
if on_win:
tf.write("call {test_file}\n".format(test_file=test_file))
if on_win:
tf.write("if errorlevel 1 exit 1\n")
else:
# TODO: Run the test/commands here instead of in run_test.py
tf.write("{shell_path} -x -e {test_file}\n".format(
shell_path=shell_path, test_file=test_file))
if on_win:
cmd = ['cmd.exe', "/d", "/c", test_script]
else:
cmd = [shell_path, '-x', '-e', test_script]
try:
subprocess.check_call(cmd, env=env, cwd=tmp_dir)
except subprocess.CalledProcessError:
tests_failed(m,
move_broken=move_broken,
broken_dir=config.broken_dir,
config=config)
print("TEST END:", m.dist())
def build(m,
config,
post=None,
need_source_download=True,
need_reparse_in_env=False):
'''
Build the package with the specified metadata.
:param m: Package metadata
:type m: Metadata
:type post: bool or None. None means run the whole build. True means run
post only. False means stop just before the post.
:type keep_old_work: bool: Keep any previous work directory.
:type need_source_download: bool: if rendering failed to download source
(due to missing tools), retry here after build env is populated
'''
if m.skip():
print_skip_message(m)
return False
if config.skip_existing:
package_exists = is_package_built(m, config)
if package_exists:
print(m.dist(),
"is already built in {0}, skipping.".format(package_exists))
return False
if post in [False, None]:
print("BUILD START:", m.dist())
if m.uses_jinja and (need_source_download or need_reparse_in_env):
print(
" (actual version deferred until further download or env creation)"
)
specs = [ms.spec for ms in m.ms_depends('build')]
create_env(config.build_prefix, specs, config=config)
vcs_source = m.uses_vcs_in_build
if vcs_source and vcs_source not in specs:
vcs_executable = "hg" if vcs_source == "mercurial" else vcs_source
has_vcs_available = os.path.isfile(
external.find_executable(vcs_executable, config.build_prefix)
or "")
if not has_vcs_available:
if (vcs_source != "mercurial" or not any(
spec.startswith('python') and "3." in spec
for spec in specs)):
specs.append(vcs_source)
log.warn(
"Your recipe depends on %s at build time (for templates), "
"but you have not listed it as a build dependency. Doing "
"so for this build.", vcs_source)
# Display the name only
# Version number could be missing due to dependency on source info.
create_env(config.build_prefix, specs, config=config)
else:
raise ValueError(
"Your recipe uses mercurial in build, but mercurial"
" does not yet support Python 3. Please handle all of "
"your mercurial actions outside of your build script.")
if need_source_download:
# Execute any commands fetching the source (e.g., git) in the _build environment.
# This makes it possible to provide source fetchers (eg. git, hg, svn) as build
# dependencies.
with path_prepended(config.build_prefix):
m, need_source_download, need_reparse_in_env = parse_or_try_download(
m,
no_download_source=False,
force_download=True,
config=config)
assert not need_source_download, "Source download failed. Please investigate."
if m.uses_jinja:
print("BUILD START (revised):", m.dist())
if need_reparse_in_env:
reparse(m, config=config)
print("BUILD START (revised):", m.dist())
if m.name() in [
i.rsplit('-', 2)[0] for i in linked(config.build_prefix)
]:
print("%s is installed as a build dependency. Removing." %
m.name())
index = get_build_index(config=config, clear_cache=False)
actions = plan.remove_actions(config.build_prefix, [m.name()],
index=index)
assert not plan.nothing_to_do(actions), actions
plan.display_actions(actions, index)
plan.execute_actions(actions, index)
print("Package:", m.dist())
with filelock.SoftFileLock(join(config.build_folder, ".conda_lock"),
timeout=config.timeout):
# get_dir here might be just work, or it might be one level deeper,
# dependening on the source.
src_dir = source.get_dir(config)
if isdir(src_dir):
print("source tree in:", src_dir)
else:
print("no source - creating empty work folder")
os.makedirs(src_dir)
rm_rf(config.info_dir)
files1 = prefix_files(prefix=config.build_prefix)
for pat in m.always_include_files():
has_matches = False
for f in set(files1):
if fnmatch.fnmatch(f, pat):
print("Including in package existing file", f)
files1.discard(f)
has_matches = True
if not has_matches:
log.warn(
"Glob %s from always_include_files does not match any files",
pat)
# Save this for later
with open(join(config.croot, 'prefix_files.txt'), 'w') as f:
f.write(u'\n'.join(sorted(list(files1))))
f.write(u'\n')
# Use script from recipe?
script = m.get_value('build/script', None)
if script:
if isinstance(script, list):
script = '\n'.join(script)
if isdir(src_dir):
if on_win:
build_file = join(m.path, 'bld.bat')
if script:
build_file = join(src_dir, 'bld.bat')
with open(build_file, 'w') as bf:
bf.write(script)
import conda_build.windows as windows
windows.build(m, build_file, config=config)
else:
build_file = join(m.path, 'build.sh')
# There is no sense in trying to run an empty build script.
if isfile(build_file) or script:
with path_prepended(config.build_prefix):
env = environ.get_dict(config=config, m=m)
env["CONDA_BUILD_STATE"] = "BUILD"
work_file = join(source.get_dir(config),
'conda_build.sh')
if script:
with open(work_file, 'w') as bf:
bf.write(script)
if config.activate:
if isfile(build_file):
data = open(build_file).read()
else:
data = open(work_file).read()
with open(work_file, 'w') as bf:
bf.write(
"source {conda_root}activate {build_prefix} &> "
"/dev/null\n".format(
conda_root=root_script_dir +
os.path.sep,
build_prefix=config.build_prefix))
bf.write(data)
else:
if not isfile(work_file):
copy_into(build_file, work_file,
config.timeout)
os.chmod(work_file, 0o766)
if isfile(work_file):
cmd = [shell_path, '-x', '-e', work_file]
# this should raise if any problems occur while building
_check_call(cmd, env=env, cwd=src_dir)
if post in [True, None]:
if post:
with open(join(config.croot, 'prefix_files.txt'), 'r') as f:
files1 = set(f.read().splitlines())
get_build_metadata(m, config=config)
create_post_scripts(m, config=config)
create_entry_points(m.get_value('build/entry_points'), config=config)
files2 = prefix_files(prefix=config.build_prefix)
post_process(sorted(files2 - files1),
prefix=config.build_prefix,
config=config,
preserve_egg_dir=bool(
m.get_value('build/preserve_egg_dir')))
# The post processing may have deleted some files (like easy-install.pth)
files2 = prefix_files(prefix=config.build_prefix)
if any(config.meta_dir in join(config.build_prefix, f)
for f in files2 - files1):
meta_files = (tuple(
f for f in files2 - files1
if config.meta_dir in join(config.build_prefix, f)), )
sys.exit(
indent(
"""Error: Untracked file(s) %s found in conda-meta directory.
This error usually comes from using conda in the build script. Avoid doing this, as it
can lead to packages that include their dependencies.""" % meta_files))
post_build(m,
sorted(files2 - files1),
prefix=config.build_prefix,
build_python=config.build_python,
croot=config.croot)
create_info_files(m,
sorted(files2 - files1),
config=config,
prefix=config.build_prefix)
if m.get_value('build/noarch_python'):
import conda_build.noarch_python as noarch_python
noarch_python.transform(m, sorted(files2 - files1),
config.build_prefix)
files3 = prefix_files(prefix=config.build_prefix)
fix_permissions(files3 - files1, config.build_prefix)
path = bldpkg_path(m, config)
# lock the output directory while we build this file
# create the tarball in a temporary directory to minimize lock time
with TemporaryDirectory() as tmp:
tmp_path = os.path.join(tmp, os.path.basename(path))
t = tarfile.open(tmp_path, 'w:bz2')
def order(f):
# we don't care about empty files so send them back via 100000
fsize = os.stat(join(config.build_prefix, f)).st_size or 100000
# info/* records will be False == 0, others will be 1.
info_order = int(os.path.dirname(f) != 'info')
return info_order, fsize
# add files in order of a) in info directory, b) increasing size so
# we can access small manifest or json files without decompressing
# possible large binary or data files
for f in sorted(files3 - files1, key=order):
t.add(join(config.build_prefix, f), f)
t.close()
# we're done building, perform some checks
tarcheck.check_all(tmp_path)
copy_into(tmp_path, path, config.timeout)
update_index(config.bldpkgs_dir, config)
else:
print("STOPPING BUILD BEFORE POST:", m.dist())
# returning true here says package is OK to test
return True
def create_env(prefix, specs, config, clear_cache=True):
'''
Create a conda envrionment for the given prefix and specs.
'''
if config.debug:
logging.getLogger("conda").setLevel(logging.DEBUG)
logging.getLogger("binstar").setLevel(logging.DEBUG)
logging.getLogger("install").setLevel(logging.DEBUG)
logging.getLogger("conda.install").setLevel(logging.DEBUG)
logging.getLogger("fetch").setLevel(logging.DEBUG)
logging.getLogger("print").setLevel(logging.DEBUG)
logging.getLogger("progress").setLevel(logging.DEBUG)
logging.getLogger("dotupdate").setLevel(logging.DEBUG)
logging.getLogger("stdoutlog").setLevel(logging.DEBUG)
logging.getLogger("requests").setLevel(logging.DEBUG)
else:
silence_loggers(show_warnings_and_errors=True)
if os.path.isdir(prefix):
rm_rf(prefix)
specs = list(specs)
for feature, value in feature_list:
if value:
specs.append('%s@' % feature)
if specs: # Don't waste time if there is nothing to do
with path_prepended(prefix):
locks = []
try:
cc.pkgs_dirs = cc.pkgs_dirs[:1]
locked_folders = cc.pkgs_dirs + list(config.bldpkgs_dirs)
for folder in locked_folders:
if not os.path.isdir(folder):
os.makedirs(folder)
lock = filelock.SoftFileLock(join(folder, '.conda_lock'))
update_index(folder, config=config, lock=lock)
locks.append(lock)
for lock in locks:
lock.acquire(timeout=config.timeout)
index = get_build_index(config=config, clear_cache=True)
actions = plan.install_actions(prefix, index, specs)
if config.disable_pip:
actions['LINK'] = [
spec for spec in actions['LINK']
if not spec.startswith('pip-')
] # noqa
actions['LINK'] = [
spec for spec in actions['LINK']
if not spec.startswith('setuptools-')
] # noqa
plan.display_actions(actions, index)
if on_win:
for k, v in os.environ.items():
os.environ[k] = str(v)
plan.execute_actions(actions, index, verbose=config.debug)
except (SystemExit, PaddingError, LinkError) as exc:
if (("too short in" in str(exc)
or 'post-link failed for: openssl' in str(exc)
or isinstance(exc, PaddingError))
and config.prefix_length > 80):
log.warn("Build prefix failed with prefix length %d",
config.prefix_length)
log.warn("Error was: ")
log.warn(str(exc))
log.warn(
"One or more of your package dependencies needs to be rebuilt "
"with a longer prefix length.")
log.warn(
"Falling back to legacy prefix length of 80 characters."
)
log.warn(
"Your package will not install into prefixes > 80 characters."
)
config.prefix_length = 80
# Set this here and use to create environ
# Setting this here is important because we use it below (symlink)
prefix = config.build_prefix
for lock in locks:
lock.release()
if os.path.isfile(lock._lock_file):
os.remove(lock._lock_file)
create_env(prefix,
specs,
config=config,
clear_cache=clear_cache)
else:
for lock in locks:
lock.release()
if os.path.isfile(lock._lock_file):
os.remove(lock._lock_file)
raise
finally:
for lock in locks:
lock.release()
if os.path.isfile(lock._lock_file):
os.remove(lock._lock_file)
warn_on_old_conda_build(index=index)
# ensure prefix exists, even if empty, i.e. when specs are empty
if not isdir(prefix):
os.makedirs(prefix)
if on_win:
shell = "cmd.exe"
else:
shell = "bash"
symlink_conda(prefix, sys.prefix, shell)
def get_dataset_downloading_lock(self, name: str) -> None:
return filelock.SoftFileLock("./bogus.lock", 0)
def get_dataset_downloading_lock(self, ds_name: str) -> None:
'Returns a lock. Use in a with statement'
f_lock = self._get_filename('download_lock', ds_name, ext='lock')
return filelock.SoftFileLock(f_lock, 0)
def update_index(dir_path,
config,
force=False,
check_md5=False,
remove=True,
lock=None):
"""
Update all index files in dir_path with changed packages.
:param verbose: Should detailed status messages be output?
:type verbose: bool
:param force: Whether to re-index all packages (including those that
haven't changed) or not.
:type force: bool
:param check_md5: Whether to check MD5s instead of mtimes for determining
if a package changed.
:type check_md5: bool
"""
if config.verbose:
print("updating index in:", dir_path)
index_path = join(dir_path, '.index.json')
if not os.path.isdir(dir_path):
os.makedirs(dir_path)
if not lock:
lock = filelock.SoftFileLock(join(dir_path, ".conda_lock"))
lock.acquire(timeout=config.timeout)
if force:
index = {}
else:
try:
mode_dict = {
'mode': 'r',
'encoding': 'utf-8'
} if PY3 else {
'mode': 'rb'
}
with open(index_path, **mode_dict) as fi:
index = json.load(fi)
except (IOError, ValueError):
index = {}
files = set(fn for fn in os.listdir(dir_path) if fn.endswith('.tar.bz2'))
if any(fn.startswith('_license-') for fn in files):
sys.exit("""\
Error:
Indexing a copy of the Anaconda conda package channel is neither
necessary nor supported. If you wish to add your own packages,
you can do so by adding them to a separate channel.
""")
for fn in files:
path = join(dir_path, fn)
if fn in index:
if check_md5:
if index[fn]['md5'] == md5_file(path):
continue
elif index[fn]['mtime'] == getmtime(path):
continue
if config.verbose:
print('updating:', fn)
d = read_index_tar(path, config, lock=lock)
d.update(file_info(path))
index[fn] = d
for fn in files:
index[fn]['sig'] = '.' if isfile(join(dir_path, fn + '.sig')) else None
if remove:
# remove files from the index which are not on disk
for fn in set(index) - files:
if config.verbose:
print("removing:", fn)
del index[fn]
# Deal with Python 2 and 3's different json module type reqs
mode_dict = {'mode': 'w', 'encoding': 'utf-8'} if PY3 else {'mode': 'wb'}
with open(index_path, **mode_dict) as fo:
json.dump(index, fo, indent=2, sort_keys=True, default=str)
# --- new repodata
for fn in index:
info = index[fn]
for varname in 'arch', 'platform', 'mtime', 'ucs':
try:
del info[varname]
except KeyError:
pass
if 'requires' in info and 'depends' not in info:
info['depends'] = info['requires']
repodata = {'packages': index, 'info': {}}
write_repodata(repodata, dir_path, config, lock=lock)
lock.release()
if os.path.isfile(join(dir_path, ".conda_lock")):
os.remove(join(dir_path, ".conda_lock"))
def prep_conversion(sid, dicoms, outdir, heuristic, converter, anon_sid,
anon_outdir, with_prov, ses, bids_options, seqinfo,
min_meta, overwrite, dcmconfig, grouping):
if dicoms:
lgr.info("Processing %d dicoms", len(dicoms))
elif seqinfo:
lgr.info("Processing %d pre-sorted seqinfo entries", len(seqinfo))
else:
raise ValueError("neither dicoms nor seqinfo dict was provided")
if bids_options is not None:
if not sid:
raise ValueError(
"BIDS requires alphanumeric subject ID. Got an empty value")
if not sid.isalnum(): # alphanumeric only
sid, old_sid = convert_sid_bids(sid)
if not anon_sid:
anon_sid = sid
if not anon_outdir:
anon_outdir = outdir
# Generate heudiconv info folder
idir = op.join(outdir, '.heudiconv', anon_sid)
if bids_options is not None and ses:
idir = op.join(idir, 'ses-%s' % str(ses))
if anon_outdir == outdir:
idir = op.join(idir, 'info')
if not op.exists(idir):
os.makedirs(idir)
ses_suffix = "_ses-%s" % ses if ses is not None else ""
info_file = op.join(idir, '%s%s.auto.txt' % (sid, ses_suffix))
edit_file = op.join(idir, '%s%s.edit.txt' % (sid, ses_suffix))
filegroup_file = op.join(idir, 'filegroup%s.json' % ses_suffix)
# if conversion table(s) do not exist -- we need to prepare them
# (the *prepare* stage in https://github.com/nipy/heudiconv/issues/134)
# if overwrite - recalculate this anyways
reuse_conversion_table = op.exists(edit_file)
# We also might need to redo it if changes in the heuristic file
# detected
# ref: https://github.com/nipy/heudiconv/issues/84#issuecomment-330048609
# for more automagical wishes
target_heuristic_filename = op.join(idir, 'heuristic.py')
# faciliates change - TODO: remove in 1.0
old_heuristic_filename = op.join(idir, op.basename(heuristic.filename))
if op.exists(old_heuristic_filename):
assure_no_file_exists(target_heuristic_filename)
safe_copyfile(old_heuristic_filename, target_heuristic_filename)
assure_no_file_exists(old_heuristic_filename)
# TODO:
# 1. add a test
# 2. possibly extract into a dedicated function for easier logic flow here
# and a dedicated unittest
if (op.exists(target_heuristic_filename)
and file_md5sum(target_heuristic_filename) != file_md5sum(
heuristic.filename)):
# remake conversion table
reuse_conversion_table = False
lgr.info(
"Will not reuse existing conversion table files because heuristic "
"has changed")
if reuse_conversion_table:
lgr.info("Reloading existing filegroup.json "
"because %s exists", edit_file)
info = read_config(edit_file)
filegroup = load_json(filegroup_file)
# XXX Yarik finally understood why basedir was dragged along!
# So we could reuse the same PATHs definitions possibly consistent
# across re-runs... BUT that wouldn't work anyways if e.g.
# DICOMs dumped with SOP UUIDs thus differing across runs etc
# So either it would need to be brought back or reconsidered altogether
# (since no sample data to test on etc)
else:
assure_no_file_exists(target_heuristic_filename)
safe_copyfile(heuristic.filename, target_heuristic_filename)
if dicoms:
seqinfo = group_dicoms_into_seqinfos(
dicoms,
grouping,
file_filter=getattr(heuristic, 'filter_files', None),
dcmfilter=getattr(heuristic, 'filter_dicom', None),
flatten=True,
custom_grouping=getattr(heuristic, 'grouping', None))
seqinfo_list = list(seqinfo.keys())
filegroup = {si.series_id: x for si, x in seqinfo.items()}
dicominfo_file = op.join(idir, 'dicominfo%s.tsv' % ses_suffix)
# allow to overwrite even if was present under git-annex already
assure_no_file_exists(dicominfo_file)
with open(dicominfo_file, 'wt') as fp:
fp.write('\t'.join([val for val in seqinfo_fields]) + '\n')
for seq in seqinfo_list:
fp.write('\t'.join([str(val) for val in seq]) + '\n')
lgr.debug("Calling out to %s.infodict", heuristic)
info = heuristic.infotodict(seqinfo_list)
lgr.debug("Writing to {}, {}, {}".format(info_file, edit_file,
filegroup_file))
assure_no_file_exists(info_file)
write_config(info_file, info)
assure_no_file_exists(edit_file)
write_config(edit_file, info)
save_json(filegroup_file, filegroup)
if bids_options is not None:
# the other portion of the path would mimic BIDS layout
# so we don't need to worry here about sub, ses at all
tdir = anon_outdir
else:
tdir = op.join(anon_outdir, anon_sid)
if converter.lower() != 'none':
lgr.info("Doing conversion using %s", converter)
cinfo = conversion_info(anon_sid, tdir, info, filegroup, ses)
convert(
cinfo,
converter=converter,
scaninfo_suffix=getattr(heuristic, 'scaninfo_suffix', '.json'),
custom_callable=getattr(heuristic, 'custom_callable', None),
with_prov=with_prov,
bids_options=bids_options,
outdir=tdir,
min_meta=min_meta,
overwrite=overwrite,
dcmconfig=dcmconfig,
)
for item_dicoms in filegroup.values():
clear_temp_dicoms(item_dicoms)
if bids_options is not None and 'notop' not in bids_options:
lockfile = op.join(anon_outdir, LOCKFILE)
if op.exists(lockfile):
lgr.warning(
"Existing lockfile found in {0} - waiting for the "
"lock to be released. To set a timeout limit, set "
"the HEUDICONV_FILELOCK_TIMEOUT environmental variable "
"to a value in seconds. If this process hangs, it may "
"require a manual deletion of the {0}.".format(lockfile))
timeout = os.getenv("HEUDICONV_LOCKFILE_TIMEOUT", -1)
with filelock.SoftFileLock(lockfile, timeout=timeout):
if seqinfo:
keys = list(seqinfo)
add_participant_record(anon_outdir, anon_sid,
keys[0].patient_age,
keys[0].patient_sex)
populate_bids_templates(anon_outdir,
getattr(heuristic, 'DEFAULT_FIELDS', {}))
