def update_index(dir_path, verbose=False, force=False):
if verbose:
print("updating index in:", dir_path)
index_path = join(dir_path, '.index.json')
if force:
index = {}
else:
try:
mode_dict = {'mode': 'r', 'encoding': 'utf-8'} if PY3 else {'mode': 'rb'}
with open(index_path, **mode_dict) as fi:
index = json.load(fi)
except (IOError, ValueError):
index = {}
files = set(fn for fn in os.listdir(dir_path) if fn.endswith('.tar.bz2'))
for fn in files:
path = join(dir_path, fn)
if fn in index and index[fn]['mtime'] == getmtime(path):
continue
if verbose:
print('updating:', fn)
d = read_index_tar(path)
d.update(file_info(path))
index[fn] = d
# remove files from the index which are not on disk
for fn in set(index) - files:
if verbose:
print("removing:", fn)
del index[fn]
# Deal with Python 2 and 3's different json module type reqs
mode_dict = {'mode': 'w', 'encoding': 'utf-8'} if PY3 else {'mode': 'wb'}
with open(index_path, **mode_dict) as fo:
json.dump(index, fo, indent=2, sort_keys=True)
# --- new repodata
icons = {}
for fn in index:
info = index[fn]
if '_icondata' in info:
icons[info['_iconmd5']] = base64.b64decode(info['_icondata'])
assert '%(_iconmd5)s.png' % info == info['icon']
for varname in ('arch', 'platform', 'mtime', 'ucs',
'_icondata', '_iconmd5'):
try:
del info[varname]
except KeyError:
pass
if icons:
icons_dir = join(dir_path, 'icons')
if not isdir(icons_dir):
os.mkdir(icons_dir)
for md5, raw in iteritems(icons):
with open(join(icons_dir, '%s.png' % md5), 'wb') as fo:
fo.write(raw)
repodata = {'packages': index, 'info': {}}
write_repodata(repodata, dir_path)
def update_index(dir_path, verbose=False, force=False):
if verbose:
print("updating index in:", dir_path)
index_path = join(dir_path, '.index.json')
if force:
index = {}
else:
try:
mode_dict = {'mode': 'r', 'encoding': 'utf-8'} if PY3 else {'mode': 'rb'}
with open(index_path, **mode_dict) as fi:
index = json.load(fi)
except (IOError, ValueError):
index = {}
files = set(fn for fn in os.listdir(dir_path) if fn.endswith('.tar.bz2'))
if any(fn.startswith('_license-') for fn in files):
sys.exit("""\
Error:
Indexing a copy of the Anaconda conda package channel is neither
necessary nor supported. If you which to add your own packages,
you can do so by adding them to a separate channel.
""")
for fn in files:
path = join(dir_path, fn)
if fn in index and index[fn]['mtime'] == getmtime(path):
continue
if verbose:
print('updating:', fn)
d = read_index_tar(path)
d.update(file_info(path))
index[fn] = d
# remove files from the index which are not on disk
for fn in set(index) - files:
if verbose:
print("removing:", fn)
del index[fn]
# Deal with Python 2 and 3's different json module type reqs
mode_dict = {'mode': 'w', 'encoding': 'utf-8'} if PY3 else {'mode': 'wb'}
with open(index_path, **mode_dict) as fo:
json.dump(index, fo, indent=2, sort_keys=True, default=str)
# --- new repodata
for fn in index:
info = index[fn]
for varname in 'arch', 'platform', 'mtime', 'ucs':
try:
del info[varname]
except KeyError:
pass
repodata = {'packages': index, 'info': {}}
write_repodata(repodata, dir_path)
def update_index(dir_path, verbose=False, force=False):
if verbose:
print("updating index in:", dir_path)
index_path = join(dir_path, '.index.json')
if force:
index = {}
else:
try:
mode_dict = {'mode': 'r', 'encoding': 'utf-8'} if PY3 else {'mode': 'rb'}
with open(index_path, **mode_dict) as fi:
index = json.load(fi)
except (IOError, ValueError):
index = {}
files = set(fn for fn in os.listdir(dir_path) if fn.endswith('.tar.bz2'))
for fn in files:
path = join(dir_path, fn)
if fn in index and index[fn]['mtime'] == getmtime(path):
continue
if verbose:
print('updating:', fn)
d = read_index_tar(path)
d.update(file_info(path))
index[fn] = d
# remove files from the index which are not on disk
for fn in set(index) - files:
if verbose:
print("removing:", fn)
del index[fn]
# Deal with Python 2 and 3's different json module type reqs
mode_dict = {'mode': 'w', 'encoding': 'utf-8'} if PY3 else {'mode': 'wb'}
with open(index_path, **mode_dict) as fo:
json.dump(index, fo, indent=2, sort_keys=True, default=str)
# --- new repodata
for fn in index:
info = index[fn]
for varname in 'arch', 'platform', 'mtime', 'ucs':
try:
del info[varname]
except KeyError:
pass
repodata = {'packages': index, 'info': {}}
write_repodata(repodata, dir_path)
def update_index(dir_path,
config,
force=False,
check_md5=False,
remove=True,
lock=None):
"""
Update all index files in dir_path with changed packages.
:param verbose: Should detailed status messages be output?
:type verbose: bool
:param force: Whether to re-index all packages (including those that
haven't changed) or not.
:type force: bool
:param check_md5: Whether to check MD5s instead of mtimes for determining
if a package changed.
:type check_md5: bool
"""
if config.verbose:
print("updating index in:", dir_path)
index_path = join(dir_path, '.index.json')
if not os.path.isdir(dir_path):
os.makedirs(dir_path)
if not lock:
lock = filelock.SoftFileLock(join(dir_path, ".conda_lock"))
lock.acquire(timeout=config.timeout)
if force:
index = {}
else:
try:
mode_dict = {
'mode': 'r',
'encoding': 'utf-8'
} if PY3 else {
'mode': 'rb'
}
with open(index_path, **mode_dict) as fi:
index = json.load(fi)
except (IOError, ValueError):
index = {}
files = set(fn for fn in os.listdir(dir_path) if fn.endswith('.tar.bz2'))
if any(fn.startswith('_license-') for fn in files):
sys.exit("""\
Error:
Indexing a copy of the Anaconda conda package channel is neither
necessary nor supported. If you wish to add your own packages,
you can do so by adding them to a separate channel.
""")
for fn in files:
path = join(dir_path, fn)
if fn in index:
if check_md5:
if index[fn]['md5'] == md5_file(path):
continue
elif index[fn]['mtime'] == getmtime(path):
continue
if config.verbose:
print('updating:', fn)
d = read_index_tar(path, config, lock=lock)
d.update(file_info(path))
index[fn] = d
for fn in files:
index[fn]['sig'] = '.' if isfile(join(dir_path, fn + '.sig')) else None
if remove:
# remove files from the index which are not on disk
for fn in set(index) - files:
if config.verbose:
print("removing:", fn)
del index[fn]
# Deal with Python 2 and 3's different json module type reqs
mode_dict = {'mode': 'w', 'encoding': 'utf-8'} if PY3 else {'mode': 'wb'}
with open(index_path, **mode_dict) as fo:
json.dump(index, fo, indent=2, sort_keys=True, default=str)
# --- new repodata
for fn in index:
info = index[fn]
for varname in 'arch', 'platform', 'mtime', 'ucs':
try:
del info[varname]
except KeyError:
pass
if 'requires' in info and 'depends' not in info:
info['depends'] = info['requires']
repodata = {'packages': index, 'info': {}}
write_repodata(repodata, dir_path, config, lock=lock)
lock.release()
if os.path.isfile(join(dir_path, ".conda_lock")):
os.remove(join(dir_path, ".conda_lock"))
def update_index(dir_path, config, force=False, check_md5=False, remove=True, lock=None,
could_be_mirror=True):
"""
Update all index files in dir_path with changed packages.
:param verbose: Should detailed status messages be output?
:type verbose: bool
:param force: Whether to re-index all packages (including those that
haven't changed) or not.
:type force: bool
:param check_md5: Whether to check MD5s instead of mtimes for determining
if a package changed.
:type check_md5: bool
"""
if config.verbose:
print("updating index in:", dir_path)
index_path = join(dir_path, '.index.json')
if not os.path.isdir(dir_path):
os.makedirs(dir_path)
if not lock:
lock = get_lock(dir_path)
with lock:
if force:
index = {}
else:
try:
mode_dict = {'mode': 'r', 'encoding': 'utf-8'} if PY3 else {'mode': 'rb'}
with open(index_path, **mode_dict) as fi:
index = json.load(fi)
except (IOError, ValueError):
index = {}
files = set(fn for fn in os.listdir(dir_path) if fn.endswith('.tar.bz2'))
if could_be_mirror and any(fn.startswith('_license-') for fn in files):
sys.exit("""\
Error:
Indexing a copy of the Anaconda conda package channel is neither
necessary nor supported. If you wish to add your own packages,
you can do so by adding them to a separate channel.
""")
for fn in files:
path = join(dir_path, fn)
if fn in index:
if check_md5:
if index[fn]['md5'] == md5_file(path):
continue
elif index[fn]['mtime'] == getmtime(path):
continue
if config.verbose:
print('updating:', fn)
d = read_index_tar(path, config, lock=lock)
d.update(file_info(path))
index[fn] = d
for fn in files:
index[fn]['sig'] = '.' if isfile(join(dir_path, fn + '.sig')) else None
if remove:
# remove files from the index which are not on disk
for fn in set(index) - files:
if config.verbose:
print("removing:", fn)
del index[fn]
# Deal with Python 2 and 3's different json module type reqs
mode_dict = {'mode': 'w', 'encoding': 'utf-8'} if PY3 else {'mode': 'wb'}
with open(index_path, **mode_dict) as fo:
json.dump(index, fo, indent=2, sort_keys=True, default=str)
# --- new repodata
for fn in index:
info = index[fn]
for varname in 'arch', 'platform', 'mtime', 'ucs':
try:
del info[varname]
except KeyError:
pass
if 'requires' in info and 'depends' not in info:
info['depends'] = info['requires']
repodata = {'packages': index, 'info': {}}
write_repodata(repodata, dir_path, lock=lock, config=config)
def update_index(dir_path, config, force=False, check_md5=False, remove=True, lock=None):
"""
Update all index files in dir_path with changed packages.
:param verbose: Should detailed status messages be output?
:type verbose: bool
:param force: Whether to re-index all packages (including those that
haven't changed) or not.
:type force: bool
:param check_md5: Whether to check MD5s instead of mtimes for determining
if a package changed.
:type check_md5: bool
"""
if config.verbose:
print("updating index in:", dir_path)
index_path = join(dir_path, ".index.json")
if not os.path.isdir(dir_path):
os.makedirs(dir_path)
if not lock:
lock = filelock.SoftFileLock(join(dir_path, ".conda_lock"))
lock.acquire(timeout=config.timeout)
if force:
index = {}
else:
try:
mode_dict = {"mode": "r", "encoding": "utf-8"} if PY3 else {"mode": "rb"}
with open(index_path, **mode_dict) as fi:
index = json.load(fi)
except (IOError, ValueError):
index = {}
files = set(fn for fn in os.listdir(dir_path) if fn.endswith(".tar.bz2"))
if any(fn.startswith("_license-") for fn in files):
sys.exit(
"""\
Error:
Indexing a copy of the Anaconda conda package channel is neither
necessary nor supported. If you wish to add your own packages,
you can do so by adding them to a separate channel.
"""
)
for fn in files:
path = join(dir_path, fn)
if fn in index:
if check_md5:
if index[fn]["md5"] == md5_file(path):
continue
elif index[fn]["mtime"] == getmtime(path):
continue
if config.verbose:
print("updating:", fn)
d = read_index_tar(path, config, lock=lock)
d.update(file_info(path))
index[fn] = d
for fn in files:
index[fn]["sig"] = "." if isfile(join(dir_path, fn + ".sig")) else None
if remove:
# remove files from the index which are not on disk
for fn in set(index) - files:
if config.verbose:
print("removing:", fn)
del index[fn]
# Deal with Python 2 and 3's different json module type reqs
mode_dict = {"mode": "w", "encoding": "utf-8"} if PY3 else {"mode": "wb"}
with open(index_path, **mode_dict) as fo:
json.dump(index, fo, indent=2, sort_keys=True, default=str)
# --- new repodata
for fn in index:
info = index[fn]
for varname in "arch", "platform", "mtime", "ucs":
try:
del info[varname]
except KeyError:
pass
if "requires" in info and "depends" not in info:
info["depends"] = info["requires"]
repodata = {"packages": index, "info": {}}
write_repodata(repodata, dir_path, config, lock=lock)
lock.release()
if os.path.isfile(join(dir_path, ".conda_lock")):
os.remove(join(dir_path, ".conda_lock"))
def update_index(dir_path, force=False, check_md5=False, remove=True, lock=None,
could_be_mirror=True, verbose=True, locking=True, timeout=90):
"""
Update all index files in dir_path with changed packages.
:param verbose: Should detailed status messages be output?
:type verbose: bool
:param force: Whether to re-index all packages (including those that
haven't changed) or not.
:type force: bool
:param check_md5: Whether to check MD5s instead of mtimes for determining
if a package changed.
:type check_md5: bool
"""
log = utils.get_logger(__name__)
log.debug("updating index in: %s", dir_path)
if not os.path.isdir(dir_path):
os.makedirs(dir_path)
index_path = join(dir_path, '.index.json')
if not lock:
lock = get_lock(dir_path)
locks = []
if locking:
locks.append(lock)
index = {}
with try_acquire_locks(locks, timeout):
if not force:
try:
mode_dict = {'mode': 'r', 'encoding': 'utf-8'} if PY3 else {'mode': 'rb'}
with open(index_path, **mode_dict) as fi:
index = json.load(fi)
except (IOError, ValueError):
index = {}
files = set(fn for fn in os.listdir(dir_path) if fn.endswith('.tar.bz2'))
for fn in files:
path = join(dir_path, fn)
if fn in index:
if check_md5:
if index[fn]['md5'] == md5_file(path):
continue
elif index[fn]['mtime'] == getmtime(path):
continue
if verbose:
print('updating:', fn)
d = read_index_tar(path, lock=lock, locking=locking, timeout=timeout)
d.update(file_info(path))
index[fn] = d
for fn in files:
index[fn]['sig'] = '.' if isfile(join(dir_path, fn + '.sig')) else None
if remove:
# remove files from the index which are not on disk
for fn in set(index) - files:
if verbose:
print("removing:", fn)
del index[fn]
# Deal with Python 2 and 3's different json module type reqs
mode_dict = {'mode': 'w', 'encoding': 'utf-8'} if PY3 else {'mode': 'wb'}
with open(index_path, **mode_dict) as fo:
json.dump(index, fo, indent=2, sort_keys=True, default=str)
# --- new repodata
for fn in index:
info = index[fn]
for varname in 'arch', 'platform', 'mtime', 'ucs':
try:
del info[varname]
except KeyError:
pass
if 'requires' in info and 'depends' not in info:
info['depends'] = info['requires']
repodata = {'packages': index, 'info': {}}
write_repodata(repodata, dir_path, lock=lock, locking=locking, timeout=timeout)
def update_index(dir_path,
config,
force=False,
check_md5=False,
remove=True,
lock=None,
could_be_mirror=True):
"""
Update all index files in dir_path with changed packages.
:param verbose: Should detailed status messages be output?
:type verbose: bool
:param force: Whether to re-index all packages (including those that
haven't changed) or not.
:type force: bool
:param check_md5: Whether to check MD5s instead of mtimes for determining
if a package changed.
:type check_md5: bool
"""
log = utils.get_logger(__name__)
log.debug("updating index in: %s", dir_path)
if not os.path.isdir(dir_path):
os.makedirs(dir_path)
index_path = join(dir_path, '.index.json')
if not lock:
lock = get_lock(dir_path)
locks = []
if config.locking:
locks.append(lock)
index = {}
with try_acquire_locks(locks, config.timeout):
if not force:
try:
mode_dict = {
'mode': 'r',
'encoding': 'utf-8'
} if PY3 else {
'mode': 'rb'
}
with open(index_path, **mode_dict) as fi:
index = json.load(fi)
except (IOError, ValueError):
index = {}
subdir = None
files = set(fn for fn in os.listdir(dir_path)
if fn.endswith('.tar.bz2'))
for fn in files:
path = join(dir_path, fn)
if fn in index:
if check_md5:
if index[fn]['md5'] == md5_file(path):
continue
elif index[fn]['mtime'] == getmtime(path):
continue
if config.verbose:
print('updating:', fn)
d = read_index_tar(path, config, lock=lock)
d.update(file_info(path))
index[fn] = d
# there's only one subdir for a given folder, so only read these contents once
if not subdir:
subdir = d['subdir']
for fn in files:
index[fn]['sig'] = '.' if isfile(join(dir_path, fn +
'.sig')) else None
if remove:
# remove files from the index which are not on disk
for fn in set(index) - files:
if config.verbose:
print("removing:", fn)
del index[fn]
# Deal with Python 2 and 3's different json module type reqs
mode_dict = {
'mode': 'w',
'encoding': 'utf-8'
} if PY3 else {
'mode': 'wb'
}
with open(index_path, **mode_dict) as fo:
json.dump(index, fo, indent=2, sort_keys=True, default=str)
# --- new repodata
for fn in index:
info = index[fn]
for varname in 'arch', 'platform', 'mtime', 'ucs':
try:
del info[varname]
except KeyError:
pass
if 'requires' in info and 'depends' not in info:
info['depends'] = info['requires']
repodata = {'packages': index, 'info': {}}
write_repodata(repodata, dir_path, lock=lock, config=config)
def update_index(dir_path,
force=False,
check_md5=False,
remove=True,
lock=None,
could_be_mirror=True,
verbose=True,
locking=True,
timeout=90,
channel_name=None):
"""
Update all index files in dir_path with changed packages.
:param verbose: Should detailed status messages be output?
:type verbose: bool
:param force: Whether to re-index all packages (including those that
haven't changed) or not.
:type force: bool
:param check_md5: Whether to check MD5s instead of mtimes for determining
if a package changed.
:type check_md5: bool
"""
log = utils.get_logger(__name__)
log.debug("updating index in: %s", dir_path)
if not os.path.isdir(dir_path):
os.makedirs(dir_path)
index_path = join(dir_path, '.index.json')
if not lock:
lock = get_lock(dir_path)
locks = []
if locking:
locks.append(lock)
index = {}
with try_acquire_locks(locks, timeout):
if not force:
try:
mode_dict = {
'mode': 'r',
'encoding': 'utf-8'
} if PY3 else {
'mode': 'rb'
}
with open(index_path, **mode_dict) as fi:
index = json.load(fi)
except (IOError, ValueError):
index = {}
files = set(fn for fn in os.listdir(dir_path)
if fn.endswith('.tar.bz2'))
for fn in files:
path = join(dir_path, fn)
if fn in index:
if check_md5:
if index[fn]['md5'] == md5_file(path):
continue
elif index[fn]['mtime'] == getmtime(path):
continue
if verbose:
print('updating:', fn)
d = read_index_tar(path,
lock=lock,
locking=locking,
timeout=timeout)
d.update(file_info(path))
index[fn] = d
for fn in files:
index[fn]['sig'] = '.' if isfile(join(dir_path, fn +
'.sig')) else None
if remove:
# remove files from the index which are not on disk
for fn in set(index) - files:
if verbose:
print("removing:", fn)
del index[fn]
# Deal with Python 2 and 3's different json module type reqs
mode_dict = {
'mode': 'w',
'encoding': 'utf-8'
} if PY3 else {
'mode': 'wb'
}
with open(index_path, **mode_dict) as fo:
json.dump(index, fo, indent=2, sort_keys=True, default=str)
# --- new repodata
for fn in index:
info = index[fn]
if 'timestamp' not in info and 'mtime' in info:
info['timestamp'] = int(info['mtime'])
# keep timestamp in original format right now. Pending further testing and eventual
# switch to standard UNIX timestamp (in sec)
# if info['timestamp'] > 253402300799: # 9999-12-31
# info['timestamp'] //= 1000 # convert milliseconds to seconds; see #1988
for varname in 'arch', 'mtime', 'platform', 'ucs':
try:
del info[varname]
except KeyError:
pass
if 'requires' in info and 'depends' not in info:
info['depends'] = info['requires']
repodata = {'packages': index, 'info': {}}
write_repodata(repodata,
dir_path,
lock=lock,
locking=locking,
timeout=timeout)
if channel_name:
extra_paths = {}
_add_extra_path(extra_paths, join(dir_path, 'repodata.json'))
_add_extra_path(extra_paths, join(dir_path, 'repodata.json.bz2'))
rendered_html = make_index_html(channel_name, basename(dir_path),
repodata, extra_paths)
with open(join(dir_path, 'index.html'), 'w') as fh:
fh.write(rendered_html)