def update_index(dir_path, verbose=False, force=False): if verbose: print("updating index in:", dir_path) index_path = join(dir_path, '.index.json') if force: index = {} else: try: mode_dict = {'mode': 'r', 'encoding': 'utf-8'} if PY3 else {'mode': 'rb'} with open(index_path, **mode_dict) as fi: index = json.load(fi) except (IOError, ValueError): index = {} files = set(fn for fn in os.listdir(dir_path) if fn.endswith('.tar.bz2')) for fn in files: path = join(dir_path, fn) if fn in index and index[fn]['mtime'] == getmtime(path): continue if verbose: print('updating:', fn) d = read_index_tar(path) d.update(file_info(path)) index[fn] = d # remove files from the index which are not on disk for fn in set(index) - files: if verbose: print("removing:", fn) del index[fn] # Deal with Python 2 and 3's different json module type reqs mode_dict = {'mode': 'w', 'encoding': 'utf-8'} if PY3 else {'mode': 'wb'} with open(index_path, **mode_dict) as fo: json.dump(index, fo, indent=2, sort_keys=True) # --- new repodata icons = {} for fn in index: info = index[fn] if '_icondata' in info: icons[info['_iconmd5']] = base64.b64decode(info['_icondata']) assert '%(_iconmd5)s.png' % info == info['icon'] for varname in ('arch', 'platform', 'mtime', 'ucs', '_icondata', '_iconmd5'): try: del info[varname] except KeyError: pass if icons: icons_dir = join(dir_path, 'icons') if not isdir(icons_dir): os.mkdir(icons_dir) for md5, raw in iteritems(icons): with open(join(icons_dir, '%s.png' % md5), 'wb') as fo: fo.write(raw) repodata = {'packages': index, 'info': {}} write_repodata(repodata, dir_path)
def update_index(dir_path, verbose=False, force=False): if verbose: print("updating index in:", dir_path) index_path = join(dir_path, '.index.json') if force: index = {} else: try: mode_dict = {'mode': 'r', 'encoding': 'utf-8'} if PY3 else {'mode': 'rb'} with open(index_path, **mode_dict) as fi: index = json.load(fi) except (IOError, ValueError): index = {} files = set(fn for fn in os.listdir(dir_path) if fn.endswith('.tar.bz2')) if any(fn.startswith('_license-') for fn in files): sys.exit("""\ Error: Indexing a copy of the Anaconda conda package channel is neither necessary nor supported. If you which to add your own packages, you can do so by adding them to a separate channel. """) for fn in files: path = join(dir_path, fn) if fn in index and index[fn]['mtime'] == getmtime(path): continue if verbose: print('updating:', fn) d = read_index_tar(path) d.update(file_info(path)) index[fn] = d # remove files from the index which are not on disk for fn in set(index) - files: if verbose: print("removing:", fn) del index[fn] # Deal with Python 2 and 3's different json module type reqs mode_dict = {'mode': 'w', 'encoding': 'utf-8'} if PY3 else {'mode': 'wb'} with open(index_path, **mode_dict) as fo: json.dump(index, fo, indent=2, sort_keys=True, default=str) # --- new repodata for fn in index: info = index[fn] for varname in 'arch', 'platform', 'mtime', 'ucs': try: del info[varname] except KeyError: pass repodata = {'packages': index, 'info': {}} write_repodata(repodata, dir_path)
def update_index(dir_path, verbose=False, force=False): if verbose: print("updating index in:", dir_path) index_path = join(dir_path, '.index.json') if force: index = {} else: try: mode_dict = {'mode': 'r', 'encoding': 'utf-8'} if PY3 else {'mode': 'rb'} with open(index_path, **mode_dict) as fi: index = json.load(fi) except (IOError, ValueError): index = {} files = set(fn for fn in os.listdir(dir_path) if fn.endswith('.tar.bz2')) for fn in files: path = join(dir_path, fn) if fn in index and index[fn]['mtime'] == getmtime(path): continue if verbose: print('updating:', fn) d = read_index_tar(path) d.update(file_info(path)) index[fn] = d # remove files from the index which are not on disk for fn in set(index) - files: if verbose: print("removing:", fn) del index[fn] # Deal with Python 2 and 3's different json module type reqs mode_dict = {'mode': 'w', 'encoding': 'utf-8'} if PY3 else {'mode': 'wb'} with open(index_path, **mode_dict) as fo: json.dump(index, fo, indent=2, sort_keys=True, default=str) # --- new repodata for fn in index: info = index[fn] for varname in 'arch', 'platform', 'mtime', 'ucs': try: del info[varname] except KeyError: pass repodata = {'packages': index, 'info': {}} write_repodata(repodata, dir_path)
def update_index(dir_path, config, force=False, check_md5=False, remove=True, lock=None): """ Update all index files in dir_path with changed packages. :param verbose: Should detailed status messages be output? :type verbose: bool :param force: Whether to re-index all packages (including those that haven't changed) or not. :type force: bool :param check_md5: Whether to check MD5s instead of mtimes for determining if a package changed. :type check_md5: bool """ if config.verbose: print("updating index in:", dir_path) index_path = join(dir_path, '.index.json') if not os.path.isdir(dir_path): os.makedirs(dir_path) if not lock: lock = filelock.SoftFileLock(join(dir_path, ".conda_lock")) lock.acquire(timeout=config.timeout) if force: index = {} else: try: mode_dict = { 'mode': 'r', 'encoding': 'utf-8' } if PY3 else { 'mode': 'rb' } with open(index_path, **mode_dict) as fi: index = json.load(fi) except (IOError, ValueError): index = {} files = set(fn for fn in os.listdir(dir_path) if fn.endswith('.tar.bz2')) if any(fn.startswith('_license-') for fn in files): sys.exit("""\ Error: Indexing a copy of the Anaconda conda package channel is neither necessary nor supported. If you wish to add your own packages, you can do so by adding them to a separate channel. """) for fn in files: path = join(dir_path, fn) if fn in index: if check_md5: if index[fn]['md5'] == md5_file(path): continue elif index[fn]['mtime'] == getmtime(path): continue if config.verbose: print('updating:', fn) d = read_index_tar(path, config, lock=lock) d.update(file_info(path)) index[fn] = d for fn in files: index[fn]['sig'] = '.' if isfile(join(dir_path, fn + '.sig')) else None if remove: # remove files from the index which are not on disk for fn in set(index) - files: if config.verbose: print("removing:", fn) del index[fn] # Deal with Python 2 and 3's different json module type reqs mode_dict = {'mode': 'w', 'encoding': 'utf-8'} if PY3 else {'mode': 'wb'} with open(index_path, **mode_dict) as fo: json.dump(index, fo, indent=2, sort_keys=True, default=str) # --- new repodata for fn in index: info = index[fn] for varname in 'arch', 'platform', 'mtime', 'ucs': try: del info[varname] except KeyError: pass if 'requires' in info and 'depends' not in info: info['depends'] = info['requires'] repodata = {'packages': index, 'info': {}} write_repodata(repodata, dir_path, config, lock=lock) lock.release() if os.path.isfile(join(dir_path, ".conda_lock")): os.remove(join(dir_path, ".conda_lock"))
def update_index(dir_path, config, force=False, check_md5=False, remove=True, lock=None, could_be_mirror=True): """ Update all index files in dir_path with changed packages. :param verbose: Should detailed status messages be output? :type verbose: bool :param force: Whether to re-index all packages (including those that haven't changed) or not. :type force: bool :param check_md5: Whether to check MD5s instead of mtimes for determining if a package changed. :type check_md5: bool """ if config.verbose: print("updating index in:", dir_path) index_path = join(dir_path, '.index.json') if not os.path.isdir(dir_path): os.makedirs(dir_path) if not lock: lock = get_lock(dir_path) with lock: if force: index = {} else: try: mode_dict = {'mode': 'r', 'encoding': 'utf-8'} if PY3 else {'mode': 'rb'} with open(index_path, **mode_dict) as fi: index = json.load(fi) except (IOError, ValueError): index = {} files = set(fn for fn in os.listdir(dir_path) if fn.endswith('.tar.bz2')) if could_be_mirror and any(fn.startswith('_license-') for fn in files): sys.exit("""\ Error: Indexing a copy of the Anaconda conda package channel is neither necessary nor supported. If you wish to add your own packages, you can do so by adding them to a separate channel. """) for fn in files: path = join(dir_path, fn) if fn in index: if check_md5: if index[fn]['md5'] == md5_file(path): continue elif index[fn]['mtime'] == getmtime(path): continue if config.verbose: print('updating:', fn) d = read_index_tar(path, config, lock=lock) d.update(file_info(path)) index[fn] = d for fn in files: index[fn]['sig'] = '.' if isfile(join(dir_path, fn + '.sig')) else None if remove: # remove files from the index which are not on disk for fn in set(index) - files: if config.verbose: print("removing:", fn) del index[fn] # Deal with Python 2 and 3's different json module type reqs mode_dict = {'mode': 'w', 'encoding': 'utf-8'} if PY3 else {'mode': 'wb'} with open(index_path, **mode_dict) as fo: json.dump(index, fo, indent=2, sort_keys=True, default=str) # --- new repodata for fn in index: info = index[fn] for varname in 'arch', 'platform', 'mtime', 'ucs': try: del info[varname] except KeyError: pass if 'requires' in info and 'depends' not in info: info['depends'] = info['requires'] repodata = {'packages': index, 'info': {}} write_repodata(repodata, dir_path, lock=lock, config=config)
def update_index(dir_path, config, force=False, check_md5=False, remove=True, lock=None): """ Update all index files in dir_path with changed packages. :param verbose: Should detailed status messages be output? :type verbose: bool :param force: Whether to re-index all packages (including those that haven't changed) or not. :type force: bool :param check_md5: Whether to check MD5s instead of mtimes for determining if a package changed. :type check_md5: bool """ if config.verbose: print("updating index in:", dir_path) index_path = join(dir_path, ".index.json") if not os.path.isdir(dir_path): os.makedirs(dir_path) if not lock: lock = filelock.SoftFileLock(join(dir_path, ".conda_lock")) lock.acquire(timeout=config.timeout) if force: index = {} else: try: mode_dict = {"mode": "r", "encoding": "utf-8"} if PY3 else {"mode": "rb"} with open(index_path, **mode_dict) as fi: index = json.load(fi) except (IOError, ValueError): index = {} files = set(fn for fn in os.listdir(dir_path) if fn.endswith(".tar.bz2")) if any(fn.startswith("_license-") for fn in files): sys.exit( """\ Error: Indexing a copy of the Anaconda conda package channel is neither necessary nor supported. If you wish to add your own packages, you can do so by adding them to a separate channel. """ ) for fn in files: path = join(dir_path, fn) if fn in index: if check_md5: if index[fn]["md5"] == md5_file(path): continue elif index[fn]["mtime"] == getmtime(path): continue if config.verbose: print("updating:", fn) d = read_index_tar(path, config, lock=lock) d.update(file_info(path)) index[fn] = d for fn in files: index[fn]["sig"] = "." if isfile(join(dir_path, fn + ".sig")) else None if remove: # remove files from the index which are not on disk for fn in set(index) - files: if config.verbose: print("removing:", fn) del index[fn] # Deal with Python 2 and 3's different json module type reqs mode_dict = {"mode": "w", "encoding": "utf-8"} if PY3 else {"mode": "wb"} with open(index_path, **mode_dict) as fo: json.dump(index, fo, indent=2, sort_keys=True, default=str) # --- new repodata for fn in index: info = index[fn] for varname in "arch", "platform", "mtime", "ucs": try: del info[varname] except KeyError: pass if "requires" in info and "depends" not in info: info["depends"] = info["requires"] repodata = {"packages": index, "info": {}} write_repodata(repodata, dir_path, config, lock=lock) lock.release() if os.path.isfile(join(dir_path, ".conda_lock")): os.remove(join(dir_path, ".conda_lock"))
def update_index(dir_path, force=False, check_md5=False, remove=True, lock=None, could_be_mirror=True, verbose=True, locking=True, timeout=90): """ Update all index files in dir_path with changed packages. :param verbose: Should detailed status messages be output? :type verbose: bool :param force: Whether to re-index all packages (including those that haven't changed) or not. :type force: bool :param check_md5: Whether to check MD5s instead of mtimes for determining if a package changed. :type check_md5: bool """ log = utils.get_logger(__name__) log.debug("updating index in: %s", dir_path) if not os.path.isdir(dir_path): os.makedirs(dir_path) index_path = join(dir_path, '.index.json') if not lock: lock = get_lock(dir_path) locks = [] if locking: locks.append(lock) index = {} with try_acquire_locks(locks, timeout): if not force: try: mode_dict = {'mode': 'r', 'encoding': 'utf-8'} if PY3 else {'mode': 'rb'} with open(index_path, **mode_dict) as fi: index = json.load(fi) except (IOError, ValueError): index = {} files = set(fn for fn in os.listdir(dir_path) if fn.endswith('.tar.bz2')) for fn in files: path = join(dir_path, fn) if fn in index: if check_md5: if index[fn]['md5'] == md5_file(path): continue elif index[fn]['mtime'] == getmtime(path): continue if verbose: print('updating:', fn) d = read_index_tar(path, lock=lock, locking=locking, timeout=timeout) d.update(file_info(path)) index[fn] = d for fn in files: index[fn]['sig'] = '.' if isfile(join(dir_path, fn + '.sig')) else None if remove: # remove files from the index which are not on disk for fn in set(index) - files: if verbose: print("removing:", fn) del index[fn] # Deal with Python 2 and 3's different json module type reqs mode_dict = {'mode': 'w', 'encoding': 'utf-8'} if PY3 else {'mode': 'wb'} with open(index_path, **mode_dict) as fo: json.dump(index, fo, indent=2, sort_keys=True, default=str) # --- new repodata for fn in index: info = index[fn] for varname in 'arch', 'platform', 'mtime', 'ucs': try: del info[varname] except KeyError: pass if 'requires' in info and 'depends' not in info: info['depends'] = info['requires'] repodata = {'packages': index, 'info': {}} write_repodata(repodata, dir_path, lock=lock, locking=locking, timeout=timeout)
def update_index(dir_path, config, force=False, check_md5=False, remove=True, lock=None, could_be_mirror=True): """ Update all index files in dir_path with changed packages. :param verbose: Should detailed status messages be output? :type verbose: bool :param force: Whether to re-index all packages (including those that haven't changed) or not. :type force: bool :param check_md5: Whether to check MD5s instead of mtimes for determining if a package changed. :type check_md5: bool """ log = utils.get_logger(__name__) log.debug("updating index in: %s", dir_path) if not os.path.isdir(dir_path): os.makedirs(dir_path) index_path = join(dir_path, '.index.json') if not lock: lock = get_lock(dir_path) locks = [] if config.locking: locks.append(lock) index = {} with try_acquire_locks(locks, config.timeout): if not force: try: mode_dict = { 'mode': 'r', 'encoding': 'utf-8' } if PY3 else { 'mode': 'rb' } with open(index_path, **mode_dict) as fi: index = json.load(fi) except (IOError, ValueError): index = {} subdir = None files = set(fn for fn in os.listdir(dir_path) if fn.endswith('.tar.bz2')) for fn in files: path = join(dir_path, fn) if fn in index: if check_md5: if index[fn]['md5'] == md5_file(path): continue elif index[fn]['mtime'] == getmtime(path): continue if config.verbose: print('updating:', fn) d = read_index_tar(path, config, lock=lock) d.update(file_info(path)) index[fn] = d # there's only one subdir for a given folder, so only read these contents once if not subdir: subdir = d['subdir'] for fn in files: index[fn]['sig'] = '.' if isfile(join(dir_path, fn + '.sig')) else None if remove: # remove files from the index which are not on disk for fn in set(index) - files: if config.verbose: print("removing:", fn) del index[fn] # Deal with Python 2 and 3's different json module type reqs mode_dict = { 'mode': 'w', 'encoding': 'utf-8' } if PY3 else { 'mode': 'wb' } with open(index_path, **mode_dict) as fo: json.dump(index, fo, indent=2, sort_keys=True, default=str) # --- new repodata for fn in index: info = index[fn] for varname in 'arch', 'platform', 'mtime', 'ucs': try: del info[varname] except KeyError: pass if 'requires' in info and 'depends' not in info: info['depends'] = info['requires'] repodata = {'packages': index, 'info': {}} write_repodata(repodata, dir_path, lock=lock, config=config)
def update_index(dir_path, force=False, check_md5=False, remove=True, lock=None, could_be_mirror=True, verbose=True, locking=True, timeout=90, channel_name=None): """ Update all index files in dir_path with changed packages. :param verbose: Should detailed status messages be output? :type verbose: bool :param force: Whether to re-index all packages (including those that haven't changed) or not. :type force: bool :param check_md5: Whether to check MD5s instead of mtimes for determining if a package changed. :type check_md5: bool """ log = utils.get_logger(__name__) log.debug("updating index in: %s", dir_path) if not os.path.isdir(dir_path): os.makedirs(dir_path) index_path = join(dir_path, '.index.json') if not lock: lock = get_lock(dir_path) locks = [] if locking: locks.append(lock) index = {} with try_acquire_locks(locks, timeout): if not force: try: mode_dict = { 'mode': 'r', 'encoding': 'utf-8' } if PY3 else { 'mode': 'rb' } with open(index_path, **mode_dict) as fi: index = json.load(fi) except (IOError, ValueError): index = {} files = set(fn for fn in os.listdir(dir_path) if fn.endswith('.tar.bz2')) for fn in files: path = join(dir_path, fn) if fn in index: if check_md5: if index[fn]['md5'] == md5_file(path): continue elif index[fn]['mtime'] == getmtime(path): continue if verbose: print('updating:', fn) d = read_index_tar(path, lock=lock, locking=locking, timeout=timeout) d.update(file_info(path)) index[fn] = d for fn in files: index[fn]['sig'] = '.' if isfile(join(dir_path, fn + '.sig')) else None if remove: # remove files from the index which are not on disk for fn in set(index) - files: if verbose: print("removing:", fn) del index[fn] # Deal with Python 2 and 3's different json module type reqs mode_dict = { 'mode': 'w', 'encoding': 'utf-8' } if PY3 else { 'mode': 'wb' } with open(index_path, **mode_dict) as fo: json.dump(index, fo, indent=2, sort_keys=True, default=str) # --- new repodata for fn in index: info = index[fn] if 'timestamp' not in info and 'mtime' in info: info['timestamp'] = int(info['mtime']) # keep timestamp in original format right now. Pending further testing and eventual # switch to standard UNIX timestamp (in sec) # if info['timestamp'] > 253402300799: # 9999-12-31 # info['timestamp'] //= 1000 # convert milliseconds to seconds; see #1988 for varname in 'arch', 'mtime', 'platform', 'ucs': try: del info[varname] except KeyError: pass if 'requires' in info and 'depends' not in info: info['depends'] = info['requires'] repodata = {'packages': index, 'info': {}} write_repodata(repodata, dir_path, lock=lock, locking=locking, timeout=timeout) if channel_name: extra_paths = {} _add_extra_path(extra_paths, join(dir_path, 'repodata.json')) _add_extra_path(extra_paths, join(dir_path, 'repodata.json.bz2')) rendered_html = make_index_html(channel_name, basename(dir_path), repodata, extra_paths) with open(join(dir_path, 'index.html'), 'w') as fh: fh.write(rendered_html)