def update_channel_db(repo): # get repository name of secret repository repo_name = OLToolbox.repo_name(repo) local_channel_file = channel_db_file % repo_name remote_channel_url = channel_db_url % repo if os.path.isfile(local_channel_file): fh = open(local_channel_file) local_hash = fh.readline().split()[0] fh.close() else: local_hash = None try: if remote_channel_url.startswith('/'): rfh = open(remote_channel_url, 'rb') else: rfh = urlopen(remote_channel_url) except (URLError, IOError) as e: print('Warning: Channel database update for repository ' + repo_name + ' failed (' + str(e) + '). Skip this repository.') return False hash_line = rfh.readline().decode() if local_hash != hash_line.split()[0]: local_hash = hashlib.md5() tmp_file = local_channel_file + '.~' + str(os.getpid()) lfh = open(tmp_file, 'w') lfh.write(hash_line.strip() + ' ' + time.strftime(OLToolbox.timeformat) + '\n') for line in rfh: lfh.write(line.decode()) local_hash.update(line.strip()) lfh.close() local_hash = local_hash.hexdigest() if local_hash == hash_line.split()[0]: os.rename(tmp_file, local_channel_file) print('updated channel database for repository', repo_name) else: print('ERROR: downloaded channel database inconsistent ' + 'for repository ' + repo_name) sys.exit(1) rfh.close() return True
def update_channel_db(repo): # get repository name of secret repository repo_name = OLToolbox.repo_name(repo) local_channel_file = channel_db_file % repo_name remote_channel_url = channel_db_url % repo if os.path.isfile(local_channel_file): fh = open(local_channel_file) local_hash = fh.readline().split()[0] fh.close() else: local_hash = None try: rfh = urllib2.urlopen(remote_channel_url) except urllib2.HTTPError: print ('*** Channel database update for repository ' + repo_name + ' failed ***') sys.exit(1) hash_line = rfh.readline() if local_hash != hash_line.split()[0]: local_hash = hashlib.md5() tmp_file = local_channel_file + '.~' + str(os.getpid()) lfh = open(tmp_file, 'w') lfh.write(hash_line.strip() + ' ' + time.strftime(OLToolbox.timeformat) + '\n') for line in rfh: lfh.write(line) local_hash.update(line.strip()) lfh.close() local_hash = local_hash.hexdigest() if local_hash == hash_line.split()[0]: os.rename(tmp_file, local_channel_file) print 'updated channel database for repository', repo_name else: print ('ERROR: downloaded channel database inconsistent ' + 'for repository ' + repo_name) sys.exit(1) rfh.close()
process_version, form='%-25s %s') print('done') process_dbs = {} libname_maps = {} max_latest_api_version = 0 if not os.path.isdir(config['process_lib_dir']): os.mkdir(config['process_lib_dir']) first_repo = True found_colls = set() for repo in config['process_repositories']: repo_name = OLToolbox.repo_name(repo) # download the channel database for this repository if not update_channel_db(repo): continue # Latest OpenLoops process API version for which processes # are available in the repository latest_api_version = int( OLToolbox.import_dictionary(latest_api_version_url % repo)['process_api_version']) max_latest_api_version = max(max_latest_api_version, latest_api_version) # This fails if the repository does not contain processes # with the API of the installed OpenLoops version. process_dbs[repo] = OLToolbox.ProcessDB(db=(version_db_url % repo)) libname_maps[repo] = OLToolbox.import_dictionary(libmappins_url % repo, fatal=False) # scan all repositories for collections to download
process_version['hash'] = available[1] OLToolbox.export_dictionary(process_version_file, process_version, form = '%-25s %s') print 'done' process_dbs = {} max_latest_api_version = 0 if not os.path.isdir(config['process_lib_dir']): os.mkdir(config['process_lib_dir']) first_repo = True found_colls = set() for repo in config['process_repositories']: repo_name = OLToolbox.repo_name(repo) # download the channel database for this repository update_channel_db(repo) # Latest OpenLoops process API version for which processes # are available in the repository latest_api_version = int(OLToolbox.import_dictionary( latest_api_version_url % repo)['process_api_version']) max_latest_api_version = max(max_latest_api_version, latest_api_version) # This fails if the repository does not contain processes # with the API of the installed OpenLoops version. process_dbs[repo] = OLToolbox.ProcessDB(db=(version_db_url % repo)) # scan all repositories for collections to download # Note that when the downloader is invoked by the build script, # the collections will already be resolved. for coll in collections: if coll == 'all.coll' or coll == repo_name + '.coll':