def update_channel_db(repo):
# get repository name of secret repository
repo_name = OLToolbox.repo_name(repo)
local_channel_file = channel_db_file % repo_name
remote_channel_url = channel_db_url % repo
if os.path.isfile(local_channel_file):
fh = open(local_channel_file)
local_hash = fh.readline().split()[0]
fh.close()
else:
local_hash = None
try:
if remote_channel_url.startswith('/'):
rfh = open(remote_channel_url, 'rb')
else:
rfh = urlopen(remote_channel_url)
except (URLError, IOError) as e:
print('Warning: Channel database update for repository ' + repo_name +
' failed (' + str(e) + '). Skip this repository.')
return False
hash_line = rfh.readline().decode()
if local_hash != hash_line.split()[0]:
local_hash = hashlib.md5()
tmp_file = local_channel_file + '.~' + str(os.getpid())
lfh = open(tmp_file, 'w')
lfh.write(hash_line.strip() + ' ' +
time.strftime(OLToolbox.timeformat) + '\n')
for line in rfh:
lfh.write(line.decode())
local_hash.update(line.strip())
lfh.close()
local_hash = local_hash.hexdigest()
if local_hash == hash_line.split()[0]:
os.rename(tmp_file, local_channel_file)
print('updated channel database for repository', repo_name)
else:
print('ERROR: downloaded channel database inconsistent ' +
'for repository ' + repo_name)
sys.exit(1)
rfh.close()
return True
def update_channel_db(repo):
# get repository name of secret repository
repo_name = OLToolbox.repo_name(repo)
local_channel_file = channel_db_file % repo_name
remote_channel_url = channel_db_url % repo
if os.path.isfile(local_channel_file):
fh = open(local_channel_file)
local_hash = fh.readline().split()[0]
fh.close()
else:
local_hash = None
try:
rfh = urllib2.urlopen(remote_channel_url)
except urllib2.HTTPError:
print ('*** Channel database update for repository ' + repo_name +
' failed ***')
sys.exit(1)
hash_line = rfh.readline()
if local_hash != hash_line.split()[0]:
local_hash = hashlib.md5()
tmp_file = local_channel_file + '.~' + str(os.getpid())
lfh = open(tmp_file, 'w')
lfh.write(hash_line.strip() + ' ' +
time.strftime(OLToolbox.timeformat) + '\n')
for line in rfh:
lfh.write(line)
local_hash.update(line.strip())
lfh.close()
local_hash = local_hash.hexdigest()
if local_hash == hash_line.split()[0]:
os.rename(tmp_file, local_channel_file)
print 'updated channel database for repository', repo_name
else:
print ('ERROR: downloaded channel database inconsistent ' +
'for repository ' + repo_name)
sys.exit(1)
rfh.close()
process_version,
form='%-25s %s')
print('done')
process_dbs = {}
libname_maps = {}
max_latest_api_version = 0
if not os.path.isdir(config['process_lib_dir']):
os.mkdir(config['process_lib_dir'])
first_repo = True
found_colls = set()
for repo in config['process_repositories']:
repo_name = OLToolbox.repo_name(repo)
# download the channel database for this repository
if not update_channel_db(repo):
continue
# Latest OpenLoops process API version for which processes
# are available in the repository
latest_api_version = int(
OLToolbox.import_dictionary(latest_api_version_url %
repo)['process_api_version'])
max_latest_api_version = max(max_latest_api_version, latest_api_version)
# This fails if the repository does not contain processes
# with the API of the installed OpenLoops version.
process_dbs[repo] = OLToolbox.ProcessDB(db=(version_db_url % repo))
libname_maps[repo] = OLToolbox.import_dictionary(libmappins_url % repo,
fatal=False)
# scan all repositories for collections to download
process_version['hash'] = available[1]
OLToolbox.export_dictionary(process_version_file, process_version,
form = '%-25s %s')
print 'done'
process_dbs = {}
max_latest_api_version = 0
if not os.path.isdir(config['process_lib_dir']):
os.mkdir(config['process_lib_dir'])
first_repo = True
found_colls = set()
for repo in config['process_repositories']:
repo_name = OLToolbox.repo_name(repo)
# download the channel database for this repository
update_channel_db(repo)
# Latest OpenLoops process API version for which processes
# are available in the repository
latest_api_version = int(OLToolbox.import_dictionary(
latest_api_version_url % repo)['process_api_version'])
max_latest_api_version = max(max_latest_api_version, latest_api_version)
# This fails if the repository does not contain processes
# with the API of the installed OpenLoops version.
process_dbs[repo] = OLToolbox.ProcessDB(db=(version_db_url % repo))
# scan all repositories for collections to download
# Note that when the downloader is invoked by the build script,
# the collections will already be resolved.
for coll in collections:
if coll == 'all.coll' or coll == repo_name + '.coll':
