def get(package_id, download_location):
package_found = False
for ld in yield_packages(urllib2.urlopen(PACKAGE_SERVER + 'urls.tsv')):
# TODO: check platform/architecture, failover to all if available?
# iid, version, platform, architecture, upstream_url, checksum, alternate_url = line.split('\t')
if ld['id'] == package_id.strip() and ld['platform'] == 'src':
package_found = True
# I worry about this being unreliable. TODO: add target filename column?
pkg_name = urlparse(ld['upstream_url']).path.split('/')[-1]
storage_path = os.path.join(download_location, pkg_name)
if len(ld['alternate_url'].strip()):
url = ld['alternate_url']
else:
url = PACKAGE_SERVER + ld['checksum']
urllib.urlretrieve(url, storage_path)
download_checksum = hashlib.sha256(
open(storage_path, 'rb').read()).hexdigest()
if ld['checksum'] != download_checksum:
print(
'Checksum does not match, something seems to be wrong.\n'
'{expected}\t(expected)\n{actual}\t(downloaded)').format(
expected=ld['checksum'], actual=download_checksum)
else:
print 'Download successful for %s.' % (pkg_name)
if not package_found:
print 'Package (%s) could not be found in this server.' % (package_id)
def get(package_id, download_location):
package_found = False
for ld in yield_packages(urllib2.urlopen(PACKAGE_SERVER + 'urls.tsv')):
# TODO: check platform/architecture, failover to all if available?
# iid, version, platform, architecture, upstream_url, checksum, alternate_url = line.split('\t')
if ld['id'] == package_id.strip() and ld['platform']== 'src':
package_found = True
# I worry about this being unreliable. TODO: add target filename column?
pkg_name = urlparse(ld['upstream_url']).path.split('/')[-1]
storage_path = os.path.join(download_location, pkg_name)
if len(ld['alternate_url'].strip()):
url = ld['alternate_url']
else:
url = PACKAGE_SERVER + ld['checksum']
urllib.urlretrieve(url, storage_path)
download_checksum = hashlib.sha256(open(storage_path, 'rb').read()).hexdigest()
if ld['checksum'] != download_checksum:
print ('Checksum does not match, something seems to be wrong.\n'
'{expected}\t(expected)\n{actual}\t(downloaded)').format(
expected=ld['checksum'],
actual=download_checksum)
else:
print 'Download successful for %s.' % (pkg_name)
if not package_found:
print 'Package (%s) could not be found in this server.' % (package_id)
def main(galaxy_package_file, id, version=None):
with open(galaxy_package_file, 'r') as handle:
for ld in yield_packages(handle):
if ld['id'].lower() != id.lower():
continue
if version is not None and ld['version'].lower() != version.lower():
continue
print """<action type="download_by_url" sha256sum="{0[sha]}">
{1}
</action>""".format(ld, depot_url(ld))
def main(galaxy_package_file):
with open(galaxy_package_file, 'r') as handle:
print '# ' + '\t'.join(['Id', 'Version', 'Platform', 'Architecture', 'Upstream Url', 'Extension', 'sha256sum', 'Alternate Url']),
res = {}
for ld in yield_packages(handle):
# id, version, platform,a rch, sha
key = '_'.join([ld[x] for x in HEADER_KEYS[0:4] + HEADER_KEYS[6:7]])
if key not in res:
res[key] = []
res[key].append(ld)
for x in sorted(res):
out = []
for key in HEADER_KEYS:
out.append(res[x][0][key])
print '\t'.join(out).rstrip("\n")
def main(galaxy_package_file):
with open(galaxy_package_file, 'r') as handle:
print '# ' + '\t'.join(['Id', 'Version', 'Platform', 'Architecture', 'Upstream url', 'sha256sum', 'Alternate Url']),
retcode = 0
res = {}
for ld in yield_packages(handle):
if ld['id'] not in res:
res[ld['id']] = []
res[ld['id']].append(ld)
for x in res:
out = []
for key in keys:
out.append(res[x][0][key])
print '\t'.join(out).rstrip("\n")
sys.exit(retcode)
def main(galaxy_package_file):
with open(galaxy_package_file, 'r') as handle:
retcode = 0
identifiers = set()
keys = ['id', 'version', 'platform', 'arch', 'url', 'sha', 'size',
'alt_url', 'comment']
for ld, lineno, line, extraret in yield_packages(handle, retcode=retcode, meta=True):
if extraret > 0:
retcode = extraret
try:
for x in keys[0:6]:
if ld.get(x, '').strip() == '':
log.error("[%s] Empty %s", lineno, x)
retcode = 1
if ld['platform'] not in ('linux', 'windows', 'darwin', 'src'):
log.error("[%s] Unknown platform %s", lineno, ld['platform'])
retcode = 1
if ld['arch'] not in ('x32', 'x64', 'all'):
log.error("[%s] Unknown architecture %s", lineno, ld['arch'])
retcode = 1
if len(ld['sha']) != 64:
log.error("[%s] Bad checksum %s", lineno, ld['sha'])
retcode = 1
platform_id = (ld['id'], ld['version'], ld['platform'], ld['arch'])
if platform_id in identifiers:
log.error("[%s] identifier is not unique: '%s'", lineno, platform_id)
retcode = 1
else:
identifiers.add(platform_id)
except:
log.error("[%s] Line not tabbed properly", lineno)
retcode = 1
sys.exit(retcode)
def main(galaxy_package_file):
with open(galaxy_package_file, 'r') as handle:
print '# ' + '\t'.join([
'Id', 'Version', 'Platform', 'Architecture', 'Upstream url',
'sha256sum', 'Alternate Url'
]),
retcode = 0
res = {}
for ld in yield_packages(handle):
if ld['id'] not in res:
res[ld['id']] = []
res[ld['id']].append(ld)
for x in res:
out = []
for key in keys:
out.append(res[x][0][key])
print '\t'.join(out).rstrip("\n")
sys.exit(retcode)
def main(galaxy_package_file):
with open(galaxy_package_file, 'r') as handle:
retcode = 0
identifiers = set()
for ld, lineno, line, extraret in yield_packages(handle, retcode=retcode, meta=True):
if extraret > 0:
retcode = extraret
try:
for x in HEADER_KEYS[0:5] + HEADER_KEYS[6:6]:
# Skip extension, as it is OK to be empty
if ld.get(x, '').strip() == '':
log.error("[%s] Empty %s", lineno, x)
retcode = 1
if ld['platform'] not in ('linux', 'windows', 'darwin', 'src'):
log.error("[%s] Unknown platform %s", lineno, ld['platform'])
retcode = 1
if ld['arch'] not in ('x32', 'x64', 'all'):
log.error("[%s] Unknown architecture %s", lineno, ld['arch'])
retcode = 1
if len(ld['sha']) != 64:
log.error("[%s] Bad checksum %s", lineno, ld['sha'])
retcode = 1
platform_id = (ld['id'], ld['version'], ld['platform'], ld['arch'])
if platform_id in identifiers:
log.error("[%s] identifier is not unique: '%s'", lineno, platform_id)
retcode = 1
else:
identifiers.add(platform_id)
except Exception, e:
log.error("[%s] Line (probably) not tabbed properly: %s", lineno, e)
retcode = 1
sys.exit(retcode)
