Exemple #1
0
def get(package_id, package_version, urls, download_location):
    package_found = False
    log.info("Searching for package: "+str(package_id)+" in "+str(urls))
    if not os.path.exists(download_location):
        os.makedirs(download_location)

    handle = None
    if '://' in urls:
        with urllib.request.urlopen(urls) as uf:
            handle = StringIO(uf.read())
    elif os.path.exists(urls):
        handle = open(urls, 'r')
    else:
        raise Exception("--urls option does not look like a url or a file path")

    for ld in yield_packages(handle):
        # TODO: check platform/architecture, failover to all if available?
        # iid, version, platform, architecture, upstream_url, checksum, alternate_url = line.split('\t')
        if ld['id'] == package_id.strip() and (package_version == None or ld['version'] == package_version):
            package_found = True
            # I worry about this being unreliable. TODO: add target filename column?
            pkg_name = package_name(ld)
            storage_path = os.path.join(download_location, pkg_name)
            url = get_url(ld)
            urllib.request.urlretrieve(url, storage_path)
            download_checksum = hashlib.sha256(open(storage_path, 'rb').read()).hexdigest()
            if ld['sha256sum'] != download_checksum:
                log.error('Checksum does not match, something seems to be wrong.\n'
                       '{expected}\t(expected)\n{actual}\t(downloaded)').format(
                           expected=ld['sha256sum'],
                           actual=download_checksum)
            else:
                log.info('Download successful for %s.' % (pkg_name))
    if not package_found:
        log.warning('Package (%s) could not be found in this server.' % (package_id))
Exemple #2
0
def get(package_id, package_version, download_location):
    package_found = False
    database = PACKAGE_SERVER + 'urls.tsv'
    log.info("Searching for package: "+str(package_id)+" in "+str(database))
    if not os.path.exists(download_location):
        os.makedirs(download_location)

    for ld in yield_packages(urllib2.urlopen(database)):
        # TODO: check platform/architecture, failover to all if available?
        # iid, version, platform, architecture, upstream_url, checksum, alternate_url = line.split('\t')
        if ld['id'] == package_id.strip() and ld['platform']== 'src' and (package_version == None or ld['version'] == package_version):
            package_found = True
            # I worry about this being unreliable. TODO: add target filename column?
            pkg_name = package_name(ld)
            storage_path = os.path.join(download_location, pkg_name)
            url = get_url(ld)
            urllib.urlretrieve(url, storage_path)
            download_checksum = hashlib.sha256(open(storage_path, 'rb').read()).hexdigest()
            if ld['sha256sum'] != download_checksum:
                log.error('Checksum does not match, something seems to be wrong.\n'
                       '{expected}\t(expected)\n{actual}\t(downloaded)').format(
                           expected=ld['sha256sum'],
                           actual=download_checksum)
            else:
                log.info('Download successful for %s.' % (pkg_name))
    if not package_found:
        log.warning('Package (%s) could not be found in this server.' % (package_id))
Exemple #3
0
def main():
    retcode = 0
    for package in yield_packages(sys.stdin):
        # For piped in diff, header can appear as +# Id ...
        if package['id'].startswith("+#"):
            continue
        print(package)
        # Remove the '+' at the beginning
        package['id'] = package['id'][1:]

        output_package_path = package_to_path(**package) + package['ext']
        err = download_url(package['url'], output_package_path)

        if err is not None:
            log.error("Could not download file: %s", err)
            retcode = 1
        else:
            log.info("%s downloaded successfully", output_package_path)

        err = verify_file(output_package_path, package['sha256sum'])

        if err is not None:
            log.error("Could not verify file: %s", err)
            retcode = 1
        else:
            log.info("%s verified successfully with hash %s", output_package_path, package['sha256sum'])

    exit(retcode)
Exemple #4
0
def main():
    retcode = 0
    for package in yield_packages(sys.stdin):
        # For piped in diff, header can appear as +# Id ...
        if package['id'].startswith("+#"):
            continue
        print package
        # Remove the '+' at the beginning
        package['id'] = package['id'][1:]

        output_package_path = package_to_path(**package) + package['ext']
        err = download_url(package['url'], output_package_path)

        if err is not None:
            log.error("Could not download file: %s", err)
            retcode = 1
        else:
            log.info("%s downloaded successfully", output_package_path)

        err = verify_file(output_package_path, package['sha256sum'])

        if err is not None:
            log.error("Could not verify file: %s", err)
            retcode = 1
        else:
            log.info("%s verified successfully with hash %s", output_package_path, package['sha256sum'])

    exit(retcode)
Exemple #5
0
def main(galaxy_package_file):
    visited_paths = []
    api_data = {'data': []}

    with open(galaxy_package_file, 'r') as handle:
        retcode = 0
        xunit = XUnitReportBuilder()

        for ld in yield_packages(handle):
            nice_name = package_to_path(**ld)

            if not os.path.exists(ld['id']):
                os.makedirs(ld['id'])

            output_package_path = os.path.join(ld['id'], nice_name) + ld['ext']
            visited_paths.append(os.path.abspath(output_package_path))

            tmpld = {}
            tmpld.update(ld)
            tmpld['_gen'] = output_package_path
            api_data['data'].append(tmpld)

            if os.path.exists(output_package_path) and os.path.getsize(output_package_path) == 0:
                log.error("Empty download, removing %s %s", ld['url'], output_package_path)
                cleanup_file(output_package_path)

            if os.path.exists(output_package_path):
                log.debug("URL exists %s", ld['url'])
                xunit.skip(nice_name)
            else:
                log.info("URL missing, downloading %s to %s", ld['url'], output_package_path)

                if ld['url'].startswith('/'):
                    err = symlink_depot(ld['url'], output_package_path)
                else:
                    err = download_url(ld['url'], output_package_path)

                if err is not None:
                    xunit.failure(nice_name, "DownloadError", err)
                    cleanup_file(output_package_path)
                    continue

                # Check sha256sum of download
                err = verify_file(output_package_path, ld['sha'].strip())
                if err is not None:
                    xunit.error(nice_name, "Sha256sumError", err)
                    cleanup_file(output_package_path)
                    continue

                xunit.ok(nice_name)

        with open('report.xml', 'w') as xunit_handle:
            xunit_handle.write(xunit.serialize())

        print(json.dumps(api_data, indent=2))
    sys.exit(retcode)
Exemple #6
0
def main(galaxy_package_file):
    with open(galaxy_package_file, 'r') as handle:
        retcode = 0
        identifiers = set()

        for ld, lineno, line, extraret in yield_packages(handle, retcode=retcode, meta=True):
            if extraret > 0:
                retcode = extraret
            try:
                for x in HEADER_KEYS[0:5] + HEADER_KEYS[6:6]:
                    # Skip extension, as it is OK to be empty
                    if ld.get(x, '').strip() == '':
                        log.error("[%s] Empty %s", lineno, x)
                        retcode = 1

                if ld['platform'] not in ('linux', 'windows', 'darwin', 'src'):
                    log.error("[%s] Unknown platform %s", lineno, ld['platform'])
                    retcode = 1

                if ld['arch'] not in ('x32', 'x64', 'all'):
                    log.error("[%s] Unknown architecture %s", lineno, ld['arch'])
                    retcode = 1

                if len(ld['sha256sum']) != 64:
                    log.error("[%s] Bad checksum %s", lineno, ld['sha256sum'])
                    retcode = 1

                if ld['sha256sum'] != ld['sha256sum'].lower():
                    log.error("[%s] Uppercase checksum needs to be lower case %s", lineno, ld['sha256sum'])
                    retcode = 1

                if len(ld['sha256sum']) != 64:
                    log.error("[%s] sha256sum is not 64 characters long. Typo?  %s", lineno, ld['sha256sum'])
                    retcode = 1

                if any([x not in list('1234567890abcdef') for x in list(ld['sha256sum'])]):
                    log.error("[%s] sha256sum contains non-hexidecimal characters. Typo?  %s", lineno, ld['sha256sum'])
                    retcode = 1

                if ld['upstream_first'] not in ('True', 'False'):
                    log.error("[%s] Upstream first must be 'True' or 'False', was '%s'", lineno, ld['upstream_first'])
                    retcode = 1

                platform_id = (ld['id'], ld['version'], ld['platform'], ld['arch'])
                if platform_id in identifiers:
                    log.error("[%s] identifier is not unique: '%s'", lineno, platform_id)
                    retcode = 1
                else:
                    identifiers.add(platform_id)
            except Exception, e:
                log.error("[%s] Line (probably) not tabbed properly: %s", lineno, e)
                retcode = 1
        sys.exit(retcode)
Exemple #7
0
def main(galaxy_package_file, id, version=None):
    with open(galaxy_package_file, 'r') as handle:
        for ld in yield_packages(handle):
            if ld['id'].lower() != id.lower():
                continue

            if version is not None and ld['version'].lower() != version.lower():
                continue

            print """<action type="download_by_url" sha256sum="{0[sha]}">
    {1}
</action>""".format(ld, get_url(ld))
Exemple #8
0
def main(galaxy_package_file, id, version=None):
    with open(galaxy_package_file, 'r') as handle:
        for ld in yield_packages(handle):
            if ld['id'].lower() != id.lower():
                continue

            if version is not None and ld['version'].lower() != version.lower():
                continue

            print("""<action type="download_by_url" sha256sum="{0[sha]}">
    {1}
</action>""".format(ld, get_url(ld)))
Exemple #9
0
def main(galaxy_package_file):
    with open(galaxy_package_file, "r") as handle:
        retcode = 0
        identifiers = set()

        for ld, lineno, line, extraret in yield_packages(handle, retcode=retcode, meta=True):
            if extraret > 0:
                retcode = extraret
            try:
                for x in HEADER_KEYS[0:5] + HEADER_KEYS[6:6]:
                    # Skip extension, as it is OK to be empty
                    if ld.get(x, "").strip() == "":
                        log.error("[%s] Empty %s", lineno, x)
                        retcode = 1

                if ld["platform"] not in ("linux", "windows", "darwin", "src"):
                    log.error("[%s] Unknown platform %s", lineno, ld["platform"])
                    retcode = 1

                if ld["arch"] not in ("x32", "x64", "all"):
                    log.error("[%s] Unknown architecture %s", lineno, ld["arch"])
                    retcode = 1

                if len(ld["sha256sum"]) != 64:
                    log.error("[%s] Bad checksum %s", lineno, ld["sha256sum"])
                    retcode = 1

                if ld["sha256sum"] != ld["sha256sum"].lower():
                    log.error("[%s] Uppercase checksum needs to be lower case %s", lineno, ld["sha256sum"])
                    retcode = 1

                if ld["upstream_first"] not in ("True", "False"):
                    log.error("[%s] Upstream first must be 'True' or 'False', was '%s'", lineno, ld["upstream_first"])
                    retcode = 1

                platform_id = (ld["id"], ld["version"], ld["platform"], ld["arch"])
                if platform_id in identifiers:
                    log.error("[%s] identifier is not unique: '%s'", lineno, platform_id)
                    retcode = 1
                else:
                    identifiers.add(platform_id)
            except Exception, e:
                log.error("[%s] Line (probably) not tabbed properly: %s", lineno, e)
                retcode = 1
        sys.exit(retcode)
Exemple #10
0
def main(galaxy_package_file):
    with open(galaxy_package_file, 'r') as handle:
        retcode = 0
        identifiers = set()

        for ld, lineno, line, extraret in yield_packages(handle, retcode=retcode, meta=True):
            if extraret > 0:
                retcode = extraret
            try:
                for x in HEADER_KEYS[0:5] + HEADER_KEYS[6:6]:
                    # Skip extension, as it is OK to be empty
                    if ld.get(x, '').strip() == '':
                        log.error("[%s] Empty %s", lineno, x)
                        retcode = 1

                if ld['platform'] not in ('linux', 'windows', 'darwin', 'src'):
                    log.error("[%s] Unknown platform %s", lineno, ld['platform'])
                    retcode = 1

                if ld['arch'] not in ('x32', 'x64', 'all'):
                    log.error("[%s] Unknown architecture %s", lineno, ld['arch'])
                    retcode = 1

                if len(ld['sha256sum']) != 64:
                    log.error("[%s] Bad checksum %s", lineno, ld['sha256sum'])
                    retcode = 1

                if ld['sha256sum'] != ld['sha256sum'].lower():
                    log.error("[%s] Uppercase checksum needs to be lower case %s", lineno, ld['sha256sum'])
                    retcode = 1

                if ld['upstream_first'] not in ('True', 'False'):
                    log.error("[%s] Upstream first must be 'True' or 'False', was '%s'", lineno, ld['upstream_first'])
                    retcode = 1

                platform_id = (ld['id'], ld['version'], ld['platform'], ld['arch'])
                if platform_id in identifiers:
                    log.error("[%s] identifier is not unique: '%s'", lineno, platform_id)
                    retcode = 1
                else:
                    identifiers.add(platform_id)
            except Exception, e:
                log.error("[%s] Line (probably) not tabbed properly: %s", lineno, e)
                retcode = 1
        sys.exit(retcode)
Exemple #11
0
def main(galaxy_package_file):
    with open(galaxy_package_file, 'r') as handle:

        print '# ' + '\t'.join(['Id', 'Version', 'Platform', 'Architecture', 'Upstream Url', 'Extension', 'sha256sum', 'Use upstream'])
        res = {}
        for ld in yield_packages(handle):
            # id, version, platform,a rch, sha
            key = '_'.join([ld[x] for x in HEADER_KEYS[0:4] + HEADER_KEYS[6:7]])

            if key not in res:
                res[key] = []

            res[key].append(ld)

        for x in sorted(res):
            out = []
            for key in HEADER_KEYS:
                out.append(res[x][0][key])
            print '\t'.join(out).rstrip("\n")
Exemple #12
0
def main(galaxy_package_file, dryrun=False):
    visited_paths = []
    api_data = {'data': []}

    with open(galaxy_package_file, 'r') as handle:
        retcode = 0
        xunit = XUnitReportBuilder()
        xunit.ok("I.Am.Alive")

        for ld in yield_packages(handle):
            nice_name = package_to_path(**ld)

            if not os.path.exists(ld['id']):
                continue

            output_package_path = os.path.join(ld['id'], nice_name) + ld['ext']

            if not os.path.exists(output_package_path):
                continue

            visited_paths.append(os.path.abspath(output_package_path))

            if os.path.exists(output_package_path) and os.path.getsize(
                    output_package_path) == 0:
                log.error("Empty download, removing %s %s", ld['url'],
                          output_package_path)
                cleanup_file(output_package_path)
                xunit.failure(nice_name, "EmptyFile",
                              "%s was found to be empty" % output_package_path)

            err = verify_file(output_package_path, ld['sha256sum'].strip())
            if err is not None:
                xunit.failure(nice_name, "ValidationError", err)

            err = verify_filetype(output_package_path,
                                  ld['ext'].strip(),
                                  dryrun=dryrun)
            if err is not None:
                xunit.failure(nice_name, "ValidationError", err)

        with open('report.xml', 'w') as xunit_handle:
            xunit_handle.write(xunit.serialize())
    sys.exit(retcode)
Exemple #13
0
def main(galaxy_package_file):
    with open(galaxy_package_file, 'r') as handle:

        print '# ' + '\t'.join(['Id', 'Version', 'Platform', 'Architecture', 'Upstream Url', 'Extension', 'sha256sum', 'Alternate Url']),
        res = {}
        for ld in yield_packages(handle):
            # id, version, platform,a rch, sha
            key = '_'.join([ld[x] for x in HEADER_KEYS[0:4] + HEADER_KEYS[6:7]])

            if key not in res:
                res[key] = []

            res[key].append(ld)

        for x in sorted(res):
            out = []
            for key in HEADER_KEYS:
                out.append(res[x][0][key])
            print '\t'.join(out).rstrip("\n")
Exemple #14
0
def main(galaxy_package_file):
    with open(galaxy_package_file, "r") as handle:

        print "# " + "\t".join(
            ["Id", "Version", "Platform", "Architecture", "Upstream Url", "Extension", "sha256sum", "Use upstream"]
        )
        res = {}
        for ld in yield_packages(handle):
            # id, version, platform,a rch, sha
            key = "_".join([ld[x] for x in HEADER_KEYS[0:4] + HEADER_KEYS[6:7]])

            if key not in res:
                res[key] = []

            res[key].append(ld)

        for x in sorted(res):
            out = []
            for key in HEADER_KEYS:
                out.append(res[x][0][key])
            print "\t".join(out).rstrip("\n")