Esempio n. 1
0
 def get_defaultsuite(self):
     from daklib.dbconn import get_suite
     suitename = self.defaultsuitename
     if not suitename:
         return None
     else:
         return get_suite(suitename)
Esempio n. 2
0
File: config.py Progetto: Debian/dak
 def get_defaultsuite(self):
     from daklib.dbconn import get_suite
     suitename = self.defaultsuitename
     if not suitename:
         return None
     else:
         return get_suite(suitename)
Esempio n. 3
0
def main():
    cnf = Config()

    Arguments = [('h',"help","DEP11::Options::Help"),
                 ('s',"suite","DEP11::Options::Suite", "HasArg"),
                 ('e',"expire","DEP11::Options::ExpireCache"),
                 ('h',"write-hints","DEP11::Options::WriteHints"),
                 ]
    for i in ["help", "suite", "ExpireCache"]:
        if not cnf.has_key("DEP11::Options::%s" % (i)):
            cnf["DEP11::Options::%s" % (i)] = ""

    arguments = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
    Options = cnf.subtree("DEP11::Options")

    if Options["Help"]:
        usage()
        return

    suitename = Options["Suite"]
    if not suitename:
        print("You need to specify a suite!")
        sys.exit(1)

    # check if we have some important config options set
    if not cnf.has_key("Dir::MetaInfo"):
        print("You need to specify a metadata export directory (Dir::MetaInfo)")
        sys.exit(1)
    if not cnf.has_key("DEP11::Url"):
        print("You need to specify a metadata public web URL (DEP11::Url)")
        sys.exit(1)
    if not cnf.has_key("DEP11::IconSizes"):
        print("You need to specify a list of allowed icon-sizes (DEP11::IconSizes)")
        sys.exit(1)
    if Options["WriteHints"] and not cnf.has_key("Dir::MetaInfoHints"):
        print("You need to specify an export directory for DEP-11 hints files (Dir::MetaInfoHints)")
        sys.exit(1)

    logger = daklog.Logger('generate-metadata')

    from daklib.dbconn import Component, DBConn, get_suite, Suite
    session = DBConn().session()
    suite = get_suite(suitename.lower(), session)

    if Options["ExpireCache"]:
        expire_dep11_data_cache(session, suitename, logger)

    process_suite(session, suite, logger)
    # export database content as Components-<arch>.xz YAML documents
    write_component_files(session, suite, logger)

    if Options["WriteHints"]:
        write_hints_files(session, suite, logger)

    # we're done
    logger.close()
Esempio n. 4
0
def main():
    cnf = Config()

    Arguments = [
        ('h', "help", "DEP11::Options::Help"),
        ('e', "expire", "DEP11::Options::ExpireCache"),
        ('s', "suite", "DEP11::Options::Suite", "HasArg"),
    ]
    for i in ["help", "suite", "ExpireCache"]:
        if not cnf.has_key("DEP11::Options::%s" % (i)):
            cnf["DEP11::Options::%s" % (i)] = ""

    arguments = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
    Options = cnf.subtree("DEP11::Options")

    if Options["Help"]:
        usage()
        return

    suitename = Options["Suite"]
    if not suitename:
        print("You need to specify a suite!")
        return

    logger = daklog.Logger('generate-metadata')

    from daklib.dbconn import Component, DBConn, get_suite, Suite
    session = DBConn().session()
    suite = get_suite(suitename.lower(), session)

    if Options["ExpireCache"]:
        expire_dep11_data_cache(session, suitename)

    process_suite(session, suite, logger)
    # export database content as Components-<arch>.xz YAML documents
    write_component_files(suite)

    # we're done
    logger.close()
def main():
    global Cnf, Options, Logger

    os.umask(0o002)

    Cnf = utils.get_conf()
    Arguments = [ ('h', "help", "Generate-Index-Diffs::Options::Help"),
                  ('a', 'archive', 'Generate-Index-Diffs::Options::Archive', 'hasArg'),
                  ('c', None, "Generate-Index-Diffs::Options::CanonicalPath", "hasArg"),
                  ('p', "patchname", "Generate-Index-Diffs::Options::PatchName", "hasArg"),
                  ('d', "tmpdir", "Generate-Index-Diffs::Options::TempDir", "hasArg"),
                  ('m', "maxdiffs", "Generate-Index-Diffs::Options::MaxDiffs", "hasArg"),
                  ('n', "n-act", "Generate-Index-Diffs::Options::NoAct"),
                ]
    suites = apt_pkg.parse_commandline(Cnf,Arguments,sys.argv)
    Options = Cnf.subtree("Generate-Index-Diffs::Options")
    if Options.has_key("Help"): usage()

    maxdiffs = Options.get("MaxDiffs::Default", "56")
    maxpackages = Options.get("MaxDiffs::Packages", maxdiffs)
    maxcontents = Options.get("MaxDiffs::Contents", maxdiffs)
    maxsources = Options.get("MaxDiffs::Sources", maxdiffs)

    if not Options.has_key("PatchName"):
        format = "%Y-%m-%d-%H%M.%S"
        Options["PatchName"] = time.strftime( format )

    session = DBConn().session()

    if not suites:
        query = session.query(Suite.suite_name)
        if Options.get('Archive'):
            query = query.join(Suite.archive).filter(Archive.archive_name == Options['Archive'])
        suites = [ s.suite_name for s in query ]

    for suitename in suites:
        print "Processing: " + suitename

        suiteobj = get_suite(suitename.lower(), session=session)

        # Use the canonical version of the suite name
        suite = suiteobj.suite_name

        if suiteobj.untouchable:
            print "Skipping: " + suite + " (untouchable)"
            continue

        architectures = get_suite_architectures(suite, skipall=True, session=session)
        components = [ c.component_name for c in session.query(Component.component_name) ]

        suite_suffix = Cnf.find("Dinstall::SuiteSuffix")
        if components and suite_suffix:
            longsuite = suite + "/" + suite_suffix
        else:
            longsuite = suite

        tree = os.path.join(suiteobj.archive.path, 'dists', longsuite)

        # See if there are Translations which might need a new pdiff
        cwd = os.getcwd()
        for component in components:
            #print "DEBUG: Working on %s" % (component)
            workpath=os.path.join(tree, component, "i18n")
            if os.path.isdir(workpath):
                os.chdir(workpath)
                for dirpath, dirnames, filenames in os.walk(".", followlinks=True, topdown=True):
                    for entry in filenames:
                        if not re_includeinpdiff.match(entry):
                            #print "EXCLUDING %s" % (entry)
                            continue
                        (fname, fext) = os.path.splitext(entry)
                        processfile=os.path.join(workpath, fname)
                        #print "Working: %s" % (processfile)
                        storename="%s/%s_%s_%s" % (Options["TempDir"], suite, component, fname)
                        #print "Storefile: %s" % (storename)
                        genchanges(Options, processfile + ".diff", storename, processfile, maxdiffs)
        os.chdir(cwd)

        for archobj in architectures:
            architecture = archobj.arch_string

            for component in components:
                if architecture == "source":
                    longarch = architecture
                    packages = "Sources"
                    maxsuite = maxsources
                else:
                    longarch = "binary-%s"% (architecture)
                    packages = "Packages"
                    maxsuite = maxpackages
                    # Process Contents
                    file = "%s/%s/Contents-%s" % (tree, component, architecture)
                    storename = "%s/%s_%s_contents_%s" % (Options["TempDir"], suite, component, architecture)
                    genchanges(Options, file + ".diff", storename, file, maxcontents)

                file = "%s/%s/%s/%s" % (tree, component, longarch, packages)
                storename = "%s/%s_%s_%s" % (Options["TempDir"], suite, component, architecture)
                genchanges(Options, file + ".diff", storename, file, maxsuite)
def main():
    from daklib.config import Config
    from daklib import daklog

    cnf = Config()

    Arguments = [('h', "help", "Generate-Packages-Sources::Options::Help"),
                 ('a', 'archive',
                  'Generate-Packages-Sources::Options::Archive', 'HasArg'),
                 ('s', "suite", "Generate-Packages-Sources::Options::Suite",
                  'HasArg'),
                 ('f', "force", "Generate-Packages-Sources::Options::Force"),
                 ('o', 'option', '', 'ArbItem')]

    apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
    try:
        Options = cnf.subtree("Generate-Packages-Sources::Options")
    except KeyError:
        Options = {}

    if Options.has_key("Help"):
        usage()

    from daklib.dakmultiprocessing import DakProcessPool, PROC_STATUS_SUCCESS, PROC_STATUS_SIGNALRAISED
    pool = DakProcessPool()

    logger = daklog.Logger('generate-packages-sources2')

    from daklib.dbconn import Component, DBConn, get_suite, Suite, Archive
    session = DBConn().session()
    session.execute("SELECT add_missing_description_md5()")
    session.commit()

    import daklib.utils

    if Options.has_key("Suite"):
        suites = []
        suite_names = daklib.utils.split_args(Options['Suite'])
        for s in suite_names:
            suite = get_suite(s.lower(), session)
            if suite:
                suites.append(suite)
            else:
                print "I: Cannot find suite %s" % s
                logger.log(['Cannot find suite %s' % s])
    else:
        query = session.query(Suite).filter(Suite.untouchable == False)
        if 'Archive' in Options:
            archive_names = daklib.utils.split_args(Options['Archive'])
            query = query.join(Suite.archive).filter(
                Archive.archive_name.in_(archive_names))
        suites = query.all()

    force = Options.has_key("Force") and Options["Force"]

    def parse_results(message):
        # Split out into (code, msg)
        code, msg = message
        if code == PROC_STATUS_SUCCESS:
            logger.log(msg)
        elif code == PROC_STATUS_SIGNALRAISED:
            logger.log(['E: Subprocess received signal ', msg])
        else:
            logger.log(['E: ', msg])

    # Lock tables so that nobody can change things underneath us
    session.execute("LOCK TABLE src_associations IN SHARE MODE")
    session.execute("LOCK TABLE bin_associations IN SHARE MODE")

    for s in suites:
        component_ids = [c.component_id for c in s.components]
        if s.untouchable and not force:
            import daklib.utils
            daklib.utils.fubar(
                "Refusing to touch %s (untouchable and not forced)" %
                s.suite_name)
        for c in component_ids:
            pool.apply_async(generate_sources, [s.suite_id, c],
                             callback=parse_results)
            if not s.include_long_description:
                pool.apply_async(generate_translations, [s.suite_id, c],
                                 callback=parse_results)
            for a in s.architectures:
                if a == 'source':
                    continue
                pool.apply_async(generate_packages,
                                 [s.suite_id, c, a.arch_id, 'deb'],
                                 callback=parse_results)
                pool.apply_async(generate_packages,
                                 [s.suite_id, c, a.arch_id, 'udeb'],
                                 callback=parse_results)

    pool.close()
    pool.join()

    # this script doesn't change the database
    session.close()

    logger.close()

    sys.exit(pool.overall_status())
Esempio n. 7
0
def main():
    from daklib.config import Config
    from daklib import daklog

    cnf = Config()

    Arguments = [('h',"help","Generate-Packages-Sources::Options::Help"),
                 ('a','archive','Generate-Packages-Sources::Options::Archive','HasArg'),
                 ('s',"suite","Generate-Packages-Sources::Options::Suite"),
                 ('f',"force","Generate-Packages-Sources::Options::Force"),
                 ('o','option','','ArbItem')]

    suite_names = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
    try:
        Options = cnf.subtree("Generate-Packages-Sources::Options")
    except KeyError:
        Options = {}

    if Options.has_key("Help"):
        usage()

    from daklib.dakmultiprocessing import DakProcessPool, PROC_STATUS_SUCCESS, PROC_STATUS_SIGNALRAISED
    pool = DakProcessPool()

    logger = daklog.Logger('generate-packages-sources2')

    from daklib.dbconn import Component, DBConn, get_suite, Suite, Archive
    session = DBConn().session()
    session.execute("SELECT add_missing_description_md5()")
    session.commit()

    if Options.has_key("Suite"):
        suites = []
        for s in suite_names:
            suite = get_suite(s.lower(), session)
            if suite:
                suites.append(suite)
            else:
                print "I: Cannot find suite %s" % s
                logger.log(['Cannot find suite %s' % s])
    else:
        query = session.query(Suite).filter(Suite.untouchable == False)
        if 'Archive' in Options:
            query = query.join(Suite.archive).filter(Archive.archive_name==Options['Archive'])
        suites = query.all()

    force = Options.has_key("Force") and Options["Force"]


    def parse_results(message):
        # Split out into (code, msg)
        code, msg = message
        if code == PROC_STATUS_SUCCESS:
            logger.log([msg])
        elif code == PROC_STATUS_SIGNALRAISED:
            logger.log(['E: Subprocess recieved signal ', msg])
        else:
            logger.log(['E: ', msg])

    for s in suites:
        component_ids = [ c.component_id for c in s.components ]
        if s.untouchable and not force:
            import daklib.utils
            daklib.utils.fubar("Refusing to touch %s (untouchable and not forced)" % s.suite_name)
        for c in component_ids:
            pool.apply_async(generate_sources, [s.suite_id, c], callback=parse_results)
            if not s.include_long_description:
                pool.apply_async(generate_translations, [s.suite_id, c], callback=parse_results)
            for a in s.architectures:
                if a == 'source':
                    continue
                pool.apply_async(generate_packages, [s.suite_id, c, a.arch_id, 'deb'], callback=parse_results)
                pool.apply_async(generate_packages, [s.suite_id, c, a.arch_id, 'udeb'], callback=parse_results)

    pool.close()
    pool.join()

    # this script doesn't change the database
    session.close()

    logger.close()

    sys.exit(pool.overall_status())
Esempio n. 8
0
def main():
    global Cnf, Options, Logger

    os.umask(0o002)

    Cnf = utils.get_conf()
    Arguments = [('h', "help", "Generate-Index-Diffs::Options::Help"),
                  ('a', 'archive', 'Generate-Index-Diffs::Options::Archive', 'hasArg'),
                  ('c', None, "Generate-Index-Diffs::Options::CanonicalPath", "hasArg"),
                  ('p', "patchname", "Generate-Index-Diffs::Options::PatchName", "hasArg"),
                  ('d', "tmpdir", "Generate-Index-Diffs::Options::TempDir", "hasArg"),
                  ('m', "maxdiffs", "Generate-Index-Diffs::Options::MaxDiffs", "hasArg"),
                  ('n', "no-act", "Generate-Index-Diffs::Options::NoAct"),
                  ('v', "verbose", "Generate-Index-Diffs::Options::Verbose"),
                ]
    suites = apt_pkg.parse_commandline(Cnf, Arguments, sys.argv)
    Options = Cnf.subtree("Generate-Index-Diffs::Options")
    if "Help" in Options:
        usage()

    maxdiffs = Options.get("MaxDiffs::Default", "56")
    maxpackages = Options.get("MaxDiffs::Packages", maxdiffs)
    maxcontents = Options.get("MaxDiffs::Contents", maxdiffs)
    maxsources = Options.get("MaxDiffs::Sources", maxdiffs)

    # can only be set via config at the moment
    max_parallel = int(Options.get("MaxParallel", "8"))

    if "PatchName" not in Options:
        format = "%Y-%m-%d-%H%M.%S"
        Options["PatchName"] = time.strftime(format)

    session = DBConn().session()
    pending_tasks = []

    if not suites:
        query = session.query(Suite.suite_name)
        if Options.get('Archive'):
            archives = utils.split_args(Options['Archive'])
            query = query.join(Suite.archive).filter(Archive.archive_name.in_(archives))
        suites = [s.suite_name for s in query]

    for suitename in suites:
        print("Processing: " + suitename)

        suiteobj = get_suite(suitename.lower(), session=session)

        # Use the canonical version of the suite name
        suite = suiteobj.suite_name

        if suiteobj.untouchable:
            print("Skipping: " + suite + " (untouchable)")
            continue

        skip_all = True
        if suiteobj.separate_contents_architecture_all or suiteobj.separate_packages_architecture_all:
            skip_all = False

        architectures = get_suite_architectures(suite, skipall=skip_all, session=session)
        components = [c.component_name for c in session.query(Component.component_name)]

        suite_suffix = utils.suite_suffix(suitename)
        if components and suite_suffix:
            longsuite = suite + "/" + suite_suffix
        else:
            longsuite = suite

        merged_pdiffs = suiteobj.merged_pdiffs

        tree = os.path.join(suiteobj.archive.path, 'dists', longsuite)

        # See if there are Translations which might need a new pdiff
        cwd = os.getcwd()
        for component in components:
            workpath = os.path.join(tree, component, "i18n")
            if os.path.isdir(workpath):
                os.chdir(workpath)
                for dirpath, dirnames, filenames in os.walk(".", followlinks=True, topdown=True):
                    for entry in filenames:
                        if not re_includeinpdiff.match(entry):
                            continue
                        (fname, fext) = os.path.splitext(entry)
                        processfile = os.path.join(workpath, fname)
                        storename = "%s/%s_%s_%s" % (Options["TempDir"], suite, component, fname)
                        coroutine = genchanges(Options, processfile + ".diff", storename, processfile, maxdiffs, merged_pdiffs)
                        pending_tasks.append(coroutine)
        os.chdir(cwd)

        for archobj in architectures:
            architecture = archobj.arch_string

            if architecture == "source":
                longarch = architecture
                packages = "Sources"
                maxsuite = maxsources
            else:
                longarch = "binary-%s" % architecture
                packages = "Packages"
                maxsuite = maxpackages

            for component in components:
                # Process Contents
                file = "%s/%s/Contents-%s" % (tree, component, architecture)

                storename = "%s/%s_%s_contents_%s" % (Options["TempDir"], suite, component, architecture)
                coroutine = genchanges(Options, file + ".diff", storename, file, maxcontents, merged_pdiffs)
                pending_tasks.append(coroutine)

                file = "%s/%s/%s/%s" % (tree, component, longarch, packages)
                storename = "%s/%s_%s_%s" % (Options["TempDir"], suite, component, architecture)
                coroutine = genchanges(Options, file + ".diff", storename, file, maxsuite, merged_pdiffs)
                pending_tasks.append(coroutine)

    asyncio.run(process_pdiff_tasks(pending_tasks, max_parallel))
Esempio n. 9
0
def main():
    global Cnf, Options, Logger

    os.umask(0o002)

    Cnf = utils.get_conf()
    Arguments = [ ('h', "help", "Generate-Index-Diffs::Options::Help"),
                  ('c', None, "Generate-Index-Diffs::Options::CanonicalPath", "hasArg"),
                  ('p', "patchname", "Generate-Index-Diffs::Options::PatchName", "hasArg"),
                  ('r', "rootdir", "Generate-Index-Diffs::Options::RootDir", "hasArg"),
                  ('d', "tmpdir", "Generate-Index-Diffs::Options::TempDir", "hasArg"),
                  ('m', "maxdiffs", "Generate-Index-Diffs::Options::MaxDiffs", "hasArg"),
                  ('n', "n-act", "Generate-Index-Diffs::Options::NoAct"),
                ]
    suites = apt_pkg.parse_commandline(Cnf,Arguments,sys.argv)
    Options = Cnf.subtree("Generate-Index-Diffs::Options")
    if Options.has_key("Help"): usage()

    maxdiffs = Options.get("MaxDiffs::Default", "56")
    maxpackages = Options.get("MaxDiffs::Packages", maxdiffs)
    maxcontents = Options.get("MaxDiffs::Contents", maxdiffs)
    maxsources = Options.get("MaxDiffs::Sources", maxdiffs)

    if not Options.has_key("PatchName"):
        format = "%Y-%m-%d-%H%M.%S"
        Options["PatchName"] = time.strftime( format )

    AptCnf = apt_pkg.newConfiguration()
    apt_pkg.ReadConfigFileISC(AptCnf,utils.which_apt_conf_file())

    if Options.has_key("RootDir"):
        Cnf["Dir::Root"] = Options["RootDir"]

    if not suites:
        suites = Cnf.subtree("Suite").list()

    for suitename in suites:
        print "Processing: " + suitename
        SuiteBlock = Cnf.subtree("Suite::" + suitename)

        suiteobj = get_suite(suitename.lower())

        # Use the canonical version of the suite name
        suite = suiteobj.suite_name

        if suiteobj.untouchable:
            print "Skipping: " + suite + " (untouchable)"
            continue

        architectures = get_suite_architectures(suite, skipall=True)

        if SuiteBlock.has_key("Components"):
            components = SuiteBlock.value_list("Components")
        else:
            components = []

        suite_suffix = Cnf.find("Dinstall::SuiteSuffix")
        if components and suite_suffix:
            longsuite = suite + "/" + suite_suffix
        else:
            longsuite = suite

        tree = SuiteBlock.get("Tree", "dists/%s" % (longsuite))

        if AptCnf.has_key("tree::%s" % (tree)):
            sections = AptCnf["tree::%s::Sections" % (tree)].split()
        elif AptCnf.has_key("bindirectory::%s" % (tree)):
            sections = AptCnf["bindirectory::%s::Sections" % (tree)].split()
        else:
            aptcnf_filename = os.path.basename(utils.which_apt_conf_file())
            print "ALERT: suite %s not in %s, nor untouchable!" % (suite, aptcnf_filename)
            continue

        # See if there are Translations which might need a new pdiff
        cwd = os.getcwd()
        for component in sections:
            #print "DEBUG: Working on %s" % (component)
            workpath=os.path.join(Cnf["Dir::Root"], tree, component, "i18n")
            if os.path.isdir(workpath):
                os.chdir(workpath)
                for dirpath, dirnames, filenames in os.walk(".", followlinks=True, topdown=True):
                    for entry in filenames:
                        if not re_includeinpdiff.match(entry):
                            #print "EXCLUDING %s" % (entry)
                            continue
                        (fname, fext) = os.path.splitext(entry)
                        processfile=os.path.join(workpath, fname)
                        #print "Working: %s" % (processfile)
                        storename="%s/%s_%s_%s" % (Options["TempDir"], suite, component, fname)
                        #print "Storefile: %s" % (storename)
                        genchanges(Options, processfile + ".diff", storename, processfile, maxdiffs)
        os.chdir(cwd)

        for archobj in architectures:
            architecture = archobj.arch_string

            # use sections instead of components since dak.conf
            # treats "foo/bar main" as suite "foo", suitesuffix "bar" and
            # component "bar/main". suck.

            for component in sections:
                if architecture == "source":
                    longarch = architecture
                    packages = "Sources"
                    maxsuite = maxsources
                else:
                    longarch = "binary-%s"% (architecture)
                    packages = "Packages"
                    maxsuite = maxpackages
                    # Process Contents
                    file = "%s/%s/Contents-%s" % (Cnf["Dir::Root"] + tree, component,
                            architecture)
                    storename = "%s/%s_%s_contents_%s" % (Options["TempDir"], suite, component, architecture)
                    genchanges(Options, file + ".diff", storename, file, \
                      Cnf.get("Suite::%s::Generate-Index-Diffs::MaxDiffs::Contents" % (suite), maxcontents))

                file = "%s/%s/%s/%s" % (Cnf["Dir::Root"] + tree,
                           component, longarch, packages)
                storename = "%s/%s_%s_%s" % (Options["TempDir"], suite, component, architecture)
                genchanges(Options, file + ".diff", storename, file, \
                  Cnf.get("Suite::%s::Generate-Index-Diffs::MaxDiffs::%s" % (suite, packages), maxsuite))