def Main(): options = ParseArguments() repoman = RepositoryManager(options.repos_dir) repoproc = RepositoryProcessor(repoman, options.statedir) if options.list: print('\n'.join(repoman.GetNames(reponames=options.reponames))) return 0 transformer = PackageTransformer(repoman, options.rules_dir) logger = StderrLogger() if options.logfile: logger = FileLogger(options.logfile) repositories_updated = [] repositories_not_updated = [] start = timer() if options.fetch or options.parse or options.reprocess: repositories_updated, repositories_not_updated = ProcessRepositories(options=options, logger=logger, repoproc=repoproc, transformer=transformer, reponames=repoman.GetNames(reponames=options.reponames)) if options.initdb or options.database: ProcessDatabase(options=options, logger=logger, repoproc=repoproc, repositories_updated=repositories_updated) if (options.parse or options.reprocess) and (options.show_unmatched_rules): ShowUnmatchedRules(options=options, logger=logger, transformer=transformer, reliable=repositories_not_updated == []) logger.Log('total time taken: {:.2f} seconds'.format((timer() - start))) return 1 if repositories_not_updated else 0
def Main(): parser = ArgumentParser() parser.add_argument('-s', '--statedir', help='path to directory with repository state') parser.add_argument('-U', '--rules', default='rules.yaml', help='path to name transformation rules yaml') parser.add_argument('-l', '--logfile', help='path to log file') parser.add_argument('-t', '--tag', action='append', help='only process repositories with this tag') parser.add_argument('-r', '--repository', action='append', help='only process repositories with this name') parser.add_argument('-S', '--no-shadow', action='store_true', help='treat shadow repositories as normal') parser.add_argument('-o', '--output', help='path to output directory') options = parser.parse_args() if not options.statedir: raise RuntimeError("please set --statedir") if not options.tag and not options.repository: raise RuntimeError("please set --tag or --repository") logger = StderrLogger() if options.logfile: logger = FileLogger(options.logfile) tags = [tag.split(',') for tag in options.tag] if options.tag else [] nametrans = NameTransformer(options.rules) repoman = RepositoryManager(options.statedir, enable_shadow=not options.no_shadow) packages = repoman.Deserialize(nametrans, tags=tags, repositories=options.repository, logger=logger) RepologyOrg(options.output, packages, repoman.GetNames(tags=tags, repositories=options.repository), repoman.GetMetadata(), logger) unmatched = nametrans.GetUnmatchedRules() if len(unmatched): wlogger = logger.GetPrefixed("WARNING: ") wlogger.Log("unmatched rules detected!") for rule in unmatched: wlogger.Log(rule) return 0
def Main(): parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('-S', '--statedir', default=repology.config.STATE_DIR, help='path to directory with repository state') parser.add_argument('-L', '--logfile', help='path to log file (log to stderr by default)') parser.add_argument('-E', '--repos-dir', default=repology.config.REPOS_DIR, help='path to directory with repository configs') parser.add_argument('-U', '--rules-dir', default=repology.config.RULES_DIR, help='path to directory with rules') parser.add_argument('-D', '--dsn', default=repology.config.DSN, help='database connection params') actions_grp = parser.add_argument_group('Actions') actions_grp.add_argument('-l', '--list', action='store_true', help='list repositories repology will work on') actions_grp.add_argument('-f', '--fetch', action='store_true', help='fetching repository data') actions_grp.add_argument('-u', '--update', action='store_true', help='when fetching, allow updating (otherwise, only fetch once)') actions_grp.add_argument('-p', '--parse', action='store_true', help='parse, process and serialize repository data') # XXX: this is dangerous as long as ignored packages are removed from dumps actions_grp.add_argument('-P', '--reprocess', action='store_true', help='reprocess repository data') actions_grp.add_argument('-i', '--initdb', action='store_true', help='(re)initialize database schema') actions_grp.add_argument('-d', '--database', action='store_true', help='store in the database') actions_grp.add_argument('-r', '--show-unmatched-rules', action='store_true', help='show unmatched rules when parsing') parser.add_argument('reponames', default=repology.config.REPOSITORIES, metavar='repo|tag', nargs='*', help='repository or tag name to process') options = parser.parse_args() repoman = RepositoryManager(options.repos_dir, options.statedir) if options.list: print('\n'.join(sorted(repoman.GetNames(reponames=options.reponames)))) return 0 transformer = PackageTransformer(options.rules_dir) logger = StderrLogger() if options.logfile: logger = FileLogger(options.logfile) repositories_updated = [] repositories_not_updated = [] start = timer() if options.fetch or options.parse or options.reprocess: repositories_updated, repositories_not_updated = ProcessRepositories(options=options, logger=logger, repoman=repoman, transformer=transformer) if options.initdb or options.database: ProcessDatabase(options=options, logger=logger, repoman=repoman, repositories_updated=repositories_updated) if (options.parse or options.reprocess) and (options.show_unmatched_rules): ShowUnmatchedRules(options=options, logger=logger, transformer=transformer, reliable=repositories_not_updated == []) logger.Log('total time taken: {:.2f} seconds'.format((timer() - start))) return 1 if repositories_not_updated else 0
def Main(): parser = ArgumentParser() parser.add_argument('-s', '--statedir', help='path to directory with repository state') parser.add_argument('-U', '--rules', default='rules.yaml', help='path to name transformation rules yaml') parser.add_argument('-l', '--logfile', help='path to log file') parser.add_argument('-t', '--tag', action='append', help='only process repositories with this tag') parser.add_argument('-r', '--repository', action='append', help='only process repositories with this name') parser.add_argument('-S', '--no-shadow', action='store_true', help='treat shadow repositories as normal') parser.add_argument('-m', '--maintainer', help='filter by maintainer') parser.add_argument('-c', '--category', help='filter by category') parser.add_argument('-n', '--little-repos', help='filter by number of repos') parser.add_argument('-N', '--many-repos', help='filter by number of repos') parser.add_argument('-i', '--in-repository', help='filter by presence in repository') parser.add_argument('-x', '--not-in-repository', help='filter by absence in repository') parser.add_argument('-O', '--outdated-in-repository', help='filter by outdatedness in repository') parser.add_argument('-o', '--output', help='path to output file') options = parser.parse_args() if not options.statedir: raise RuntimeError("please set --statedir") if not options.tag and not options.repository: raise RuntimeError("please set --tag or --repository") logger = StderrLogger() if options.logfile: logger = FileLogger(options.logfile) tags = [tag.split(',') for tag in options.tag] if options.tag else [] nametrans = NameTransformer(options.rules) repoman = RepositoryManager(options.statedir, enable_shadow=not options.no_shadow) packages = repoman.Deserialize(nametrans, tags=tags, repositories=options.repository, logger=logger) packages = FilterPackages( packages, options.maintainer, options.category, int(options.many_repos) if options.many_repos is not None else None, int(options.little_repos) if options.little_repos is not None else None, options.in_repository, options.not_in_repository, options.outdated_in_repository) template = Template() rp = ReportProducer(template, "table.html") rp.RenderToFile(options.output, packages, repoman.GetNames(tags=tags, repositories=options.repository), repositories=repoman.GetMetadata()) unmatched = nametrans.GetUnmatchedRules() if len(unmatched): wlogger = logger.GetPrefixed("WARNING: ") wlogger.Log("unmatched rules detected!") for rule in unmatched: wlogger.Log(rule) return 0
# (at your option) any later version. # # repology is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with repology. If not, see <http://www.gnu.org/licenses/>. import flask from repology.config import config from repology.database import Database from repology.repoman import RepositoryManager __all__ = ['get_db', 'repometadata', 'reponames'] repoman = RepositoryManager(config['REPOS_DIR'], 'dummy') repometadata = repoman.GetMetadata(config['REPOSITORIES']) reponames = repoman.GetNames(config['REPOSITORIES']) def get_db(): # XXX: this is not really a persistent DB connection! if not hasattr(flask.g, 'database'): flask.g.database = Database(config['DSN'], readonly=False, autocommit=True) return flask.g.database
def Main(): parser = ArgumentParser() parser.add_argument('-s', '--statedir', help='path to directory with repository state') parser.add_argument('-l', '--logfile', help='path to log file') parser.add_argument('-f', '--fetch', action='store_true', help='allow fetching repository data') parser.add_argument('-u', '--update', action='store_true', help='allow updating repository data') parser.add_argument('-p', '--parse', action='store_true', help='parse and serialize repository data') parser.add_argument('-t', '--tag', action='append', help='only process repositories with this tag') parser.add_argument('-r', '--repository', action='append', help='only process repositories with this name') options = parser.parse_args() if not options.statedir: raise RuntimeError("please set --statedir") if not options.tag and not options.repository: raise RuntimeError("please set --tag or --repository") logger = StderrLogger() if options.logfile: logger = FileLogger(options.logfile) repoman = RepositoryManager(options.statedir) total_count = 0 success_count = 0 for reponame in repoman.GetNames(tags=options.tag, repositories=options.repository): repo_logger = logger.GetPrefixed(reponame + ": ") repo_logger.Log("processing started") try: if options.fetch: repoman.FetchOne(reponame, update=options.update, logger=repo_logger.GetIndented()) if options.parse: repoman.ParseAndSerializeOne(reponame, logger=repo_logger.GetIndented()) except KeyboardInterrupt: logger.Log("processing interrupted") return 1 except: repo_logger.Log("processing failed, exception follows") for item in traceback.format_exception(*sys.exc_info()): for line in item.split('\n'): if line: repo_logger.GetIndented().Log(line) else: repo_logger.Log("processing complete") success_count += 1 total_count += 1 logger.Log("{}/{} repositories processed successfully".format( success_count, total_count)) return 0 if success_count == total_count else 1