def Main():
options = ParseArguments()
repoman = RepositoryManager(options.repos_dir)
repoproc = RepositoryProcessor(repoman, options.statedir)
if options.list:
print('\n'.join(repoman.GetNames(reponames=options.reponames)))
return 0
transformer = PackageTransformer(repoman, options.rules_dir)
logger = StderrLogger()
if options.logfile:
logger = FileLogger(options.logfile)
repositories_updated = []
repositories_not_updated = []
start = timer()
if options.fetch or options.parse or options.reprocess:
repositories_updated, repositories_not_updated = ProcessRepositories(options=options, logger=logger, repoproc=repoproc, transformer=transformer, reponames=repoman.GetNames(reponames=options.reponames))
if options.initdb or options.database:
ProcessDatabase(options=options, logger=logger, repoproc=repoproc, repositories_updated=repositories_updated)
if (options.parse or options.reprocess) and (options.show_unmatched_rules):
ShowUnmatchedRules(options=options, logger=logger, transformer=transformer, reliable=repositories_not_updated == [])
logger.Log('total time taken: {:.2f} seconds'.format((timer() - start)))
return 1 if repositories_not_updated else 0
def Main():
parser = ArgumentParser()
parser.add_argument('-s',
'--statedir',
help='path to directory with repository state')
parser.add_argument('-U',
'--rules',
default='rules.yaml',
help='path to name transformation rules yaml')
parser.add_argument('-l', '--logfile', help='path to log file')
parser.add_argument('-t',
'--tag',
action='append',
help='only process repositories with this tag')
parser.add_argument('-r',
'--repository',
action='append',
help='only process repositories with this name')
parser.add_argument('-S',
'--no-shadow',
action='store_true',
help='treat shadow repositories as normal')
parser.add_argument('-o', '--output', help='path to output directory')
options = parser.parse_args()
if not options.statedir:
raise RuntimeError("please set --statedir")
if not options.tag and not options.repository:
raise RuntimeError("please set --tag or --repository")
logger = StderrLogger()
if options.logfile:
logger = FileLogger(options.logfile)
tags = [tag.split(',') for tag in options.tag] if options.tag else []
nametrans = NameTransformer(options.rules)
repoman = RepositoryManager(options.statedir,
enable_shadow=not options.no_shadow)
packages = repoman.Deserialize(nametrans,
tags=tags,
repositories=options.repository,
logger=logger)
RepologyOrg(options.output, packages,
repoman.GetNames(tags=tags, repositories=options.repository),
repoman.GetMetadata(), logger)
unmatched = nametrans.GetUnmatchedRules()
if len(unmatched):
wlogger = logger.GetPrefixed("WARNING: ")
wlogger.Log("unmatched rules detected!")
for rule in unmatched:
wlogger.Log(rule)
return 0
def Main():
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('-S', '--statedir', default=repology.config.STATE_DIR, help='path to directory with repository state')
parser.add_argument('-L', '--logfile', help='path to log file (log to stderr by default)')
parser.add_argument('-E', '--repos-dir', default=repology.config.REPOS_DIR, help='path to directory with repository configs')
parser.add_argument('-U', '--rules-dir', default=repology.config.RULES_DIR, help='path to directory with rules')
parser.add_argument('-D', '--dsn', default=repology.config.DSN, help='database connection params')
actions_grp = parser.add_argument_group('Actions')
actions_grp.add_argument('-l', '--list', action='store_true', help='list repositories repology will work on')
actions_grp.add_argument('-f', '--fetch', action='store_true', help='fetching repository data')
actions_grp.add_argument('-u', '--update', action='store_true', help='when fetching, allow updating (otherwise, only fetch once)')
actions_grp.add_argument('-p', '--parse', action='store_true', help='parse, process and serialize repository data')
# XXX: this is dangerous as long as ignored packages are removed from dumps
actions_grp.add_argument('-P', '--reprocess', action='store_true', help='reprocess repository data')
actions_grp.add_argument('-i', '--initdb', action='store_true', help='(re)initialize database schema')
actions_grp.add_argument('-d', '--database', action='store_true', help='store in the database')
actions_grp.add_argument('-r', '--show-unmatched-rules', action='store_true', help='show unmatched rules when parsing')
parser.add_argument('reponames', default=repology.config.REPOSITORIES, metavar='repo|tag', nargs='*', help='repository or tag name to process')
options = parser.parse_args()
repoman = RepositoryManager(options.repos_dir, options.statedir)
if options.list:
print('\n'.join(sorted(repoman.GetNames(reponames=options.reponames))))
return 0
transformer = PackageTransformer(options.rules_dir)
logger = StderrLogger()
if options.logfile:
logger = FileLogger(options.logfile)
repositories_updated = []
repositories_not_updated = []
start = timer()
if options.fetch or options.parse or options.reprocess:
repositories_updated, repositories_not_updated = ProcessRepositories(options=options, logger=logger, repoman=repoman, transformer=transformer)
if options.initdb or options.database:
ProcessDatabase(options=options, logger=logger, repoman=repoman, repositories_updated=repositories_updated)
if (options.parse or options.reprocess) and (options.show_unmatched_rules):
ShowUnmatchedRules(options=options, logger=logger, transformer=transformer, reliable=repositories_not_updated == [])
logger.Log('total time taken: {:.2f} seconds'.format((timer() - start)))
return 1 if repositories_not_updated else 0
def Main():
parser = ArgumentParser()
parser.add_argument('-s',
'--statedir',
help='path to directory with repository state')
parser.add_argument('-U',
'--rules',
default='rules.yaml',
help='path to name transformation rules yaml')
parser.add_argument('-l', '--logfile', help='path to log file')
parser.add_argument('-t',
'--tag',
action='append',
help='only process repositories with this tag')
parser.add_argument('-r',
'--repository',
action='append',
help='only process repositories with this name')
parser.add_argument('-S',
'--no-shadow',
action='store_true',
help='treat shadow repositories as normal')
parser.add_argument('-m', '--maintainer', help='filter by maintainer')
parser.add_argument('-c', '--category', help='filter by category')
parser.add_argument('-n',
'--little-repos',
help='filter by number of repos')
parser.add_argument('-N', '--many-repos', help='filter by number of repos')
parser.add_argument('-i',
'--in-repository',
help='filter by presence in repository')
parser.add_argument('-x',
'--not-in-repository',
help='filter by absence in repository')
parser.add_argument('-O',
'--outdated-in-repository',
help='filter by outdatedness in repository')
parser.add_argument('-o', '--output', help='path to output file')
options = parser.parse_args()
if not options.statedir:
raise RuntimeError("please set --statedir")
if not options.tag and not options.repository:
raise RuntimeError("please set --tag or --repository")
logger = StderrLogger()
if options.logfile:
logger = FileLogger(options.logfile)
tags = [tag.split(',') for tag in options.tag] if options.tag else []
nametrans = NameTransformer(options.rules)
repoman = RepositoryManager(options.statedir,
enable_shadow=not options.no_shadow)
packages = repoman.Deserialize(nametrans,
tags=tags,
repositories=options.repository,
logger=logger)
packages = FilterPackages(
packages, options.maintainer, options.category,
int(options.many_repos) if options.many_repos is not None else None,
int(options.little_repos)
if options.little_repos is not None else None, options.in_repository,
options.not_in_repository, options.outdated_in_repository)
template = Template()
rp = ReportProducer(template, "table.html")
rp.RenderToFile(options.output,
packages,
repoman.GetNames(tags=tags,
repositories=options.repository),
repositories=repoman.GetMetadata())
unmatched = nametrans.GetUnmatchedRules()
if len(unmatched):
wlogger = logger.GetPrefixed("WARNING: ")
wlogger.Log("unmatched rules detected!")
for rule in unmatched:
wlogger.Log(rule)
return 0
# (at your option) any later version.
#
# repology is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with repology. If not, see <http://www.gnu.org/licenses/>.
import flask
from repology.config import config
from repology.database import Database
from repology.repoman import RepositoryManager
__all__ = ['get_db', 'repometadata', 'reponames']
repoman = RepositoryManager(config['REPOS_DIR'], 'dummy')
repometadata = repoman.GetMetadata(config['REPOSITORIES'])
reponames = repoman.GetNames(config['REPOSITORIES'])
def get_db():
# XXX: this is not really a persistent DB connection!
if not hasattr(flask.g, 'database'):
flask.g.database = Database(config['DSN'],
readonly=False,
autocommit=True)
return flask.g.database
def Main():
parser = ArgumentParser()
parser.add_argument('-s',
'--statedir',
help='path to directory with repository state')
parser.add_argument('-l', '--logfile', help='path to log file')
parser.add_argument('-f',
'--fetch',
action='store_true',
help='allow fetching repository data')
parser.add_argument('-u',
'--update',
action='store_true',
help='allow updating repository data')
parser.add_argument('-p',
'--parse',
action='store_true',
help='parse and serialize repository data')
parser.add_argument('-t',
'--tag',
action='append',
help='only process repositories with this tag')
parser.add_argument('-r',
'--repository',
action='append',
help='only process repositories with this name')
options = parser.parse_args()
if not options.statedir:
raise RuntimeError("please set --statedir")
if not options.tag and not options.repository:
raise RuntimeError("please set --tag or --repository")
logger = StderrLogger()
if options.logfile:
logger = FileLogger(options.logfile)
repoman = RepositoryManager(options.statedir)
total_count = 0
success_count = 0
for reponame in repoman.GetNames(tags=options.tag,
repositories=options.repository):
repo_logger = logger.GetPrefixed(reponame + ": ")
repo_logger.Log("processing started")
try:
if options.fetch:
repoman.FetchOne(reponame,
update=options.update,
logger=repo_logger.GetIndented())
if options.parse:
repoman.ParseAndSerializeOne(reponame,
logger=repo_logger.GetIndented())
except KeyboardInterrupt:
logger.Log("processing interrupted")
return 1
except:
repo_logger.Log("processing failed, exception follows")
for item in traceback.format_exception(*sys.exc_info()):
for line in item.split('\n'):
if line:
repo_logger.GetIndented().Log(line)
else:
repo_logger.Log("processing complete")
success_count += 1
total_count += 1
logger.Log("{}/{} repositories processed successfully".format(
success_count, total_count))
return 0 if success_count == total_count else 1