def Main():
options = ParseArguments()
repoman = RepositoryManager(options.repos_dir)
repoproc = RepositoryProcessor(repoman, options.statedir)
if options.list:
print('\n'.join(repoman.GetNames(reponames=options.reponames)))
return 0
transformer = PackageTransformer(repoman, options.rules_dir)
logger = StderrLogger()
if options.logfile:
logger = FileLogger(options.logfile)
repositories_updated = []
repositories_not_updated = []
start = timer()
if options.fetch or options.parse or options.reprocess:
repositories_updated, repositories_not_updated = ProcessRepositories(options=options, logger=logger, repoproc=repoproc, transformer=transformer, reponames=repoman.GetNames(reponames=options.reponames))
if options.initdb or options.database:
ProcessDatabase(options=options, logger=logger, repoproc=repoproc, repositories_updated=repositories_updated)
if (options.parse or options.reprocess) and (options.show_unmatched_rules):
ShowUnmatchedRules(options=options, logger=logger, transformer=transformer, reliable=repositories_not_updated == [])
logger.Log('total time taken: {:.2f} seconds'.format((timer() - start)))
return 1 if repositories_not_updated else 0
def Main():
parser = ArgumentParser()
parser.add_argument('-s',
'--statedir',
help='path to directory with repository state')
parser.add_argument('-U',
'--rules',
default='rules.yaml',
help='path to name transformation rules yaml')
parser.add_argument('-l', '--logfile', help='path to log file')
parser.add_argument('-t',
'--tag',
action='append',
help='only process repositories with this tag')
parser.add_argument('-r',
'--repository',
action='append',
help='only process repositories with this name')
parser.add_argument('-S',
'--no-shadow',
action='store_true',
help='treat shadow repositories as normal')
parser.add_argument('-o', '--output', help='path to output directory')
options = parser.parse_args()
if not options.statedir:
raise RuntimeError("please set --statedir")
if not options.tag and not options.repository:
raise RuntimeError("please set --tag or --repository")
logger = StderrLogger()
if options.logfile:
logger = FileLogger(options.logfile)
tags = [tag.split(',') for tag in options.tag] if options.tag else []
nametrans = NameTransformer(options.rules)
repoman = RepositoryManager(options.statedir,
enable_shadow=not options.no_shadow)
packages = repoman.Deserialize(nametrans,
tags=tags,
repositories=options.repository,
logger=logger)
RepologyOrg(options.output, packages,
repoman.GetNames(tags=tags, repositories=options.repository),
repoman.GetMetadata(), logger)
unmatched = nametrans.GetUnmatchedRules()
if len(unmatched):
wlogger = logger.GetPrefixed("WARNING: ")
wlogger.Log("unmatched rules detected!")
for rule in unmatched:
wlogger.Log(rule)
return 0
def Main():
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('-S', '--statedir', default=repology.config.STATE_DIR, help='path to directory with repository state')
parser.add_argument('-L', '--logfile', help='path to log file (log to stderr by default)')
parser.add_argument('-E', '--repos-dir', default=repology.config.REPOS_DIR, help='path to directory with repository configs')
parser.add_argument('-U', '--rules-dir', default=repology.config.RULES_DIR, help='path to directory with rules')
parser.add_argument('-D', '--dsn', default=repology.config.DSN, help='database connection params')
actions_grp = parser.add_argument_group('Actions')
actions_grp.add_argument('-l', '--list', action='store_true', help='list repositories repology will work on')
actions_grp.add_argument('-f', '--fetch', action='store_true', help='fetching repository data')
actions_grp.add_argument('-u', '--update', action='store_true', help='when fetching, allow updating (otherwise, only fetch once)')
actions_grp.add_argument('-p', '--parse', action='store_true', help='parse, process and serialize repository data')
# XXX: this is dangerous as long as ignored packages are removed from dumps
actions_grp.add_argument('-P', '--reprocess', action='store_true', help='reprocess repository data')
actions_grp.add_argument('-i', '--initdb', action='store_true', help='(re)initialize database schema')
actions_grp.add_argument('-d', '--database', action='store_true', help='store in the database')
actions_grp.add_argument('-r', '--show-unmatched-rules', action='store_true', help='show unmatched rules when parsing')
parser.add_argument('reponames', default=repology.config.REPOSITORIES, metavar='repo|tag', nargs='*', help='repository or tag name to process')
options = parser.parse_args()
repoman = RepositoryManager(options.repos_dir, options.statedir)
if options.list:
print('\n'.join(sorted(repoman.GetNames(reponames=options.reponames))))
return 0
transformer = PackageTransformer(options.rules_dir)
logger = StderrLogger()
if options.logfile:
logger = FileLogger(options.logfile)
repositories_updated = []
repositories_not_updated = []
start = timer()
if options.fetch or options.parse or options.reprocess:
repositories_updated, repositories_not_updated = ProcessRepositories(options=options, logger=logger, repoman=repoman, transformer=transformer)
if options.initdb or options.database:
ProcessDatabase(options=options, logger=logger, repoman=repoman, repositories_updated=repositories_updated)
if (options.parse or options.reprocess) and (options.show_unmatched_rules):
ShowUnmatchedRules(options=options, logger=logger, transformer=transformer, reliable=repositories_not_updated == [])
logger.Log('total time taken: {:.2f} seconds'.format((timer() - start)))
return 1 if repositories_not_updated else 0
def Main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('-S',
'--statedir',
default=repology.config.STATE_DIR,
help='path to directory with repository state')
parser.add_argument('-L',
'--logfile',
help='path to log file (log to stderr by default)')
parser.add_argument('-E',
'--repos-dir',
default=repology.config.REPOS_DIR,
help='path directory with reposotory configs')
parser.add_argument('-M',
'--mode',
choices=['batch', 'stream'],
default='stream',
help='processing mode')
filters_grp = parser.add_argument_group('Filters')
filters_grp.add_argument('--no-shadow',
action='store_true',
help='treat shadow repositories as normal')
filters_grp.add_argument('--maintainer', help='filter by maintainer')
filters_grp.add_argument('--category', help='filter by category')
filters_grp.add_argument('--less-repos', help='filter by number of repos')
filters_grp.add_argument('--more-repos', help='filter by number of repos')
filters_grp.add_argument('--in-repository',
help='filter by presence in repository')
filters_grp.add_argument('--not-in-repository',
help='filter by absence in repository')
filters_grp.add_argument('--outdated-in-repository',
help='filter by outdatedness in repository')
parser.add_argument('-D',
'--dump',
choices=['packages', 'summaries'],
default='packages',
help='dump mode')
parser.add_argument('-f',
'--fields',
default='repo,effname,version',
help='fields to list for the package')
parser.add_argument('-s',
'--field-separator',
default=' ',
help='field separator')
parser.add_argument('reponames',
default=repology.config.REPOSITORIES,
metavar='repo|tag',
nargs='*',
help='repository or tag name to process')
options = parser.parse_args()
logger = StderrLogger()
if options.logfile:
logger = FileLogger(options.logfile)
if options.fields == 'all':
options.fields = sorted(Package().__dict__.keys())
else:
options.fields = options.fields.split(',')
# Set up filters
filters = []
if options.maintainer:
filters.append(MaintainerFilter(options.maintainer))
if options.category:
filters.append(CategoryFilter(options.maintainer))
if options.more_repos is not None or options.less_repos is not None:
filters.append(
FamilyCountFilter(more=options.more_repos,
less=options.less_repos))
if options.in_repository:
filters.append(InRepoFilter(options.in_repository))
if options.not_in_repository:
filters.append(NotInRepoFilter(options.not_in_repository))
if options.outdated_in_repository:
filters.append(OutdatedInRepoFilter(options.not_in_repository))
if not options.no_shadow:
filters.append(ShadowFilter())
repoman = RepositoryManager(options.repos_dir, options.statedir)
def PackageProcessor(packageset):
FillPackagesetVersions(packageset)
if not PackagesetCheckFilters(packageset, *filters):
return
if options.dump == 'packages':
for package in packageset:
print(
options.field_separator.join([
str(getattr(package, field))
for field in options.fields
]))
if options.dump == 'summaries':
print(packageset[0].effname)
summaries = PackagesetToSummaries(packageset)
for reponame in repoman.GetNames(options.reponames):
if reponame in summaries:
print(' {}: {} ({}) *{}'.format(
reponame,
summaries[reponame]['version'],
RepositoryVersionClass2Letter(
summaries[reponame]['versionclass']),
summaries[reponame]['numpackages'],
))
if options.mode == 'stream':
logger.Log('dumping...')
repoman.StreamDeserializeMulti(processor=PackageProcessor,
reponames=options.reponames)
else:
logger.Log('loading packages...')
all_packages = repoman.DeserializeMulti(reponames=options.reponames,
logger=logger)
logger.Log('merging packages...')
metapackages = PackagesToMetapackages(all_packages)
logger.Log('dumping...')
for metapackage_name, packageset in sorted(metapackages.items()):
PackageProcessor(packageset)
return 0
# repology is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with repology. If not, see <http://www.gnu.org/licenses/>.
import unittest
from repology.config import config
from repology.repoman import RepositoryManager
from repology.repoproc import RepositoryProcessor
repoman = RepositoryManager(config['REPOS_DIR'])
repoproc = RepositoryProcessor(repoman, 'testdata')
class TestParsers(unittest.TestCase):
def setUp(self):
self.maxDiff = None
self.packages = repoproc.ParseMulti(reponames=['have_testdata'])
def check_package(self, name, reference):
reference_with_default = {
# repo must be filled
# family must be filled
'subrepo': None,
# name must be filled
def Main():
parser = ArgumentParser()
parser.add_argument('-s',
'--statedir',
help='path to directory with repository state')
parser.add_argument('-U',
'--rules',
default='rules.yaml',
help='path to name transformation rules yaml')
parser.add_argument('-l', '--logfile', help='path to log file')
parser.add_argument('-t',
'--tag',
action='append',
help='only process repositories with this tag')
parser.add_argument('-r',
'--repository',
action='append',
help='only process repositories with this name')
parser.add_argument('-S',
'--no-shadow',
action='store_true',
help='treat shadow repositories as normal')
parser.add_argument('-m', '--maintainer', help='filter by maintainer')
parser.add_argument('-c', '--category', help='filter by category')
parser.add_argument('-n',
'--little-repos',
help='filter by number of repos')
parser.add_argument('-N', '--many-repos', help='filter by number of repos')
parser.add_argument('-i',
'--in-repository',
help='filter by presence in repository')
parser.add_argument('-x',
'--not-in-repository',
help='filter by absence in repository')
parser.add_argument('-O',
'--outdated-in-repository',
help='filter by outdatedness in repository')
parser.add_argument('-o', '--output', help='path to output file')
options = parser.parse_args()
if not options.statedir:
raise RuntimeError("please set --statedir")
if not options.tag and not options.repository:
raise RuntimeError("please set --tag or --repository")
logger = StderrLogger()
if options.logfile:
logger = FileLogger(options.logfile)
tags = [tag.split(',') for tag in options.tag] if options.tag else []
nametrans = NameTransformer(options.rules)
repoman = RepositoryManager(options.statedir,
enable_shadow=not options.no_shadow)
packages = repoman.Deserialize(nametrans,
tags=tags,
repositories=options.repository,
logger=logger)
packages = FilterPackages(
packages, options.maintainer, options.category,
int(options.many_repos) if options.many_repos is not None else None,
int(options.little_repos)
if options.little_repos is not None else None, options.in_repository,
options.not_in_repository, options.outdated_in_repository)
template = Template()
rp = ReportProducer(template, "table.html")
rp.RenderToFile(options.output,
packages,
repoman.GetNames(tags=tags,
repositories=options.repository),
repositories=repoman.GetMetadata())
unmatched = nametrans.GetUnmatchedRules()
if len(unmatched):
wlogger = logger.GetPrefixed("WARNING: ")
wlogger.Log("unmatched rules detected!")
for rule in unmatched:
wlogger.Log(rule)
return 0
# (at your option) any later version.
#
# repology is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with repology. If not, see <http://www.gnu.org/licenses/>.
import flask
from repology.config import config
from repology.database import Database
from repology.repoman import RepositoryManager
__all__ = ['get_db', 'repometadata', 'reponames']
repoman = RepositoryManager(config['REPOS_DIR'], 'dummy')
repometadata = repoman.GetMetadata(config['REPOSITORIES'])
reponames = repoman.GetNames(config['REPOSITORIES'])
def get_db():
# XXX: this is not really a persistent DB connection!
if not hasattr(flask.g, 'database'):
flask.g.database = Database(config['DSN'],
readonly=False,
autocommit=True)
return flask.g.database
from repology.packageproc import *
from repology.queryfilters import *
from repology.repoman import RepositoryManager
from repology.template_helpers import *
from repology.version import VersionCompare
# create application and handle configuration
app = flask.Flask(__name__)
app.config.from_pyfile('repology.conf.default')
app.config.from_pyfile('repology.conf', silent=True)
app.config.from_envvar('REPOLOGY_CONFIG', silent=True)
# global repology objects
repoman = RepositoryManager(
app.config['REPOS_DIR'],
'dummy') # XXX: should not construct fetchers and parsers here
repometadata = repoman.GetMetadata(app.config['REPOSITORIES'])
reponames = repoman.GetNames(app.config['REPOSITORIES'])
# templates: tuning
app.jinja_env.trim_blocks = True
app.jinja_env.lstrip_blocks = True
# templates: custom filters
app.jinja_env.filters['pkg_format'] = pkg_format
app.jinja_env.filters[
'css_for_package_versionclass'] = css_for_package_versionclass
app.jinja_env.filters[
'css_for_summary_versionclass'] = css_for_summary_versionclass
app.jinja_env.filters['maintainer_to_links'] = maintainer_to_links
def setUp(self):
self.maxDiff = None
repoman = RepositoryManager(repology.config.REPOS_DIR, 'testdata')
self.packages = repoman.ParseMulti(reponames=['have_testdata'])
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with repology. If not, see <http://www.gnu.org/licenses/>.
import unittest
from repology.package import Package, PackageFlags
from repology.repoman import RepositoryManager
from repology.transformer import PackageTransformer
repoman = RepositoryManager(repostext="""
[
{ name: foo, desc: foo, family: foo, sources: [] },
{ name: bar, desc: bar, family: bar, sources: [] },
{ name: baz, desc: baz, family: baz, sources: [] }
]
""")
class TestPackageTransformer(unittest.TestCase):
def check_transformer(self, rulestext, *packages):
transformer = PackageTransformer(repoman, rulestext=rulestext)
for packagedict in packages:
create_params = {}
expected_params = {}
for field, value in packagedict.items():
if field.startswith('expect_'):
expected_params[field[7:]] = value
else:
def Main():
options = ParseArguments()
logger = StderrLogger()
if options.logfile:
logger = FileLogger(options.logfile)
if options.fields == 'all':
options.fields = sorted(Package().__dict__.keys())
else:
options.fields = options.fields.split(',')
# Set up filters
filters = []
if options.maintainer:
filters.append(MaintainerFilter(options.maintainer))
if options.category:
filters.append(CategoryFilter(options.maintainer))
if options.more_repos is not None or options.less_repos is not None:
filters.append(FamilyCountFilter(more=options.more_repos, less=options.less_repos))
if options.in_repository:
filters.append(InRepoFilter(options.in_repository))
if options.not_in_repository:
filters.append(NotInRepoFilter(options.not_in_repository))
if options.outdated_in_repository:
filters.append(OutdatedInRepoFilter(options.not_in_repository))
if not options.no_shadow:
filters.append(ShadowFilter())
repoman = RepositoryManager(options.repos_dir)
repoproc = RepositoryProcessor(repoman, options.statedir)
def PackageProcessor(packageset):
FillPackagesetVersions(packageset)
if not PackagesetCheckFilters(packageset, *filters):
return
if options.dump == 'packages':
for package in packageset:
print(
options.field_separator.join(
[
VersionClass.ToString(package.versionclass) if field == 'versionclass' else str(getattr(package, field))
for field in options.fields
]
)
)
if options.dump == 'summaries':
print(packageset[0].effname)
best_pkg_by_repo = PackagesetToBestByRepo(packageset)
for reponame in repoman.GetNames(options.reponames):
if reponame in best_pkg_by_repo:
print(' {}: {} ({})'.format(
reponame,
best_pkg_by_repo[reponame].version,
VersionClass.ToString(best_pkg_by_repo[reponame].versionclass)
))
logger.Log('dumping...')
repoproc.StreamDeserializeMulti(processor=PackageProcessor, reponames=options.reponames)
return 0
def Main():
parser = ArgumentParser()
parser.add_argument('-s',
'--statedir',
help='path to directory with repository state')
parser.add_argument('-l', '--logfile', help='path to log file')
parser.add_argument('-f',
'--fetch',
action='store_true',
help='allow fetching repository data')
parser.add_argument('-u',
'--update',
action='store_true',
help='allow updating repository data')
parser.add_argument('-p',
'--parse',
action='store_true',
help='parse and serialize repository data')
parser.add_argument('-t',
'--tag',
action='append',
help='only process repositories with this tag')
parser.add_argument('-r',
'--repository',
action='append',
help='only process repositories with this name')
options = parser.parse_args()
if not options.statedir:
raise RuntimeError("please set --statedir")
if not options.tag and not options.repository:
raise RuntimeError("please set --tag or --repository")
logger = StderrLogger()
if options.logfile:
logger = FileLogger(options.logfile)
repoman = RepositoryManager(options.statedir)
total_count = 0
success_count = 0
for reponame in repoman.GetNames(tags=options.tag,
repositories=options.repository):
repo_logger = logger.GetPrefixed(reponame + ": ")
repo_logger.Log("processing started")
try:
if options.fetch:
repoman.FetchOne(reponame,
update=options.update,
logger=repo_logger.GetIndented())
if options.parse:
repoman.ParseAndSerializeOne(reponame,
logger=repo_logger.GetIndented())
except KeyboardInterrupt:
logger.Log("processing interrupted")
return 1
except:
repo_logger.Log("processing failed, exception follows")
for item in traceback.format_exception(*sys.exc_info()):
for line in item.split('\n'):
if line:
repo_logger.GetIndented().Log(line)
else:
repo_logger.Log("processing complete")
success_count += 1
total_count += 1
logger.Log("{}/{} repositories processed successfully".format(
success_count, total_count))
return 0 if success_count == total_count else 1