def internetarchive(**kw): # pragma: no cover
"""
Add information about availability on Internet Archive to Source objects.
"""
add_args = [(("command",), dict(help="download|verify|update"))]
kw.setdefault("description", internetarchive.__doc__)
args = parsed_args(*add_args, **kw)
with transaction.manager:
ia_func(args.command, args, kw.get("sources"))
def internetarchive(**kw): # pragma: no cover
"""
Add information about availability on Internet Archive to Source objects.
"""
add_args = [(("command",), dict(help="download|verify|update"))]
kw.setdefault('description', internetarchive.__doc__)
args = parsed_args(*add_args, **kw)
with transaction.manager:
ia_func(args.command, args, kw.get('sources'))
def prime_cache(args):
"""If data needs to be denormalized for lookup, do that here.
This procedure should be separate from the db initialization, because
it will have to be run periodically whenever data has been updated.
"""
q = DBSession.query(common.Parameter).join(common.ValueSet).distinct()
n = q.count()
m = DBSession.query(models.Inventory).count()
for segment in q:
#
# TODO: this ratio (number of inventories a segment appears in by number of
# distinct segment total) doesn't make much sense, does it?
#
segment.frequency = float(len(segment.valuesets)) / float(n)
segment.in_inventories = len(segment.valuesets)
segment.total_inventories = m
for inventory in DBSession.query(models.Inventory).options(
joinedload_all(common.Contribution.valuesets, common.ValueSet.parameter)
):
if '(UPSID)' not in inventory.name:
inventory.count_tone = 0
for vs in inventory.valuesets:
attr = 'count_' + vs.parameter.segment_class
if hasattr(inventory, attr):
val = getattr(inventory, attr) or 0
setattr(inventory, attr, val + 1)
ficons = cycle(ORDERED_ICONS)
gicons = cycle(ORDERED_ICONS)
for root, genus in groupby(
DBSession.query(models.Genus).order_by(models.Genus.description),
lambda g: g.description):
ficon = ficons.next().name
for g in genus:
g.ficon = ficon
g.gicon = gicons.next().name
for variety in DBSession.query(models.Variety).options(
joinedload(models.Variety.inventories)):
variety.count_inventories = len(variety.inventories)
if astroman:
ia_func('update', args)
gbs_func('update', args)
print('added', add_wikipedia_urls(args), 'wikipedia urls')
def prime_cache(args):
"""If data needs to be denormalized for lookup, do that here.
This procedure should be separate from the db initialization, because
it will have to be run periodically whenever data has been updated.
"""
q = DBSession.query(common.Parameter).join(common.ValueSet).distinct()
n = q.count()
m = DBSession.query(models.Inventory).count()
for segment in q:
#
# TODO: this ratio (number of inventories a segment appears in by number of
# distinct segment total) doesn't make much sense, does it?
#
segment.frequency = float(len(segment.valuesets)) / float(n)
segment.in_inventories = len(segment.valuesets)
segment.total_inventories = m
for inventory in DBSession.query(models.Inventory).options(
joinedload_all(common.Contribution.valuesets,
common.ValueSet.parameter)):
if '(UPSID)' not in inventory.name:
inventory.count_tone = 0
for vs in inventory.valuesets:
attr = 'count_' + vs.parameter.segment_class
if hasattr(inventory, attr):
val = getattr(inventory, attr) or 0
setattr(inventory, attr, val + 1)
ficons = cycle(ORDERED_ICONS)
gicons = cycle(ORDERED_ICONS)
for root, genus in groupby(
DBSession.query(models.Genus).order_by(models.Genus.description),
lambda g: g.description):
ficon = ficons.next().name
for g in genus:
g.ficon = ficon
g.gicon = gicons.next().name
for variety in DBSession.query(models.Variety).options(
joinedload(models.Variety.inventories)):
variety.count_inventories = len(variety.inventories)
if 0:
ia_func('update', args)
gbs_func('update', args)
print('added', add_wikipedia_urls(args), 'wikipedia urls')
from clld.scripts.util import parsed_args
from clld.scripts.internetarchive import ia_func
if __name__ == '__main__':
ia_func('update', parsed_args(bootstrap=True))
from clld.scripts.util import parsed_args
from clld.scripts.internetarchive import ia_func
if __name__ == '__main__':
ia_func('update', parsed_args(bootstrap=True))