def internetarchive(**kw): # pragma: no cover """ Add information about availability on Internet Archive to Source objects. """ add_args = [(("command",), dict(help="download|verify|update"))] kw.setdefault("description", internetarchive.__doc__) args = parsed_args(*add_args, **kw) with transaction.manager: ia_func(args.command, args, kw.get("sources"))
def internetarchive(**kw): # pragma: no cover """ Add information about availability on Internet Archive to Source objects. """ add_args = [(("command",), dict(help="download|verify|update"))] kw.setdefault('description', internetarchive.__doc__) args = parsed_args(*add_args, **kw) with transaction.manager: ia_func(args.command, args, kw.get('sources'))
def prime_cache(args): """If data needs to be denormalized for lookup, do that here. This procedure should be separate from the db initialization, because it will have to be run periodically whenever data has been updated. """ q = DBSession.query(common.Parameter).join(common.ValueSet).distinct() n = q.count() m = DBSession.query(models.Inventory).count() for segment in q: # # TODO: this ratio (number of inventories a segment appears in by number of # distinct segment total) doesn't make much sense, does it? # segment.frequency = float(len(segment.valuesets)) / float(n) segment.in_inventories = len(segment.valuesets) segment.total_inventories = m for inventory in DBSession.query(models.Inventory).options( joinedload_all(common.Contribution.valuesets, common.ValueSet.parameter) ): if '(UPSID)' not in inventory.name: inventory.count_tone = 0 for vs in inventory.valuesets: attr = 'count_' + vs.parameter.segment_class if hasattr(inventory, attr): val = getattr(inventory, attr) or 0 setattr(inventory, attr, val + 1) ficons = cycle(ORDERED_ICONS) gicons = cycle(ORDERED_ICONS) for root, genus in groupby( DBSession.query(models.Genus).order_by(models.Genus.description), lambda g: g.description): ficon = ficons.next().name for g in genus: g.ficon = ficon g.gicon = gicons.next().name for variety in DBSession.query(models.Variety).options( joinedload(models.Variety.inventories)): variety.count_inventories = len(variety.inventories) if astroman: ia_func('update', args) gbs_func('update', args) print('added', add_wikipedia_urls(args), 'wikipedia urls')
def prime_cache(args): """If data needs to be denormalized for lookup, do that here. This procedure should be separate from the db initialization, because it will have to be run periodically whenever data has been updated. """ q = DBSession.query(common.Parameter).join(common.ValueSet).distinct() n = q.count() m = DBSession.query(models.Inventory).count() for segment in q: # # TODO: this ratio (number of inventories a segment appears in by number of # distinct segment total) doesn't make much sense, does it? # segment.frequency = float(len(segment.valuesets)) / float(n) segment.in_inventories = len(segment.valuesets) segment.total_inventories = m for inventory in DBSession.query(models.Inventory).options( joinedload_all(common.Contribution.valuesets, common.ValueSet.parameter)): if '(UPSID)' not in inventory.name: inventory.count_tone = 0 for vs in inventory.valuesets: attr = 'count_' + vs.parameter.segment_class if hasattr(inventory, attr): val = getattr(inventory, attr) or 0 setattr(inventory, attr, val + 1) ficons = cycle(ORDERED_ICONS) gicons = cycle(ORDERED_ICONS) for root, genus in groupby( DBSession.query(models.Genus).order_by(models.Genus.description), lambda g: g.description): ficon = ficons.next().name for g in genus: g.ficon = ficon g.gicon = gicons.next().name for variety in DBSession.query(models.Variety).options( joinedload(models.Variety.inventories)): variety.count_inventories = len(variety.inventories) if 0: ia_func('update', args) gbs_func('update', args) print('added', add_wikipedia_urls(args), 'wikipedia urls')
from clld.scripts.util import parsed_args from clld.scripts.internetarchive import ia_func if __name__ == '__main__': ia_func('update', parsed_args(bootstrap=True))