Beispiel #1
0
def internetarchive(**kw):  # pragma: no cover
    """
    Add information about availability on Internet Archive to Source objects.
    """
    add_args = [(("command",), dict(help="download|verify|update"))]
    kw.setdefault("description", internetarchive.__doc__)
    args = parsed_args(*add_args, **kw)
    with transaction.manager:
        ia_func(args.command, args, kw.get("sources"))
Beispiel #2
0
def internetarchive(**kw):  # pragma: no cover
    """
    Add information about availability on Internet Archive to Source objects.
    """
    add_args = [(("command",), dict(help="download|verify|update"))]
    kw.setdefault('description', internetarchive.__doc__)
    args = parsed_args(*add_args, **kw)
    with transaction.manager:
        ia_func(args.command, args, kw.get('sources'))
Beispiel #3
0
def prime_cache(args):
    """If data needs to be denormalized for lookup, do that here.
    This procedure should be separate from the db initialization, because
    it will have to be run periodically whenever data has been updated.
    """
    q = DBSession.query(common.Parameter).join(common.ValueSet).distinct()
    n = q.count()
    m = DBSession.query(models.Inventory).count()
    for segment in q:
        #
        # TODO: this ratio (number of inventories a segment appears in by number of
        # distinct segment total) doesn't make much sense, does it?
        #
        segment.frequency = float(len(segment.valuesets)) / float(n)
        segment.in_inventories = len(segment.valuesets)
        segment.total_inventories = m

    for inventory in DBSession.query(models.Inventory).options(
            joinedload_all(common.Contribution.valuesets, common.ValueSet.parameter)
    ):
        if '(UPSID)' not in inventory.name:
            inventory.count_tone = 0

        for vs in inventory.valuesets:
            attr = 'count_' + vs.parameter.segment_class
            if hasattr(inventory, attr):
                val = getattr(inventory, attr) or 0
                setattr(inventory, attr, val + 1)

    ficons = cycle(ORDERED_ICONS)
    gicons = cycle(ORDERED_ICONS)
    for root, genus in groupby(
            DBSession.query(models.Genus).order_by(models.Genus.description),
            lambda g: g.description):
        ficon = ficons.next().name
        for g in genus:
            g.ficon = ficon
            g.gicon = gicons.next().name

    for variety in DBSession.query(models.Variety).options(
            joinedload(models.Variety.inventories)):
        variety.count_inventories = len(variety.inventories)

    if astroman:
        ia_func('update', args)
        gbs_func('update', args)
        print('added', add_wikipedia_urls(args), 'wikipedia urls')
Beispiel #4
0
def prime_cache(args):
    """If data needs to be denormalized for lookup, do that here.
    This procedure should be separate from the db initialization, because
    it will have to be run periodically whenever data has been updated.
    """
    q = DBSession.query(common.Parameter).join(common.ValueSet).distinct()
    n = q.count()
    m = DBSession.query(models.Inventory).count()
    for segment in q:
        #
        # TODO: this ratio (number of inventories a segment appears in by number of
        # distinct segment total) doesn't make much sense, does it?
        #
        segment.frequency = float(len(segment.valuesets)) / float(n)
        segment.in_inventories = len(segment.valuesets)
        segment.total_inventories = m

    for inventory in DBSession.query(models.Inventory).options(
            joinedload_all(common.Contribution.valuesets,
                           common.ValueSet.parameter)):
        if '(UPSID)' not in inventory.name:
            inventory.count_tone = 0

        for vs in inventory.valuesets:
            attr = 'count_' + vs.parameter.segment_class
            if hasattr(inventory, attr):
                val = getattr(inventory, attr) or 0
                setattr(inventory, attr, val + 1)

    ficons = cycle(ORDERED_ICONS)
    gicons = cycle(ORDERED_ICONS)
    for root, genus in groupby(
            DBSession.query(models.Genus).order_by(models.Genus.description),
            lambda g: g.description):
        ficon = ficons.next().name
        for g in genus:
            g.ficon = ficon
            g.gicon = gicons.next().name

    for variety in DBSession.query(models.Variety).options(
            joinedload(models.Variety.inventories)):
        variety.count_inventories = len(variety.inventories)

    if 0:
        ia_func('update', args)
        gbs_func('update', args)
        print('added', add_wikipedia_urls(args), 'wikipedia urls')
Beispiel #5
0
from clld.scripts.util import parsed_args
from clld.scripts.internetarchive import ia_func

if __name__ == '__main__':
    ia_func('update', parsed_args(bootstrap=True))
Beispiel #6
0
from clld.scripts.util import parsed_args
from clld.scripts.internetarchive import ia_func


if __name__ == '__main__':
    ia_func('update', parsed_args(bootstrap=True))