Ejemplo n.º 1
0
            attr = getattr(Package, k)
            q = q.filter(attr == v)

    total_count = q.count()

    q = q.filter(Package.impact != None)
    q = q.order_by(Package.impact.desc(), Package.num_downloads.desc())
    q = q.limit(page_size)

    objects = q.all()
    return (total_count, objects)


q = db.session.query(CranPackage.id)
update_registry.register(
    Update(job=CranPackage.set_rev_deps_tree,
           query=q,
           shortcut_fn=CranPackage.shortcut_rev_deps_pairs))

q = db.session.query(PypiPackage.id)
update_registry.register(
    Update(job=PypiPackage.set_rev_deps_tree,
           query=q,
           shortcut_fn=PypiPackage.shortcut_rev_deps_pairs))

q = db.session.query(PypiPackage.id)
q = q.filter(PypiPackage.requires_files != None)
update_registry.register(
    Update(job=PypiPackage.set_host_deps, query=q, queue_id=5))

q = db.session.query(PypiPackage.id)
q = q.filter(PypiPackage.import_name == None)
Ejemplo n.º 2
0
from jobs import update_registry
from jobs import UpdateDbQueue
from date_range import DateRange
from weekly_stats import WeeklyStats

# run with python doi_queue.py --dates --run
update_registry.register(
    UpdateDbQueue(job=DateRange.get_events, action_table="date_range"))

update_registry.register(UpdateDbQueue(job=WeeklyStats.run))
Ejemplo n.º 3
0
from sqlalchemy import text
from sqlalchemy import orm
from sqlalchemy.dialects.postgresql import JSONB

from app import db
import app
from jobs import update_registry
from jobs import Update

from models.product import Product
from models.person import Person
from models import person

q = db.session.query(Person.id)
q = q.filter(Person.orcid_id != None)
update_registry.register(Update(job=Person.refresh, query=q, queue_id=0))

q = db.session.query(Person.id)
q = q.filter(Person.claimed_at != None)
update_registry.register(Update(
    job=Person.email_new_stuff,
    query=q,
))

q = db.session.query(Person.id)
update_registry.register(Update(
    job=Person.calculate,
    query=q,
))

q = db.session.query(Person.id)
Ejemplo n.º 4
0
    q = db.session.query(GithubRepo.login, GithubRepo.repo_name)
    q = q.filter(GithubRepo.api_raw.contains({"fork": False}))
    q = q.filter(GithubRepo.language == 'r')
    q = q.filter(GithubRepo.cran_descr_file == None)
    q = q.order_by(GithubRepo.login)
    q = q.limit(limit)

    enqueue_jobs(GithubRepo, "set_cran_descr_file", q, 3, use_rq)


q = db.session.query(GithubRepo.id)
q = q.filter(GithubRepo.bucket != None)
q = q.filter(GithubRepo.setup_py == None)

update_registry.register(
    Update(job=GithubRepo.set_setup_py, query=q, queue_id=4))

q = db.session.query(GithubRepo.id)
q = q.filter(GithubRepo.api_raw == None)

update_registry.register(
    Update(job=GithubRepo.set_github_about, query=q, queue_id=4))

q = db.session.query(GithubRepo.id)
q = q.filter(GithubRepo.language == 'r')
q = q.filter(GithubRepo.dep_lines != None)
update_registry.register(
    Update(job=GithubRepo.set_cran_dependencies, query=q, queue_id=6))

q = db.session.query(GithubRepo.id)
q = q.filter(GithubRepo.language == 'r')
Ejemplo n.º 5
0
    q = q.filter(GithubRepo.language == 'r')
    q = q.filter(GithubRepo.cran_descr_file == None)
    q = q.order_by(GithubRepo.login)
    q = q.limit(limit)

    enqueue_jobs(GithubRepo, "set_cran_descr_file", q, 3, use_rq)



q = db.session.query(GithubRepo.id)
q = q.filter(GithubRepo.bucket != None)
q = q.filter(GithubRepo.setup_py == None)

update_registry.register(Update(
    job=GithubRepo.set_setup_py,
    query=q,
    queue_id=4
))


q = db.session.query(GithubRepo.id)
q = q.filter(GithubRepo.api_raw == None)

update_registry.register(Update(
    job=GithubRepo.set_github_about,
    query=q,
    queue_id=4
))


q = db.session.query(GithubRepo.id)
Ejemplo n.º 6
0
from sqlalchemy.dialects.postgresql import JSONB

from app import db
import app
from jobs import update_registry
from jobs import Update

from models.product import Product
from models.person import Person
from models import person

q = db.session.query(Person.id)
q = q.filter(Person.orcid_id != None)
update_registry.register(Update(
    job=Person.refresh,
    query=q,
    queue_id=0
))

q = db.session.query(Person.id)
update_registry.register(Update(
    job=Person.calculate,
    query=q,
))

q = db.session.query(Person.id)
update_registry.register(Update(
    job=Person.mini_calculate,
    query=q,
    queue_id=1
))
Ejemplo n.º 7
0
from sqlalchemy.dialects.postgresql import JSONB

from app import db
import app
from jobs import update_registry
from jobs import Update

from models.product import Product
from models.person import Person
from models import person

q = db.session.query(Person.id)
q = q.filter(Person.orcid_id != None)
update_registry.register(Update(
    job=Person.refresh,
    query=q,
    queue_id=0
))

q = db.session.query(Person.id)
q = q.filter(Person.claimed_at != None)
update_registry.register(Update(
    job=Person.email_new_stuff,
    query=q,
))


q = db.session.query(Person.id)
update_registry.register(Update(
    job=Person.calculate,
    query=q,
Ejemplo n.º 8
0
        else:
            attr = getattr(Package, k)
            q = q.filter(attr == v)

    total_count = q.count()

    q = q.order_by(Package.impact.desc())
    q = q.limit(page_size)

    objects = q.all()
    return (total_count, objects)


q = db.session.query(CranPackage.id)
update_registry.register(
    Update(job=CranPackage.set_rev_deps_tree, query=q, shortcut_fn=CranPackage.shortcut_rev_deps_pairs)
)

q = db.session.query(PypiPackage.id)
update_registry.register(
    Update(job=PypiPackage.set_rev_deps_tree, query=q, shortcut_fn=PypiPackage.shortcut_rev_deps_pairs)
)


q = db.session.query(PypiPackage.id)
q = q.filter(PypiPackage.requires_files != None)
update_registry.register(Update(job=PypiPackage.set_host_deps, query=q, queue_id=5))


q = db.session.query(PypiPackage.id)
q = q.filter(PypiPackage.import_name == None)
Ejemplo n.º 9
0
from sqlalchemy.dialects.postgresql import JSONB

from app import db
import app
from jobs import update_registry
from jobs import Update

from models.product import Product
from models.person import Person
from models import person

q = db.session.query(Person.id)
q = q.filter(Person.orcid_id != None)
update_registry.register(Update(
    job=Person.refresh,
    query=q,
    shortcut_fn=person.shortcut_all_percentile_refsets
))


q = db.session.query(Person.id)
update_registry.register(Update(
    job=Person.set_first_name,
    query=q
))
q = db.session.query(Person.id)
update_registry.register(Update(
    job=Person.calculate,
    query=q
))
Ejemplo n.º 10
0
# create table green_base_ids as (select jsonb_array_elements_text(response::jsonb->'_open_base_ids') from crossref where (response::jsonb->>'oa_color'='green'))
# update_registry.register(UpdateDbQueue(
#     job=Base.find_fulltext,
#     action_table="base",
#     # where="(id in (select jsonb_array_elements_text(response::jsonb->'_open_base_ids') from crossref where (response::jsonb->>'oa_color'='green')))",
#     where="(exists (select 1 from base_green_ids_20170515 b where base.id=b.id)) ORDER BY ((((body -> '_source'::text) ->> 'random'::text)::numeric))",
#     # where="base.id in (select jsonb_array_elements_text(response_jsonb->'_closed_base_ids') from crossref where crossref.id in (select id from doi_queue))",
#     # where="queue='queue_me'",
#     # where="(position('ftunivpretoria', base.id) > 0)",
#     queue_name="base_green_ids_20170608"
# ))

update_registry.register(UpdateDbQueue(
    job=Pub.run,
    action_table="pub",
    where="(TRUE)",
    queue_name="run_201705011b"
))

update_registry.register(UpdateDbQueue(
    job=Pub.recalculate,
    action_table="pub"
))

update_registry.register(UpdateDbQueue(
    job=Pub.refresh,
    action_table="pub"
))

update_registry.register(UpdateDbQueue(
    job=Pub.run_with_hybrid,
Ejemplo n.º 11
0
from sqlalchemy import text
from sqlalchemy import orm
from sqlalchemy.dialects.postgresql import JSONB

from app import db
import app
from jobs import update_registry
from jobs import Update

from publication import Publication

q = db.session.query(Publication.id)
update_registry.register(Update(job=Publication.refresh, query=q, queue_id=0))