Пример #1
0
def process_new_links(period=media_period, force=False):
    """Fetches links from the last period and sets their media
    properities. If force is True, it will fetch properities for links
    even if the properties already exist"""
    links = Link._query(Link.c._date > timeago(period),
                        sort=desc('_date'),
                        data=True)
    results = {}
    jobs = []
    for link in fetch_things2(links):
        if link.is_self or link.promoted:
            continue
        elif not force and (link.has_thumbnail or link.media_object):
            continue

        jobs.append(make_link_info_job(results, link, g.useragent))

    #send links to a queue
    wq = WorkQueue(jobs, num_workers=20, timeout=30)
    wq.start()
    wq.jobs.join()

    #when the queue is finished, do the db writes in this thread
    for link, info in results.items():
        update_link(link, info[0], info[1])
Пример #2
0
def find_tz():
    q = Link._query(sort=desc('_hot'), limit=1)
    link = list(q)[0]
    t = tdb_sql.get_thing_table(Link._type_id)[0]

    s = sa.select([sa.func.hot(t.c.ups, t.c.downs, t.c.date), t.c.thing_id],
                  t.c.thing_id == link._id)
    db_hot = s.execute().fetchall()[0].hot.__float__()

    db_hot == round(db_hot, 7)

    for tz_name in pytz.common_timezones:
        tz = pytz.timezone(tz_name)
        sorts.epoch = datetime(1970, 1, 1, tzinfo=tz)

        if db_hot == link._hot:
            print tz_name
Пример #3
0
def find_tz():
    q = Link._query(sort = desc('_hot'), limit = 1)
    link = list(q)[0]
    t = tdb_sql.get_thing_table(Link._type_id)[0]

    s = sa.select([sa.func.hot(t.c.ups, t.c.downs, t.c.date),
                   t.c.thing_id],
                  t.c.thing_id == link._id)
    db_hot = s.execute().fetchall()[0].hot.__float__()

    db_hot == round(db_hot, 7)

    for tz_name in pytz.common_timezones:
        tz = pytz.timezone(tz_name)
        sorts.epoch = datetime(1970, 1, 1, tzinfo = tz)
        
        if db_hot == link._hot:
            print tz_name
Пример #4
0
def process_new_links(period = media_period, force = False):
    """Fetches links from the last period and sets their media
    properities. If force is True, it will fetch properities for links
    even if the properties already exist"""
    links = Link._query(Link.c._date > timeago(period), sort = desc('_date'),
                        data = True)
    results = {}
    jobs = []
    for link in fetch_things2(links):
        if link.is_self or link.promoted:
            continue
        elif not force and (link.has_thumbnail or link.media_object):
            continue

        jobs.append(make_link_info_job(results, link, g.useragent))

    #send links to a queue
    wq = WorkQueue(jobs, num_workers = 20, timeout = 30)
    wq.start()
    wq.jobs.join()

    #when the queue is finished, do the db writes in this thread
    for link, info in results.items():
        update_link(link, info[0], info[1])
Пример #5
0
def retro_burn():
    for foo in Link._query():
        amqp.add_item('scraper_q',foo._fullname)