def process_new_links(period=media_period, force=False):
"""Fetches links from the last period and sets their media
properities. If force is True, it will fetch properities for links
even if the properties already exist"""
links = Link._query(Link.c._date > timeago(period),
sort=desc('_date'),
data=True)
results = {}
jobs = []
for link in fetch_things2(links):
if link.is_self or link.promoted:
continue
elif not force and (link.has_thumbnail or link.media_object):
continue
jobs.append(make_link_info_job(results, link, g.useragent))
#send links to a queue
wq = WorkQueue(jobs, num_workers=20, timeout=30)
wq.start()
wq.jobs.join()
#when the queue is finished, do the db writes in this thread
for link, info in results.items():
update_link(link, info[0], info[1])
def find_tz():
q = Link._query(sort=desc('_hot'), limit=1)
link = list(q)[0]
t = tdb_sql.get_thing_table(Link._type_id)[0]
s = sa.select([sa.func.hot(t.c.ups, t.c.downs, t.c.date), t.c.thing_id],
t.c.thing_id == link._id)
db_hot = s.execute().fetchall()[0].hot.__float__()
db_hot == round(db_hot, 7)
for tz_name in pytz.common_timezones:
tz = pytz.timezone(tz_name)
sorts.epoch = datetime(1970, 1, 1, tzinfo=tz)
if db_hot == link._hot:
print tz_name
def find_tz():
q = Link._query(sort = desc('_hot'), limit = 1)
link = list(q)[0]
t = tdb_sql.get_thing_table(Link._type_id)[0]
s = sa.select([sa.func.hot(t.c.ups, t.c.downs, t.c.date),
t.c.thing_id],
t.c.thing_id == link._id)
db_hot = s.execute().fetchall()[0].hot.__float__()
db_hot == round(db_hot, 7)
for tz_name in pytz.common_timezones:
tz = pytz.timezone(tz_name)
sorts.epoch = datetime(1970, 1, 1, tzinfo = tz)
if db_hot == link._hot:
print tz_name
def process_new_links(period = media_period, force = False):
"""Fetches links from the last period and sets their media
properities. If force is True, it will fetch properities for links
even if the properties already exist"""
links = Link._query(Link.c._date > timeago(period), sort = desc('_date'),
data = True)
results = {}
jobs = []
for link in fetch_things2(links):
if link.is_self or link.promoted:
continue
elif not force and (link.has_thumbnail or link.media_object):
continue
jobs.append(make_link_info_job(results, link, g.useragent))
#send links to a queue
wq = WorkQueue(jobs, num_workers = 20, timeout = 30)
wq.start()
wq.jobs.join()
#when the queue is finished, do the db writes in this thread
for link, info in results.items():
update_link(link, info[0], info[1])
def retro_burn():
for foo in Link._query():
amqp.add_item('scraper_q',foo._fullname)