def handle(self, **options):
pid = Pid('mturk_agregates', True)
start_time = time.time()
log.info('Updating crawl agregates')
update_crawl_agregates(only_new=True)
log.info('db_update_agregates took: %s' % (time.time() - start_time))
pid.remove_pid()
def handle(self, **options):
pid = Pid('mturk_agregates', True)
start_time = time.time()
logging.info('Updating crawl agregates')
update_crawl_agregates(1, only_new=True)
logging.info('db_update_agregates took: %s' %
(time.time() - start_time))
pid.remove_pid()
def handle(self, **options):
update_crawl_agregates(only_new = False)
f = open(os.path.join(settings.ROOT_PATH,'crawl.errors.csv'),"rb")
progress = 10
execute_sql("update main_crawl set success = true where old_id is not null")
for i,id in enumerate(f):
id = id.strip()
execute_sql("delete from main_crawlagregates where crawl_id = (select id from main_crawl where old_id = %s)" % id)
execute_sql("update main_crawl set success = false where old_id = %s" % id)
if i % progress == 0:
print "processed %s rows" % i
execute_sql("commit;")
execute_sql("commit;")
def handle(self, **options):
update_crawl_agregates(only_new=False)
f = open(os.path.join(settings.ROOT_PATH, 'crawl.errors.csv'), "rb")
progress = 10
execute_sql(
"update main_crawl set success = true where old_id is not null")
for i, id in enumerate(f):
id = id.strip()
execute_sql(
"delete from main_crawlagregates where crawl_id = (select id from main_crawl where old_id = %s)"
% id)
execute_sql(
"update main_crawl set success = false where old_id = %s" % id)
if i % progress == 0:
print "processed %s rows" % i
execute_sql("commit;")
execute_sql("commit;")
