def handle(self, **options): pid = Pid('mturk_agregates', True) start_time = time.time() log.info('Updating crawl agregates') update_crawl_agregates(only_new=True) log.info('db_update_agregates took: %s' % (time.time() - start_time)) pid.remove_pid()
def handle(self, **options): pid = Pid('mturk_agregates', True) start_time = time.time() logging.info('Updating crawl agregates') update_crawl_agregates(1, only_new=True) logging.info('db_update_agregates took: %s' % (time.time() - start_time)) pid.remove_pid()
def handle(self, **options): update_crawl_agregates(only_new = False) f = open(os.path.join(settings.ROOT_PATH,'crawl.errors.csv'),"rb") progress = 10 execute_sql("update main_crawl set success = true where old_id is not null") for i,id in enumerate(f): id = id.strip() execute_sql("delete from main_crawlagregates where crawl_id = (select id from main_crawl where old_id = %s)" % id) execute_sql("update main_crawl set success = false where old_id = %s" % id) if i % progress == 0: print "processed %s rows" % i execute_sql("commit;") execute_sql("commit;")
def handle(self, **options): update_crawl_agregates(only_new=False) f = open(os.path.join(settings.ROOT_PATH, 'crawl.errors.csv'), "rb") progress = 10 execute_sql( "update main_crawl set success = true where old_id is not null") for i, id in enumerate(f): id = id.strip() execute_sql( "delete from main_crawlagregates where crawl_id = (select id from main_crawl where old_id = %s)" % id) execute_sql( "update main_crawl set success = false where old_id = %s" % id) if i % progress == 0: print "processed %s rows" % i execute_sql("commit;") execute_sql("commit;")