def update_apk_links(status_list): try: cursor = _db.cursor() for item in status_list: if item.status == 1: sql = 'UPDATE ' + table_dic[ item.source] + ' set tag = %s where download_link = %s' #print datetime.datetime.now(), sql cursor.execute(sql, (get_epoch_datetime(), item.url)) _db.conn.commit() sql = 'INSERT INTO entry (source_link, download_link, source, file_name, vol_id) VALUES (%s, %s, %s, %s, %s)' #print datetime.datetime.now(), sql cursor.execute(sql, (item.source_link, item.url, item.source, item.file, item.vol_id)) _db.conn.commit() else: # sql = 'UPDATE ' + table_dic[item.source] + ' set tag = %s where download_link = %s' # cursor.execute(sql, (item.status, item.url)) # sql = 'UPDATE new_link SET last_crawl = 1,priority=6 where link=%s' # cursor.execute(sql, (item.source_link)) sql = 'UPDATE ' + table_dic[ item.source] + ' set tag = %s where download_link = %s' #print datetime.datetime.now(), sql cursor.execute(sql, (item.status, item.url)) _db.conn.commit() except MySQLdb.Error, e: print e print sql
def update_apk_links(status_list): try: cursor = _db.cursor() for item in status_list: if item.status == 1: sql = 'UPDATE ' + table_dic[item.source] + ' set tag = %s where download_link = %s' #print datetime.datetime.now(), sql cursor.execute(sql, (get_epoch_datetime(), item.url)) _db.conn.commit() sql = 'INSERT INTO entry (source_link, download_link, source, file_name, vol_id) VALUES (%s, %s, %s, %s, %s)' #print datetime.datetime.now(), sql cursor.execute(sql, (item.source_link, item.url, item.source, item.file, item.vol_id)) _db.conn.commit() else: # sql = 'UPDATE ' + table_dic[item.source] + ' set tag = %s where download_link = %s' # cursor.execute(sql, (item.status, item.url)) # sql = 'UPDATE new_link SET last_crawl = 1,priority=6 where link=%s' # cursor.execute(sql, (item.source_link)) sql = 'UPDATE ' + table_dic[item.source] + ' set tag = %s where download_link = %s' #print datetime.datetime.now(), sql cursor.execute(sql, (item.status, item.url)) _db.conn.commit() except MySQLdb.Error, e: print e print sql
def _assign_last_crawl(status, type, when_insert): if status == Status.FOUND: #Initialize last_crawl field value to 1 when the link is newly found, otherwise keep original value. return 1 if when_insert else None else: return get_epoch_datetime() + (_LEAF_CRAWL_DELAY if type == LinkType.LEAF else _CATELOG_CRAWL_DELAY)
def get_update_crawl_limit(): ''' Only links with last crawl time less than threshold get crawled. ''' now = get_epoch_datetime() return now - _UPDATE_CRAWL_THRESHOLD