def update_apk_links(status_list):
try:
cursor = _db.cursor()
for item in status_list:
if item.status == 1:
sql = 'UPDATE ' + table_dic[
item.source] + ' set tag = %s where download_link = %s'
#print datetime.datetime.now(), sql
cursor.execute(sql, (get_epoch_datetime(), item.url))
_db.conn.commit()
sql = 'INSERT INTO entry (source_link, download_link, source, file_name, vol_id) VALUES (%s, %s, %s, %s, %s)'
#print datetime.datetime.now(), sql
cursor.execute(sql, (item.source_link, item.url, item.source,
item.file, item.vol_id))
_db.conn.commit()
else:
# sql = 'UPDATE ' + table_dic[item.source] + ' set tag = %s where download_link = %s'
# cursor.execute(sql, (item.status, item.url))
# sql = 'UPDATE new_link SET last_crawl = 1,priority=6 where link=%s'
# cursor.execute(sql, (item.source_link))
sql = 'UPDATE ' + table_dic[
item.source] + ' set tag = %s where download_link = %s'
#print datetime.datetime.now(), sql
cursor.execute(sql, (item.status, item.url))
_db.conn.commit()
except MySQLdb.Error, e:
print e
print sql
def update_apk_links(status_list):
try:
cursor = _db.cursor()
for item in status_list:
if item.status == 1:
sql = 'UPDATE ' + table_dic[item.source] + ' set tag = %s where download_link = %s'
#print datetime.datetime.now(), sql
cursor.execute(sql, (get_epoch_datetime(), item.url))
_db.conn.commit()
sql = 'INSERT INTO entry (source_link, download_link, source, file_name, vol_id) VALUES (%s, %s, %s, %s, %s)'
#print datetime.datetime.now(), sql
cursor.execute(sql, (item.source_link, item.url, item.source, item.file, item.vol_id))
_db.conn.commit()
else:
# sql = 'UPDATE ' + table_dic[item.source] + ' set tag = %s where download_link = %s'
# cursor.execute(sql, (item.status, item.url))
# sql = 'UPDATE new_link SET last_crawl = 1,priority=6 where link=%s'
# cursor.execute(sql, (item.source_link))
sql = 'UPDATE ' + table_dic[item.source] + ' set tag = %s where download_link = %s'
#print datetime.datetime.now(), sql
cursor.execute(sql, (item.status, item.url))
_db.conn.commit()
except MySQLdb.Error, e:
print e
print sql
def _assign_last_crawl(status, type, when_insert):
if status == Status.FOUND:
#Initialize last_crawl field value to 1 when the link is newly found, otherwise keep original value.
return 1 if when_insert else None
else:
return get_epoch_datetime() + (_LEAF_CRAWL_DELAY if type == LinkType.LEAF else _CATELOG_CRAWL_DELAY)
def get_update_crawl_limit():
'''
Only links with last crawl time less than threshold get crawled.
'''
now = get_epoch_datetime()
return now - _UPDATE_CRAWL_THRESHOLD