def update_price():
M = N = 1
mylog.configLogging('price_task_%s_%s' %(M,N))
data_task = Jd_Price_Task()
data_task.configTask(is_daily=False,interval_hours=0,sleep_time=0.1, group_num=2000)
data_task.doTaskOnce(M,N)
# encoding: utf-8
import sys
import dbhelper
import time
from task_class import DataTask
import jd_api_crawler
import mylog
import category_helper
reload(sys)
sys.setdefaultencoding('utf8')
mylog.configLogging('promo_category_task')
class Jd_Promo_Category_DataTask(DataTask):
# VIRTUAL
def __load_all_tasks__(self):
return category_helper.load_all_white_sub_categories()
# VIRTUAL
def __task_order__(self, task_id):
return jd_api_crawler.crawl_category_promo(task_id)
#
# ==================================================================================
#
# encoding: utf-8
import sys
import dbhelper
import time
from task_class import DataTask
import jd_detail_crawler
import mylog
import timeHelper
reload(sys)
sys.setdefaultencoding('utf8')
mylog.configLogging('property_task')
class Jd_Property_DataTask(DataTask):
def __init__(self, job_name):
self.job_name = job_name
# VIRTUAL
def __load_all_tasks__(self):
daysago3 = timeHelper.getTimeAheadOfNowDays(3)
sql = 'select distinct sku_id from jd_item_dynamic_latest where update_date >= "%s"' % daysago3
retrows = dbhelper.executeSqlRead2(sql, is_dirty=True)
sku_list = []
for row in retrows:
sku_list.append(row[0])
return sku_list
# VIRTUAL
# encoding: utf-8
import sys
import dbhelper
import time
from task_class import DataTask
import jd_api_crawler
import mylog
import timeHelper
reload(sys)
sys.setdefaultencoding('utf8')
mylog.configLogging('comment_stats_task')
class Jd_Comment_Stats_DataTask(DataTask):
# VIRTUAL
def __load_all_tasks__(self):
daysago3 = timeHelper.getTimeAheadOfNowDays(3)
sql = 'select distinct sku_id from jd_item_dynamic_latest where update_date >= "%s"' % daysago3
retrows = dbhelper.executeSqlRead2(sql, is_dirty=True)
sku_list = []
for row in retrows:
sku_list.append(row[0])
return sku_list
# VIRTUAL
def __task_order__(self, task_id):
return jd_api_crawler.crawl_sku_comment_count(task_id)
# encoding: utf-8
import sys
import dbhelper
import time
from task_class import DataTask
import jd_api_crawler
import mylog
import timeHelper
reload(sys)
sys.setdefaultencoding('utf8')
mylog.configLogging('stock_task')
class Jd_Stock_DataTask(DataTask):
# VIRTUAL
def __load_all_tasks__(self):
daysago3 = timeHelper.getTimeAheadOfNowDays(3)
sql = 'select distinct sku_id from jd_item_dynamic_latest where update_date >= "%s"' % daysago3
retrows = dbhelper.executeSqlRead2(sql, is_dirty=True)
sku_list = []
for row in retrows:
sku_list.append(row[0])
return sku_list
# VIRTUAL
def __task_order__(self, task_id):
return jd_api_crawler.crawl_sku_stock_status(task_id)
return sku_list
# VIRTUAL
def __task_order__(self,task_id):
return jd_api_crawler.crawl_item_promo(task_id)
#
# ==================================================================================
#
if __name__ == "__main__":
argv = sys.argv
M = N = 1
if len(argv) == 2:
print 'Error in arguments'
elif len(argv) == 3:
try:
M = int(argv[1])
N = int(argv[2])
except:
print 'Error in arguments'
mylog.configLogging('promo_item_task_%s_%s' %(M,N))
data_task = Jd_Promo_item_DataTask()
data_task.configTask(is_daily=False,interval_hours=4,sleep_time=0.08)
data_task.doTask(M,N)
mylog.configLogging('price_task_%s_%s' %(M,N))
data_task = Jd_Price_Task()
data_task.configTask(is_daily=False,interval_hours=0,sleep_time=0.1, group_num=2000)
data_task.doTaskOnce(M,N)
#
# ==================================================================================
#
if __name__ == "__main__":
argv = sys.argv
M = N = 1
if len(argv) == 2:
print 'Error in arguments'
elif len(argv) == 3:
try:
M = int(argv[1])
N = int(argv[2])
except:
print 'Error in arguments'
mylog.configLogging('price_task_%s_%s' %(M,N))
data_task = Jd_Price_Task()
data_task.configTask(is_daily=False,interval_hours=0,sleep_time=0.1, group_num=2000)
data_task.doTask(M,N)
# encoding: utf-8
import sys
import dbhelper
import time
from task_class import DataTask
import jd_list_crawler
import mylog
reload(sys)
sys.setdefaultencoding('utf8')
mylog.configLogging('list_task')
class Jd_List_DataTask(DataTask):
# VIRTUAL
def __load_all_tasks__(self):
cat_list = __load_white_categories___()
sub_cat_list = []
for cat_id in cat_list:
sub_cat_list = sub_cat_list + __expand_to_sub_categories__(cat_id)
task_list = __remove_duplicate_categories__(sub_cat_list)
return task_list
# VIRTUAL
def __task_order__(self, task_id):
return jd_list_crawler.crawl_category(task_id)
affected_rows2)
if affected_rows2 <= 0:
logging.error('Saving to item_category error, category_id = %s' %
category_id)
# HANDLE JD_ITEM_FIRSTSEEN
nowtime = timeHelper.getNowLong()
nowdate = timeHelper.getNow()
sql3 = 'insert ignore into jd_item_firstseen values(%s,"%s","%s")'
ftlist = []
for item in product_list:
ftlist.append([item[0], nowtime, nowdate])
affected_rows3 = dbhelper.executeSqlWriteMany(sql3, ftlist)
ret_obj = {
'status': 0 if ret['status'] == 0 and affected_rows2 > 0 else -1,
'item_dynamic': ret,
'item_category': affected_rows2,
'item_first_seen': affected_rows3,
}
return ret_obj
if __name__ == '__main__':
import mylog
mylog.configLogging('test_list_crawler')
cat_id = '12218-12219'
print crawl_category(cat_id)
# print __up_roll_category_id__('652')