def test_init_unhandle_tasks(self): context = Context() context.attach(Context.KEY_CURRENT_TASK_TYPE, TaobaoTaskType.sycm_init) action = TaskCollectAction() action.execute(context=context) tasks = context.get(Context.KEY_CURRENT_TASKS, []) for task in tasks: id_ = task.id raw_data = task.raw_data date = raw_data['dateStr'] with write_session_scope() as session: stream_unhandle_task_dao = get_stream_unhandle_task_dao( session=session) mod_dict = {'date': date} stream_unhandle_task_dao.update( mod_dict=mod_dict, _filter=[CmmSysStreamUnhandleTask.id == id_]) context = Context() context.attach(Context.KEY_CURRENT_TASK_TYPE, TaobaoTaskType.sycm_list) action = TaskCollectAction() action.execute(context=context) tasks = context.get(Context.KEY_CURRENT_TASKS, []) for task in tasks: id_ = task.id raw_data = task.raw_data good = Good(raw_data['goodResult']) date = good.get_date() with write_session_scope() as session: stream_unhandle_task_dao = get_stream_unhandle_task_dao( session=session) mod_dict = {'date': date} stream_unhandle_task_dao.update( mod_dict=mod_dict, _filter=[CmmSysStreamUnhandleTask.id == id_]) context = Context() context.attach(Context.KEY_CURRENT_TASK_TYPE, TaobaoTaskType.taobao_list) action = TaskCollectAction() action.execute(context=context) tasks = context.get(Context.KEY_CURRENT_TASKS, []) for task in tasks: id_ = task.id raw_data = task.raw_data good = Good(raw_data['goodResult']) date = good.get_date() with write_session_scope() as session: stream_unhandle_task_dao = get_stream_unhandle_task_dao( session=session) mod_dict = {'date': date} stream_unhandle_task_dao.update( mod_dict=mod_dict, _filter=[CmmSysStreamUnhandleTask.id == id_])
def do_execute(self, context): task = context.get(Context.KEY_CURRENT_TASK) with write_session_scope() as session: good = context.get(Context.KEY_GOOD_DICT) # stream_opt_data_dao = get_stream_opt_data_dao(session=session) stream_unhandle_task_dao = get_stream_unhandle_task_dao(session=session) stream_handle_task_dao = get_stream_handle_task_dao(session=session) stream_good_data_dao = get_stream_good_data_dao(session=session) good_entity = { 'raw_data': good, 'flag': int(good['flag']), 'category_name': good['categoryName'], 'brand_name': good['brandName'], 'category_id': good['categoryId'], 'model_name': good['modelName'], 'date': good['date'] } stream_good_data_dao.insert(**good_entity) entity = stream_unhandle_task_dao.delete(_filter=[CmmSysStreamUnhandleTask.id == task.id]) # stream_handle_task_dao.insert_entity(entity=task) stream_handle_task_dao.insert(**{ 'type': task.type, 'raw_data': task.raw_data, 'origin_id': task.origin_id, 'date': good['date'] }) with mongo_collection_scope(collection_name='stream_good_data') as collection: collection.insert(good) return True
def do_execute(self, context): integrate_result = context.get(Context.KEY_TAOBAO_INTERGRATE_RESULT) sale_result = context.get(Context.KEY_TAOBAO_SALE_RESULT) with write_session_scope() as session: good = context.get(Context.KEY_GOOD_DICT) stream_opt_data_dao = get_stream_opt_data_dao(session=session) stream_unhandle_task_dao = get_stream_unhandle_task_dao( session=session) stream_handle_task_dao = get_stream_handle_task_dao( session=session) opt_data_entity = { 'raw_data': { 'integrateResult': integrate_result, 'saleResult': sale_result, 'goodResult': good }, 'type': int(TaobaoPageType.taobao_list) } entity = stream_opt_data_dao.insert(**opt_data_entity) unhandle_task_entity = { 'raw_data': { 'integrateInfos': self.build_integrate_infos(integrate_result), 'saleInfos': self.build_sale_infos(sale_result), 'goodResult': good }, 'type': int(TaobaoTaskType.taobao_list), 'origin_id': entity.id, 'date': good['date'] } stream_unhandle_task_dao.insert(**unhandle_task_entity) task = context.get(Context.KEY_CURRENT_TASK) task_entity = stream_unhandle_task_dao.delete( _filter=[CmmSysStreamUnhandleTask.id == task.id]) stream_handle_task_dao.insert( **{ 'type': task.type, 'raw_data': task.raw_data, 'origin_id': task.origin_id, 'date': good['date'] }) # stream_handle_task_dao.insert_entity(entity=task) # stream_opt_data_entities = list() # # stream_opt_data_entity = CmmSysStreamOptData() # # stream_opt_data_entity.raw_data = integrate_result # # stream_opt_data_entities.append({'raw_data': integrate_result, 'type':}) # stream_opt_data_entities.append({'raw_data': sale_result}) # # # stream_opt_data_dao.insert_entity() # stream_opt_data_dao.bulk_insert(stream_opt_data_entities) return True
def do_execute(self, context): account = context.get(Context.KEY_CURRENT_TASK_ACCOUNT) # brands_result = context.get(Context.KEY_SYCM_PRODUCT_GET_BRANDS_RESULT) hot_rank_result = context.get( Context.KEY_SYCM_PRODUCT_PROD_HOT_RANK_RESULT) hot_rank_result = hot_rank_result['data'] task = context.get(Context.KEY_CURRENT_TASK) with write_session_scope() as session: good = context.get(Context.KEY_GOOD_DICT) stream_opt_data_dao = get_stream_opt_data_dao(session=session) stream_unhandle_task_dao = get_stream_unhandle_task_dao( session=session) stream_handle_task_dao = get_stream_handle_task_dao( session=session) opt_data_entity = { 'raw_data': { 'hotRankResult': hot_rank_result, 'goodResult': good }, 'type': int(TaobaoPageType.sycm_list) } entity = stream_opt_data_dao.insert(**opt_data_entity) unhandle_task_entities = [] for item in hot_rank_result: c_good = Good(deepcopy(good)) c_good.set_brand_id(item['brandId']) c_good.set_brand_name(item['brandName']) c_good.set_model_id(item['modelId']) c_good.set_model_name(item['modelName']) c_good.set_sell_count(item['payItmCnt']) unhandle_task_entities.append({ 'raw_data': { 'hotRankInfo': item, 'goodResult': c_good, 'account': account }, 'type': int(TaobaoTaskType.sycm_list), 'origin_id': entity.id, 'date': good['date'] }) stream_unhandle_task_dao.bulk_insert(unhandle_task_entities) entity = stream_unhandle_task_dao.delete( _filter=[CmmSysStreamUnhandleTask.id == task.id]) stream_handle_task_dao.insert( **{ 'type': task.type, 'raw_data': task.raw_data, 'origin_id': task.origin_id, 'date': good['date'] }) return True
def execute(self): with write_session_scope() as session: _stream_risk_dao = get_stream_risk_dao(session=session) rsts = _stream_risk_dao.base_query.limit(self.account_num).all() if rsts: for item in rsts: username = item.raw_data account = global_config.s_accounts_dict[username] proxy = self._proxy_service.get_origin_static_proxy(account['username']) self._login(account=account, force=True, risk=True, proxy=proxy) _stream_risk_dao.delete(_filter=[CmmSysStreamRisk.id == item.id]) session.commit()
def test_db(self): entity = CmmSysCommodity() entity.content = { 'category': u'冰箱', 'brand': u'海尔', 'model': 'abcd-abcd', 'price': '12.30', 'sales': '123', 'date': '2018-10-10' } with write_session_scope() as session: commodity_dao = get_commodity_dao(session=session) entity = commodity_dao.insert_entity(entity) delete = {'id': entity.id} commodity_dao.delete(**delete)
def do_execute(self, context): task_type = int(context.get(Context.KEY_CURRENT_TASK_TYPE)) with write_session_scope() as session: # stream_opt_data_dao = get_stream_opt_data_dao(session=session) stream_unhandle_task_dao = get_stream_unhandle_task_dao( session=session) tasks = stream_unhandle_task_dao.base_query.with_entities( entities=[ CmmSysStreamUnhandleTask.id, CmmSysStreamUnhandleTask.type, CmmSysStreamUnhandleTask.raw_data, CmmSysStreamUnhandleTask.origin_id ]).filter( filters_=[CmmSysStreamUnhandleTask.type == task_type ]).limit(1000).all() # tasks = stream_unhandle_task_dao.query(entities=[CmmSysStreamUnhandleTask.id, CmmSysStreamUnhandleTask.type, # CmmSysStreamUnhandleTask.raw_data, # CmmSysStreamUnhandleTask.origin_id], # _filter=[CmmSysStreamUnhandleTask.type == task_type]) if tasks: context.attach(Context.KEY_CURRENT_TASKS, tasks) # opt_data_entity = { # 'raw_data': { # 'integrateResult': integrate_result, # 'saleResult': sale_result, # 'goodResult': good # }, # 'type': int(TaobaoPageType.taobao_list) # } # entity = stream_opt_data_dao.insert(**opt_data_entity) # # unhandle_task_entity = { # 'raw_data': { # 'integrateInfos': self.build_integrate_infos(integrate_result), # 'saleInfos': self.build_sale_infos(sale_result), # 'goodResult': good # }, # 'type': int(TaobaoTaskType.taobao_list), # 'origin_id': entity.id # } # stream_unhandle_task_dao.insert(**unhandle_task_entity) return True
def do_execute(self, context): date_str = yesterday().strftime("%Y-%m-%d") date_str = context.get(Context.KEY_SYCM_SPECIFIC_DATE, date_str) tasks = Category.get_tasks(date_str, Category.sys_category) entities = list() for task in tasks: entities.append({ 'type': int(TaobaoTaskType.sycm_init), 'raw_data': task, 'date': date_str }) if entities: with write_session_scope() as session: stream_unhandle_task_dao = get_stream_unhandle_task_dao( session=session) stream_unhandle_task_dao.bulk_insert(entities) return True
def do_execute(self, context): with write_session_scope() as session: good = context.get(Context.KEY_GOOD_DICT) stream_good_data_dao = get_stream_good_data_dao(session=session) good['flag'] = int(GoodDataType.direct) good_entity = { 'raw_data': good, 'flag': int(good['flag']), 'category_name': good['categoryName'], 'brand_name': good['brandName'], 'category_id': good['categoryId'], 'model_name': good['modelName'], 'date': good['date'] } stream_good_data_dao.insert(**good_entity) with mongo_collection_scope( collection_name='stream_good_data') as collection: collection.insert(good) return True
def do_execute(self, context): task_type = int(context.get(Context.KEY_CURRENT_TASK_TYPE)) date_str = context.get(Context.KEY_DIRECT_COLLECT_DATE) with write_session_scope() as session: # stream_opt_data_dao = get_stream_opt_data_dao(session=session) stream_unhandle_task_dao = get_stream_unhandle_task_dao( session=session) tasks = stream_unhandle_task_dao.query( entities=[ CmmSysStreamUnhandleTask.id, CmmSysStreamUnhandleTask.type, CmmSysStreamUnhandleTask.raw_data, CmmSysStreamUnhandleTask.origin_id ], _filter=[ CmmSysStreamUnhandleTask.type == task_type, CmmSysStreamUnhandleTask.date == date_str ]) context.attach(Context.KEY_CURRENT_TASKS, tasks) return True
def execute(self, num): cycle_login_num = 0 tasks = self.execute_taobao_integrate_list_actions() i = 0 future_tasks = {} with read_session_scope() as session: _stream_risk_dao = get_stream_risk_dao(session=session) rsts = _stream_risk_dao.base_query.all() risk_usernames = set(item.raw_data for item in rsts) s_accounts = global_config.s_accounts for task in tasks: account = s_accounts[self._counter % len(s_accounts)] proxy = self._proxy_service.get_static_proxy(account['username']) # raw_data = task.raw_data # account = raw_data['account'] self._counter += 1 i += 1 if account['username'] in risk_usernames: continue if i < num: future_tasks[self._executor.submit( self._execute_taobao_integrate_list_actions, task, account, proxy)] = task for future in as_completed(future_tasks): try: account, flag, force = future.result() if flag: if force: with write_session_scope() as session: _stream_risk_dao = get_stream_risk_dao( session=session) self._risk(stream_risk_dao=_stream_risk_dao, account=account) # self._login(account=account, force=True if cycle_login_num == 0 else False) cycle_login_num += 1 else: self._fail_account_counter[account['username']] += 1 if self._fail_account_counter[account['username']] > 2: self._cookie_service.remove(account=account) with write_session_scope() as session: _stream_risk_dao = get_stream_risk_dao( session=session) self._risk(stream_risk_dao=_stream_risk_dao, account=account) # self._login(account=account, force=True if cycle_login_num == 0 else False) cycle_login_num += 1 else: url = 'https://s.m.taobao.com/h5?q=Flyco%2BFR5218&search=%E6%8F%90%E4%BA%A4&tab=all' # url = 'https://s.m.taobao.com/h5?q=Flyco%2BFR5218&search=%E6%8F%90%E4%BA%A4&tab=all' proxy = self._proxy_service.get_origin_static_proxy( account['username']) cookies = self._cookie_service.load( account=account, type_=PickleFileType.origin_cookie) time.sleep(5) cookies, origin_cookies = spider_qt5_bootstrap( url=url, account=account, risk=False, proxy=proxy, cookies=cookies) self._cookie_service.dump(cookies=cookies, account=account) self._cookie_service.dump( cookies=origin_cookies, account=account, type_=PickleFileType.origin_cookie) self._account_counter[account['username']] = 0 else: self._fail_account_counter[account['username']] = 0 self._account_counter[account['username']] += 1 if self._account_counter[account['username']] >= 2: url = 'https://s.m.taobao.com/h5?q=Flyco%2BFR5218&search=%E6%8F%90%E4%BA%A4&tab=all' # url = 'https://s.m.taobao.com/h5?q=Flyco%2BFR5218&search=%E6%8F%90%E4%BA%A4&tab=all' proxy = self._proxy_service.get_origin_static_proxy( account['username']) cookies = self._cookie_service.load( account=account, type_=PickleFileType.origin_cookie) time.sleep(5) cookies, origin_cookies = spider_qt5_bootstrap( url=url, account=account, risk=False, proxy=proxy, cookies=cookies) self._cookie_service.dump(cookies=cookies, account=account) self._cookie_service.dump( cookies=origin_cookies, account=account, type_=PickleFileType.origin_cookie) self._account_counter[account['username']] = 0 except Exception as e: logger.error(e)