def parse_article_detail_js(self, response): """ 文章详情解析 js 版 :param response: :return: """ article_detail_body = response.body_as_unicode() article_detail_rule = r'<script>FM.view\({"ns":.*?"html":"(.*?)"}\)</script>' article_detail_re_parse = re.compile(article_detail_rule, re.S).findall(article_detail_body) if not article_detail_re_parse: return article_detail_html = ''.join(article_detail_re_parse) # 转义字符处理 article_detail_html = article_detail_html.replace('\\r', '') article_detail_html = article_detail_html.replace('\\t', '') article_detail_html = article_detail_html.replace('\\n', '') article_detail_html = article_detail_html.replace('\\"', '"') article_detail_html = article_detail_html.replace('\\/', '/') article_detail_doc = fromstring(article_detail_html) article_title_parse = article_detail_doc.xpath( '//h1[@class="title"]/text()') article_title = article_title_parse[0].strip( ) if article_title_parse else '' article_pub_time_parse = article_detail_doc.xpath( '//span[@class="time"]/text()') article_pub_time = self.trans_time(article_pub_time_parse[0].strip( )) if article_pub_time_parse else time.strftime('%Y-%m-%d %H:%M:%S') article_content_parse = article_detail_doc.xpath( '//div[@class="WBA_content"]') article_content = tostring( article_content_parse[0], encoding='unicode').strip() if article_content_parse else '' fetch_result_item = FetchResultItem() fetch_result_item['task_id'] = response.meta['task_id'] fetch_result_item['platform_id'] = response.meta['platform_id'] fetch_result_item['platform_name'] = platform_name_map.get( response.meta['platform_id'], '') fetch_result_item['channel_id'] = response.meta['channel_id'] fetch_result_item['channel_name'] = channel_name_map.get( response.meta['channel_id'], '') fetch_result_item['article_id'] = response.meta['article_id'] fetch_result_item['article_title'] = article_title fetch_result_item['article_author_id'] = response.meta['follow_id'] fetch_result_item['article_author_name'] = response.meta['follow_name'] fetch_result_item['article_pub_time'] = time_local_to_utc( article_pub_time).strftime('%Y-%m-%d %H:%M:%S') fetch_result_item['article_url'] = response.url fetch_result_item['article_tags'] = '' fetch_result_item['article_abstract'] = response.meta[ 'article_abstract'] fetch_result_item['article_content'] = article_content yield fetch_result_item
def test_local_to_utc(self): """ 测试 :return: """ local_time_obj = datetime.datetime.strptime(self.local_time, '%Y-%m-%d %H:%M:%S') utc_time_obj = time_local_to_utc(self.local_time) self.assertEqual( utc_time_obj, local_time_obj + datetime.timedelta(seconds=self.time_offset)) self.assertEqual(self.utc_time, utc_time_obj.strftime('%Y-%m-%d %H:%M:%S'))
def parse_detail(self, response): """ 详细页面 :param response: :return: """ article_content = ''.join([ i.strip() for i in response.xpath('//div[@id="js_content"]/*').extract() ]) # 原创内容处理(处理内容为空) if not article_content: share_source_url = response.xpath( '//a[@id="js_share_source"]/@href').extract_first() yield scrapy.Request(url=share_source_url, callback=self.parse_detail, meta=response.meta) return fetch_result_item = FetchResultItem() fetch_result_item['task_id'] = response.meta['task_id'] fetch_result_item['platform_id'] = response.meta['platform_id'] fetch_result_item['platform_name'] = platform_name_map.get( response.meta['platform_id'], '') fetch_result_item['channel_id'] = response.meta['channel_id'] fetch_result_item['channel_name'] = channel_name_map.get( response.meta['channel_id'], '') fetch_result_item['article_id'] = response.meta['article_id'] fetch_result_item['article_title'] = response.meta['article_title'] fetch_result_item['article_author_id'] = response.meta['follow_id'] fetch_result_item['article_author_name'] = response.meta['follow_name'] fetch_result_item['article_pub_time'] = time_local_to_utc( response.meta['article_pub_time']).strftime('%Y-%m-%d %H:%M:%S') fetch_result_item['article_url'] = response.meta['article_url'] fetch_result_item['article_tags'] = '' fetch_result_item['article_abstract'] = response.meta[ 'article_abstract'] fetch_result_item['article_content'] = article_content yield fetch_result_item
def parse_article_detail(self, response): """ 文章详情 :param response: :return: """ toutiao_body = response.body_as_unicode() js_body = parse_toutiao_js_body(toutiao_body, response.meta['detail_url']) if not js_body: return pj = ParseJsTt(js_body=js_body) article_id = pj.parse_js_item_id() article_title = pj.parse_js_title() article_abstract = pj.parse_js_abstract() article_content = pj.parse_js_content() article_pub_time = pj.parse_js_pub_time() article_tags = pj.parse_js_tags() fetch_result_item = FetchResultItem() fetch_result_item['task_id'] = response.meta['task_id'] fetch_result_item['platform_id'] = response.meta['platform_id'] fetch_result_item['platform_name'] = platform_name_map.get( response.meta['platform_id'], '') fetch_result_item['channel_id'] = response.meta['channel_id'] fetch_result_item['channel_name'] = channel_name_map.get( response.meta['channel_id'], '') fetch_result_item['article_id'] = article_id fetch_result_item['article_title'] = article_title fetch_result_item['article_author_id'] = response.meta['follow_id'] fetch_result_item['article_author_name'] = response.meta['follow_name'] fetch_result_item['article_pub_time'] = time_local_to_utc( article_pub_time).strftime('%Y-%m-%d %H:%M:%S') fetch_result_item[ 'article_url'] = response.url or response.meta['detail_url'] fetch_result_item['article_tags'] = article_tags fetch_result_item['article_abstract'] = article_abstract fetch_result_item['article_content'] = article_content yield fetch_result_item
def parse_article_detail(self, response): """ 文章详情 :param response: :return: """ body = response.body_as_unicode() result = json.loads(body)['data'] # print("==article body:", toutiao_body) # fixme add 评论数,阅读数; print('\n====result:', result) impression_count = result[ 'impression_count'] if 'impression_count' in result else 0 comment_count = result[ 'comment_count'] if 'comment_count' in result else 0 fetch_result_item = FetchResultItem() fetch_result_item['task_id'] = 0 fetch_result_item['platform_id'] = 0 fetch_result_item['platform_name'] = platform_name_map.get(3, '') fetch_result_item['channel_id'] = 0 fetch_result_item['channel_name'] = '0' fetch_result_item['article_id'] = response.meta['item_id'] fetch_result_item['article_title'] = response.meta['article_title'] fetch_result_item['article_pub_time'] = time_local_to_utc( response.meta['article_pub_time']).strftime('%Y-%m-%d %H:%M:%S') fetch_result_item['article_url'] = response.meta['article_url'] fetch_result_item['article_tags'] = response.meta.get('keywords') fetch_result_item['article_abstract'] = '' fetch_result_item['article_content'] = result['content'] fetch_result_item['impression_count'] = impression_count fetch_result_item['comment_count'] = comment_count print("===crawl url:", fetch_result_item['article_url']) yield fetch_result_item
def sync(): rack_client = MigrationClient('rack', SourceRack, TargetRack, MigrationContrast) while 1: s_rows = rack_client.s_api.get_limit_rows_by_last_id( last_pk=rack_client.s_id, limit_num=rack_client.limit_num, ) if not s_rows: break for s_data in s_rows: rack_client.s_data = s_data rack_client.s_id = rack_client.s_data.id rack_client.latest_time = time_local_to_utc( rack_client.s_data.updated) # ---------- m_data_warehouse = rack_client.m_api.get_row( table_name='warehouse', pk_source=rack_client.s_data.idwarehouse, ) if not m_data_warehouse: continue # ---------- rack_client.m_detail() # 存在历史数据 if rack_client.m_data: if not rack_client.s_data.updated: continue if rack_client.latest_time <= rack_client.m_data.latest_time: continue # ---------- # 更新目标数据 rack_client.t_id = rack_client.m_data.pk_target current_time = datetime.utcnow() rack_client.t_data = { 'name': rack_client.s_data.name, 'warehouse_id': m_data_warehouse.pk_target, 'update_time': current_time, } # 删除条件 if rack_client.s_data.disabled: rack_client.t_data['status_delete'] = True rack_client.t_data['delete_time'] = current_time # ---------- rack_client.t_update() rack_client.m_update() # 没有历史数据 else: # 目标数据去重 rack_client.t_data = rack_client.t_api.get_row( name=rack_client.s_data.name) if rack_client.t_data: continue # ---------- # 创建目标数据 current_time = datetime.utcnow() rack_client.t_data = { 'name': rack_client.s_data.name, 'warehouse_id': m_data_warehouse.pk_target, 'create_time': current_time, 'update_time': current_time, } # 删除条件 if rack_client.s_data.disabled: rack_client.t_data['status_delete'] = True rack_client.t_data['delete_time'] = current_time # ---------- rack_client.t_create() rack_client.m_create() result = { '来源总数': rack_client.s_api.count(), '目标总数': rack_client.t_api.count(), } return result
def sync(): warehouse_client = MigrationClient('warehouse', SourceWarehouse, TargetWarehouse, MigrationContrast) while 1: s_rows = warehouse_client.s_api.get_limit_rows_by_last_id( last_pk=warehouse_client.s_id, limit_num=warehouse_client.limit_num, ) if not s_rows: break for s_data in s_rows: warehouse_client.s_data = s_data warehouse_client.s_id = warehouse_client.s_data.id warehouse_client.latest_time = time_local_to_utc( warehouse_client.s_data.updated) warehouse_client.m_detail() # 存在历史数据 if warehouse_client.m_data: if not warehouse_client.s_data.updated: continue if warehouse_client.latest_time <= warehouse_client.m_data.latest_time: continue # ---------- # 更新目标数据 warehouse_client.t_id = warehouse_client.m_data.pk_target current_time = datetime.utcnow() warehouse_client.t_data = { 'name': warehouse_client.s_data.name, 'address': warehouse_client.s_data.address, 'update_time': current_time, } # 删除条件 if warehouse_client.s_data.disabled: warehouse_client.t_data['status_delete'] = True warehouse_client.t_data['delete_time'] = current_time # ---------- warehouse_client.t_update() warehouse_client.m_update() # 没有历史数据 else: # 目标数据去重 warehouse_client.t_data = warehouse_client.t_api.get_row( name=warehouse_client.s_data.name) if warehouse_client.t_data: continue # ---------- # 创建目标数据 current_time = datetime.utcnow() warehouse_client.t_data = { 'name': warehouse_client.s_data.name, 'address': warehouse_client.s_data.address, 'create_time': current_time, 'update_time': current_time, } # 删除条件 if warehouse_client.s_data.disabled: warehouse_client.t_data['status_delete'] = True warehouse_client.t_data['delete_time'] = current_time # ---------- warehouse_client.t_create() warehouse_client.m_create() result = { '来源总数': warehouse_client.s_api.count(), '目标总数': warehouse_client.t_api.count(), } return result
def sync(): purchase_client = MigrationClient('purchase', SourcePurchase, TargetPurchase, MigrationContrast) purchase_items_client = MigrationClient('purchase_items', SourcePurchaseItems, TargetPurchaseItems, MigrationContrast) user_client = MigrationClient('user', SourceUser, TargetUser, MigrationContrast) auditor_client = MigrationClient('user', None, None, MigrationContrast) supplier_client = MigrationClient('supplier', None, TargetSupplier, MigrationContrast) supplier_contact_client = MigrationClient('supplier_contact', None, TargetSupplierContact, MigrationContrast) warehouse_client = MigrationClient('warehouse', None, TargetWarehouse, MigrationContrast) rack_client = MigrationClient('rack', None, TargetRack, MigrationContrast) production_client = MigrationClient('production', SourceProduction, TargetProduction, MigrationContrast) while 1: s_rows = purchase_client.s_api.get_limit_rows_by_last_id( last_pk=purchase_client.s_id, limit_num=purchase_client.limit_num, ) if not s_rows: break for s_data in s_rows: purchase_client.s_data = s_data purchase_client.s_id = purchase_client.s_data.id purchase_client.latest_time = time_local_to_utc( purchase_client.s_data.updated) # ---------- # 制单人员 user_client.m_data = user_client.m_api.get_row( table_name='user', pk_source=purchase_client.s_data.makerid, ) if not user_client.m_data: continue # 审核人员 auditor_client.m_data = user_client.m_api.get_row( table_name='user', pk_source=purchase_client.s_data.auditorid, ) auditor_client.t_id = auditor_client.m_data.pk_target if auditor_client.m_data else 0 # 供应商信息 supplier_client.m_data = supplier_client.m_api.get_row( table_name='supplier', pk_source=purchase_client.s_data.idpartner, ) if not supplier_client.m_data: continue supplier_client.t_data = supplier_client.t_api.get_row_by_id( supplier_client.m_data.pk_target) if not supplier_client.t_data: continue # 联系方式 supplier_contact_client.t_data = supplier_contact_client.t_api.get_row( cid=supplier_client.m_data.pk_target, ) if not supplier_contact_client.t_data: continue # 仓库 warehouse_client.m_data = warehouse_client.m_api.get_row( table_name='warehouse', pk_source=purchase_client.s_data.idwarehouse, ) if not warehouse_client.m_data: continue # ---------- purchase_client.m_detail() # 存在历史数据 if purchase_client.m_data: if not purchase_client.s_data.updated: continue if purchase_client.latest_time <= purchase_client.m_data.latest_time: continue # ---------- # 更新目标数据 purchase_client.t_id = purchase_client.m_data.pk_target create_time = time_local_to_utc( purchase_client.s_data.createdtime) update_time = time_local_to_utc(purchase_client.s_data.updated) type_tax = TYPE_TAX_HAS if purchase_client.s_data.totalTaxAmount > purchase_client.s_data.totalAmount else TYPE_TAX_NOT purchase_client.t_data = { 'uid': user_client.m_data.pk_target, 'supplier_cid': supplier_client.t_data.id, 'supplier_company_name': supplier_client.t_data.company_name, 'supplier_contact_id': supplier_contact_client.t_data.id, 'type_purchase': TYPE_PURCHASE_NORMAL, 'amount_production': purchase_client.s_data.totalTaxAmount, 'amount_purchase': purchase_client.s_data.totalTaxAmount, 'warehouse_id': warehouse_client.m_data.pk_target, 'note': purchase_client.s_data.memo, 'type_tax': type_tax, 'update_time': update_time, # 本地时间修改为UTC时间 } # 删除条件 # if purchase_client.s_data.disabled: # purchase_client.t_data['status_delete'] = True # purchase_client.t_data['delete_time'] = current_time # 审核条件 if auditor_client.t_id: purchase_client.t_data['status_audit'] = True purchase_client.t_data['audit_uid'] = auditor_client.t_id purchase_client.t_data['audit_time'] = update_time # ---------- purchase_client.t_update() purchase_client.m_update() # 明细数据 # 清空历史 purchase_items_history_rows = purchase_items_client.t_api.get_rows( purchase_id=purchase_client.t_id, ) for purchase_items_history_data in purchase_items_history_rows: purchase_items_client.t_api.delete( purchase_items_history_data.id) # 全部更新 s_i_rows = purchase_items_client.s_api.get_rows( idPurchaseArrivalDTO=purchase_client.s_data.id) for s_i_data in s_i_rows: purchase_items_client.s_data = s_i_data purchase_items_client.s_id = purchase_items_client.s_data.id purchase_items_client.latest_time = update_time # 产品 production_client.m_data = production_client.m_api.get_row( table_name='production', pk_source=purchase_items_client.s_data.idinventory, ) if not production_client.m_data: continue production_client.t_data = production_client.t_api.get_row_by_id( production_client.m_data.pk_target) if not production_client.t_data: continue production_client.s_data = production_client.s_api.get_row_by_id( production_client.m_data.pk_source) if not production_client.s_data: continue # 仓位(产品编号 > 产品详情 > 产品仓位) rack_client.m_data = rack_client.m_api.get_row( table_name='rack', pk_source=production_client.s_data. idinvlocation, # Fixme ) # 更新明细 purchase_items_client.t_data = { 'purchase_id': purchase_client.t_id, 'uid': user_client.m_data.pk_target, 'supplier_cid': supplier_client.t_data.id, 'supplier_company_name': supplier_client.t_data.company_name, 'production_id': production_client.t_data.id, 'production_brand': production_client.t_data.production_brand, 'production_model': production_client.t_data.production_model, 'production_sku': production_client.t_data.production_sku, 'warehouse_id': warehouse_client.m_data.pk_target, 'rack_id': rack_client.m_data.pk_target if rack_client.m_data else 0, 'type_tax': type_tax, 'quantity': purchase_items_client.s_data.quantity, 'unit_price': purchase_items_client.s_data.taxPrice, 'create_time': create_time, # 本地时间修改为UTC时间 'update_time': update_time, # 本地时间修改为UTC时间 } # 删除条件 Fixme # if purchase_client.s_data.disabled: # current_time = datetime.utcnow() # purchase_items_client.t_data['status_delete'] = True, # purchase_items_client.t_data['delete_time'] = current_time, purchase_items_client.t_create() # 没有历史数据 else: # 目标数据去重 # purchase_client.t_data = purchase_client.t_api.get_row(name=purchase_client.s_data.name) # if purchase_client.t_data: # continue # ---------- # 创建目标数据 create_time = time_local_to_utc( purchase_client.s_data.createdtime) update_time = time_local_to_utc(purchase_client.s_data.updated) type_tax = TYPE_TAX_HAS if purchase_client.s_data.totalTaxAmount > purchase_client.s_data.totalAmount else TYPE_TAX_NOT purchase_client.t_data = { 'uid': user_client.m_data.pk_target, 'supplier_cid': supplier_client.t_data.id, 'supplier_company_name': supplier_client.t_data.company_name, 'supplier_contact_id': supplier_contact_client.t_data.id, 'type_purchase': TYPE_PURCHASE_NORMAL, 'amount_production': purchase_client.s_data.totalTaxAmount, 'amount_purchase': purchase_client.s_data.totalTaxAmount, 'warehouse_id': warehouse_client.m_data.pk_target, 'note': purchase_client.s_data.memo, 'type_tax': type_tax, 'create_time': create_time, # 本地时间修改为UTC时间 'update_time': update_time, # 本地时间修改为UTC时间 } # 删除条件 # if purchase_client.s_data.disabled: # purchase_client.t_data['status_delete'] = True # purchase_client.t_data['delete_time'] = current_time # 审核条件 if auditor_client.t_id: purchase_client.t_data['status_audit'] = True purchase_client.t_data['audit_uid'] = auditor_client.t_id purchase_client.t_data['audit_time'] = update_time # ---------- purchase_client.t_create() purchase_client.m_create() # 明细数据 s_i_rows = purchase_items_client.s_api.get_rows( idPurchaseArrivalDTO=purchase_client.s_data.id) for s_i_data in s_i_rows: purchase_items_client.s_data = s_i_data purchase_items_client.s_id = purchase_items_client.s_data.id purchase_items_client.latest_time = time_local_to_utc( purchase_client.s_data.updated) # 产品 production_client.m_data = production_client.m_api.get_row( table_name='production', pk_source=purchase_items_client.s_data.idinventory, ) if not production_client.m_data: continue production_client.t_data = production_client.t_api.get_row_by_id( production_client.m_data.pk_target) if not production_client.t_data: continue production_client.s_data = production_client.s_api.get_row_by_id( production_client.m_data.pk_source) if not production_client.s_data: continue # 仓位(产品编号 > 产品详情 > 产品仓位) rack_client.m_data = rack_client.m_api.get_row( table_name='rack', pk_source=production_client.s_data. idinvlocation, # Fixme ) # 插入明细 purchase_items_client.t_data = { 'purchase_id': purchase_client.t_id, 'uid': user_client.m_data.pk_target, 'supplier_cid': supplier_client.t_data.id, 'supplier_company_name': supplier_client.t_data.company_name, 'production_id': production_client.t_data.id, 'production_brand': production_client.t_data.production_brand, 'production_model': production_client.t_data.production_model, 'production_sku': production_client.t_data.production_sku, 'warehouse_id': warehouse_client.m_data.pk_target, 'rack_id': rack_client.m_data.pk_target if rack_client.m_data else 0, 'type_tax': type_tax, 'quantity': purchase_items_client.s_data.quantity, 'unit_price': purchase_items_client.s_data.taxPrice, 'create_time': create_time, # 本地时间修改为UTC时间 'update_time': update_time, # 本地时间修改为UTC时间 } # 删除条件 Fixme # if purchase_client.s_data.disabled: # current_time = datetime.utcnow() # purchase_items_client.t_data['status_delete'] = True, # purchase_items_client.t_data['delete_time'] = current_time, purchase_items_client.t_create() result = { '来源总数': purchase_client.s_api.count(), '目标总数': purchase_client.t_api.count(), } return result
def sync(): production_client = MigrationClient('production', SourceProduction, TargetProduction, MigrationContrast) brand_client = MigrationClient('brand', s_model=SourceBrand) while 1: s_rows = production_client.s_api.get_limit_rows_by_last_id( last_pk=production_client.s_id, limit_num=production_client.limit_num, ) if not s_rows: break for s_data in s_rows: production_client.s_data = s_data production_client.s_id = production_client.s_data.id production_client.latest_time = time_local_to_utc( production_client.s_data.updated) # ---------- production_brand = '' if s_data.productInfo: s_data_brand = brand_client.s_api.get_row_by_id( s_data.productInfo) production_brand = s_data_brand.Name.strip().upper( ) if s_data_brand else '' production_model = s_data.specification.strip().upper() # ---------- production_client.m_detail() # 存在历史数据 if production_client.m_data: if not production_client.s_data.updated: continue if production_client.latest_time <= production_client.m_data.latest_time: continue # ---------- # 更新目标数据 production_client.t_id = production_client.m_data.pk_target current_time = datetime.utcnow() production_client.t_data = { 'production_brand': production_brand, 'production_model': production_model, 'note': production_client.s_data.name, 'cost_ref': production_client.s_data.invSCost, 'cost_new': production_client.s_data.latestCost, 'cost_avg': production_client.s_data.avagCost, 'update_time': current_time, } # 删除条件 if production_client.s_data.disabled: production_client.t_data['status_delete'] = True production_client.t_data['delete_time'] = current_time # ---------- production_client.t_update() production_client.m_update() # 没有历史数据 else: # 目标数据去重 production_client.t_data = production_client.t_api.get_row( production_brand=production_brand, production_model=production_model, ) if production_client.t_data: continue # ---------- # 创建目标数据 current_time = datetime.utcnow() production_client.t_data = { 'production_brand': production_brand, 'production_model': production_model, 'note': production_client.s_data.name, 'cost_ref': production_client.s_data.invSCost, 'cost_new': production_client.s_data.latestCost, 'cost_avg': production_client.s_data.avagCost, 'create_time': current_time, 'update_time': current_time, } # 删除条件 if production_client.s_data.disabled: production_client.t_data['status_delete'] = True production_client.t_data['delete_time'] = current_time # ---------- production_client.t_create() production_client.m_create() result = { '来源总数': production_client.s_api.count(), '目标总数': production_client.t_api.count(), } return result
def sync(): supplier_client = MigrationClient( 'supplier', SourceSupplier, TargetSupplier, MigrationContrast, ) supplier_contact_client = MigrationClient( 'supplier_contact', SourceSupplier, TargetSupplierContact, MigrationContrast, ) while 1: s_rows = supplier_client.s_api.get_limit_rows_by_last_id( last_pk=supplier_client.s_id, limit_num=supplier_client.limit_num, ) if not s_rows: break for s_data in s_rows: supplier_client.s_data = s_data supplier_client.s_id = supplier_client.s_data.id supplier_client.latest_time = time_local_to_utc( supplier_client.s_data.updated) # ---------- # 往来单位性质([eap_EnumItem]) # {{endpoint}}/sources/enum_items?page=1&per_page=20000&id_enum=165e3c70-718a-4938-8ca7-01bcaa30b814 # 0c90a0c3-960d-493a-869d-72d97189e4fc 客户 # 8e849744-d6f6-4a46-9428-7b438df6be54 供应商 # 45a62402-6aaf-42de-9a27-7ba96b9b9d2c 客户/供应商 partner_type_supplier = [ '8e849744-d6f6-4a46-9428-7b438df6be54', '45a62402-6aaf-42de-9a27-7ba96b9b9d2c' ] partner_type = str( supplier_client.s_data.partnerType) # UUID 转 str # 类型判断 if partner_type not in partner_type_supplier: continue # ---------- supplier_client.m_detail() # 存在历史数据 if supplier_client.m_data: if not supplier_client.s_data.updated: continue if supplier_client.latest_time <= supplier_client.m_data.latest_time: continue # ---------- # 更新目标数据 supplier_client.t_id = supplier_client.m_data.pk_target current_time = datetime.utcnow() # 提取基本数据 company_name = supplier_client.s_data.name.strip( ) if supplier_client.s_data.name else '' company_address = supplier_client.s_data.ShipmentAddress.strip( ) if supplier_client.s_data.ShipmentAddress else '' company_tel = supplier_client.s_data.TelephoneNo.strip( ) if supplier_client.s_data.TelephoneNo else '' company_fax = supplier_client.s_data.Fax.strip( ) if supplier_client.s_data.Fax else '' # owner_uid = supplier_client.s_data.idsaleman # todo supplier_client.t_data = { 'company_name': company_name, 'company_address': company_address, # 'company_site': '', 'company_tel': company_tel, 'company_fax': company_fax, # 'company_email': '', # 'company_type': '', # 'owner_uid': owner_uid, 'update_time': current_time, } # 删除条件 if supplier_client.s_data.disabled: supplier_client.t_data['status_delete'] = True supplier_client.t_data['delete_time'] = current_time # ---------- supplier_client.t_update() # ---------- # 更新关联数据 t_data_supplier_contact = supplier_contact_client.t_api.get_row( cid=supplier_client.t_id) if not t_data_supplier_contact: continue supplier_contact_data = { 'cid': supplier_client.t_id, 'name': supplier_client.s_data.Contact.strip() if supplier_client.s_data.Contact else '', 'mobile': supplier_client.s_data.TelephoneNo.strip() if supplier_client.s_data.TelephoneNo else '', 'address': supplier_client.s_data.ShipmentAddress.strip() if supplier_client.s_data.ShipmentAddress else '', # 'status_default': True, } supplier_contact_client.t_api.edit(t_data_supplier_contact.id, supplier_contact_data) # ---------- supplier_client.m_update() # 没有历史数据 else: # 目标数据去重 supplier_client.t_data = supplier_client.t_api.get_row( company_name=supplier_client.s_data.name.strip()) if supplier_client.t_data: continue # ---------- # 创建目标数据 current_time = datetime.utcnow() # 提取基本数据 company_name = supplier_client.s_data.name.strip( ) if supplier_client.s_data.name else '' company_address = supplier_client.s_data.ShipmentAddress.strip( ) if supplier_client.s_data.ShipmentAddress else '' company_tel = supplier_client.s_data.TelephoneNo.strip( ) if supplier_client.s_data.TelephoneNo else '' company_fax = supplier_client.s_data.Fax.strip( ) if supplier_client.s_data.Fax else '' # owner_uid = supplier_client.s_data.idsaleman # todo supplier_client.t_data = { 'company_name': company_name, 'company_address': company_address, # 'company_site': '', 'company_tel': company_tel, 'company_fax': company_fax, # 'company_email': '', # 'company_type': '', # 'owner_uid': owner_uid, 'create_time': current_time, 'update_time': current_time, } # 删除条件 if supplier_client.s_data.disabled: supplier_client.t_data['status_delete'] = True supplier_client.t_data['delete_time'] = current_time # ---------- supplier_client.t_create() # ---------- # 创建关联数据 supplier_contact_data = { 'cid': supplier_client.t_id, 'name': supplier_client.s_data.Contact.strip() if supplier_client.s_data.Contact else '', 'mobile': supplier_client.s_data.TelephoneNo.strip() if supplier_client.s_data.TelephoneNo else '', 'address': supplier_client.s_data.ShipmentAddress.strip() if supplier_client.s_data.ShipmentAddress else '', 'status_default': True, } supplier_contact_client.t_api.add(supplier_contact_data) # ---------- supplier_client.m_create() result = { '来源总数': supplier_client.s_api.count(), '目标总数': supplier_client.t_api.count(), } return result
def sync(): delivery_client = MigrationClient('delivery', SourceDelivery, TargetDelivery, MigrationContrast) delivery_items_client = MigrationClient('delivery_items', SourceDeliveryItems, TargetDeliveryItems, MigrationContrast) user_client = MigrationClient('user', SourceUser, TargetUser, MigrationContrast) auditor_client = MigrationClient('user', None, None, MigrationContrast) customer_client = MigrationClient('customer', None, TargetCustomer, MigrationContrast) customer_contact_client = MigrationClient('customer_contact', None, TargetCustomerContact, MigrationContrast) warehouse_client = MigrationClient('warehouse', None, TargetWarehouse, MigrationContrast) rack_client = MigrationClient('rack', None, TargetRack, MigrationContrast) production_client = MigrationClient('production', SourceProduction, TargetProduction, MigrationContrast) while 1: s_rows = delivery_client.s_api.get_limit_rows_by_last_id( last_pk=delivery_client.s_id, limit_num=delivery_client.limit_num, ) if not s_rows: break for s_data in s_rows: delivery_client.s_data = s_data delivery_client.s_id = delivery_client.s_data.id delivery_client.latest_time = time_local_to_utc(delivery_client.s_data.updated) # ---------- # 制单人员 user_client.m_data = user_client.m_api.get_row( table_name='user', pk_source=delivery_client.s_data.makerid, ) if not user_client.m_data: continue # 审核人员 auditor_client.m_data = user_client.m_api.get_row( table_name='user', pk_source=delivery_client.s_data.auditorid, ) auditor_client.t_id = auditor_client.m_data.pk_target if auditor_client.m_data else 0 # 客户信息 customer_client.m_data = customer_client.m_api.get_row( table_name='customer', pk_source=delivery_client.s_data.idsettleCustomer, ) if not customer_client.m_data: continue customer_client.t_data = customer_client.t_api.get_row_by_id(customer_client.m_data.pk_target) if not customer_client.t_data: continue # 联系方式 customer_contact_client.t_data = customer_contact_client.t_api.get_row( cid=customer_client.m_data.pk_target, ) if not customer_contact_client.t_data: continue # 仓库 warehouse_client.m_data = warehouse_client.m_api.get_row( table_name='warehouse', pk_source=delivery_client.s_data.idwarehouse, ) if not warehouse_client.m_data: continue # ---------- delivery_client.m_detail() # 存在历史数据 if delivery_client.m_data: if not delivery_client.s_data.updated: continue if delivery_client.latest_time <= delivery_client.m_data.latest_time: continue # ---------- # 更新目标数据 delivery_client.t_id = delivery_client.m_data.pk_target create_time = time_local_to_utc(delivery_client.s_data.createdtime) update_time = time_local_to_utc(delivery_client.s_data.updated) type_tax = TYPE_TAX_HAS if delivery_client.s_data.taxAmount > delivery_client.s_data.amount else TYPE_TAX_NOT delivery_client.t_data = { 'uid': user_client.m_data.pk_target, 'customer_cid': customer_client.t_data.id, 'customer_company_name': customer_client.t_data.company_name, 'customer_contact_id': customer_contact_client.t_data.id, 'type_delivery': TYPE_DELIVERY_NORMAL, 'amount_production': delivery_client.s_data.taxAmount, 'amount_delivery': delivery_client.s_data.taxAmount, 'warehouse_id': warehouse_client.m_data.pk_target, 'note': delivery_client.s_data.memo, 'type_tax': type_tax, 'update_time': update_time, # 本地时间修改为UTC时间 } # 删除条件 # if delivery_client.s_data.disabled: # delivery_client.t_data['status_delete'] = True # delivery_client.t_data['delete_time'] = current_time # 审核条件 if auditor_client.t_id: delivery_client.t_data['status_audit'] = True delivery_client.t_data['audit_uid'] = auditor_client.t_id delivery_client.t_data['audit_time'] = update_time # ---------- delivery_client.t_update() delivery_client.m_update() # 明细数据 # 清空历史 delivery_items_history_rows = delivery_items_client.t_api.get_rows( delivery_id=delivery_client.t_id, ) for delivery_items_history_data in delivery_items_history_rows: delivery_items_client.t_api.delete(delivery_items_history_data.id) # 全部更新 s_i_rows = delivery_items_client.s_api.get_rows(idSaleDeliveryDTO=delivery_client.s_data.id) for s_i_data in s_i_rows: delivery_items_client.s_data = s_i_data delivery_items_client.s_id = delivery_items_client.s_data.id delivery_items_client.latest_time = time_local_to_utc(delivery_client.s_data.updated) # 产品 production_client.m_data = production_client.m_api.get_row( table_name='production', pk_source=delivery_items_client.s_data.idinventory, ) if not production_client.m_data: continue production_client.t_data = production_client.t_api.get_row_by_id(production_client.m_data.pk_target) if not production_client.t_data: continue production_client.s_data = production_client.s_api.get_row_by_id(production_client.m_data.pk_source) if not production_client.s_data: continue # 仓位(产品编号 > 产品详情 > 产品仓位) rack_client.m_data = rack_client.m_api.get_row( table_name='rack', pk_source=production_client.s_data.idinvlocation, # Fixme ) # 更新明细 delivery_items_client.t_data = { 'delivery_id': delivery_client.t_id, 'uid': user_client.m_data.pk_target, 'customer_cid': customer_client.t_data.id, 'customer_company_name': customer_client.t_data.company_name, 'production_id': production_client.t_data.id, 'production_brand': production_client.t_data.production_brand, 'production_model': production_client.t_data.production_model, 'production_sku': production_client.t_data.production_sku, 'warehouse_id': warehouse_client.m_data.pk_target, 'rack_id': rack_client.m_data.pk_target if rack_client.m_data else 0, 'type_tax': type_tax, 'quantity': delivery_items_client.s_data.quantity, 'unit_price': delivery_items_client.s_data.taxPrice, 'create_time': create_time, # 本地时间修改为UTC时间 'update_time': update_time, # 本地时间修改为UTC时间 } # 删除条件 Fixme # if delivery_client.s_data.disabled: # current_time = datetime.utcnow() # delivery_items_client.t_data['status_delete'] = True, # delivery_items_client.t_data['delete_time'] = current_time, delivery_items_client.t_create() # 没有历史数据 else: # 目标数据去重 # delivery_client.t_data = delivery_client.t_api.get_row(name=delivery_client.s_data.name) # if delivery_client.t_data: # continue # ---------- # 创建目标数据 create_time = time_local_to_utc(delivery_client.s_data.createdtime) update_time = time_local_to_utc(delivery_client.s_data.updated) type_tax = TYPE_TAX_HAS if delivery_client.s_data.taxAmount > delivery_client.s_data.amount else TYPE_TAX_NOT delivery_client.t_data = { 'uid': user_client.m_data.pk_target, 'customer_cid': customer_client.t_data.id, 'customer_company_name': customer_client.t_data.company_name, 'customer_contact_id': customer_contact_client.t_data.id, 'type_delivery': TYPE_DELIVERY_NORMAL, 'amount_production': delivery_client.s_data.taxAmount, 'amount_delivery': delivery_client.s_data.taxAmount, 'warehouse_id': warehouse_client.m_data.pk_target, 'note': delivery_client.s_data.memo, 'type_tax': type_tax, 'create_time': create_time, # 本地时间修改为UTC时间 'update_time': update_time, # 本地时间修改为UTC时间 } # 删除条件 # if delivery_client.s_data.disabled: # delivery_client.t_data['status_delete'] = True # delivery_client.t_data['delete_time'] = current_time # 审核条件 if auditor_client.t_id: delivery_client.t_data['status_audit'] = True delivery_client.t_data['audit_uid'] = auditor_client.t_id delivery_client.t_data['audit_time'] = update_time # ---------- delivery_client.t_create() delivery_client.m_create() # 明细数据 s_i_rows = delivery_items_client.s_api.get_rows(idSaleDeliveryDTO=delivery_client.s_data.id) for s_i_data in s_i_rows: delivery_items_client.s_data = s_i_data delivery_items_client.s_id = delivery_items_client.s_data.id delivery_items_client.latest_time = time_local_to_utc(delivery_client.s_data.updated) # 产品 production_client.m_data = production_client.m_api.get_row( table_name='production', pk_source=delivery_items_client.s_data.idinventory, ) if not production_client.m_data: continue production_client.t_data = production_client.t_api.get_row_by_id(production_client.m_data.pk_target) if not production_client.t_data: continue production_client.s_data = production_client.s_api.get_row_by_id(production_client.m_data.pk_source) if not production_client.s_data: continue # 仓位(产品编号 > 产品详情 > 产品仓位) rack_client.m_data = rack_client.m_api.get_row( table_name='rack', pk_source=production_client.s_data.idinvlocation, # Fixme ) # 插入明细 delivery_items_client.t_data = { 'delivery_id': delivery_client.t_id, 'uid': user_client.m_data.pk_target, 'customer_cid': customer_client.t_data.id, 'customer_company_name': customer_client.t_data.company_name, 'production_id': production_client.t_data.id, 'production_brand': production_client.t_data.production_brand, 'production_model': production_client.t_data.production_model, 'production_sku': production_client.t_data.production_sku, 'warehouse_id': warehouse_client.m_data.pk_target, 'rack_id': rack_client.m_data.pk_target if rack_client.m_data else 0, 'type_tax': type_tax, 'quantity': delivery_items_client.s_data.quantity, 'unit_price': delivery_items_client.s_data.taxPrice, 'create_time': create_time, # 本地时间修改为UTC时间 'update_time': update_time, # 本地时间修改为UTC时间 } # 删除条件 Fixme # if delivery_client.s_data.disabled: # current_time = datetime.utcnow() # delivery_items_client.t_data['status_delete'] = True, # delivery_items_client.t_data['delete_time'] = current_time, delivery_items_client.t_create() result = { '来源总数': delivery_client.s_api.count(), '目标总数': delivery_client.t_api.count(), } return result
def sync(): inventory_client = MigrationClient('inventory', SourceInventory, TargetInventory, MigrationContrast) production_client = MigrationClient('production', SourceProduction, TargetProduction, MigrationContrast) warehouse_client = MigrationClient('warehouse', None, TargetWarehouse, MigrationContrast) rack_client = MigrationClient('rack', None, TargetRack, MigrationContrast) while 1: s_rows = inventory_client.s_api.get_limit_rows_by_last_id( last_pk=inventory_client.s_id, limit_num=inventory_client.limit_num, ) if not s_rows: break for s_data in s_rows: inventory_client.s_data = s_data inventory_client.s_id = inventory_client.s_data.id inventory_client.latest_time = time_local_to_utc( inventory_client.s_data.updated) # ---------- # 产品 production_client.m_data = production_client.m_api.get_row( table_name='production', pk_source=inventory_client.s_data.idinventory, ) if not production_client.m_data: continue production_client.t_data = production_client.t_api.get_row_by_id( production_client.m_data.pk_target) # 仓库 warehouse_client.m_data = warehouse_client.m_api.get_row( table_name='warehouse', pk_source=inventory_client.s_data.idwarehouse, ) if not warehouse_client.m_data: continue warehouse_client.t_data = warehouse_client.t_api.get_row_by_id( warehouse_client.m_data.pk_target) # 仓位 production_client.s_data = production_client.s_api.get_row_by_id( inventory_client.s_data.idinventory) if not production_client.s_data: continue rack_client.m_data = rack_client.m_api.get_row( table_name='rack', pk_source=production_client.s_data.idinvlocation, ) if not rack_client.m_data: continue rack_client.t_data = rack_client.t_api.get_row_by_id( rack_client.m_data.pk_target) # ---------- inventory_client.m_detail() # 存在历史数据 if inventory_client.m_data: if not inventory_client.s_data.updated: continue if inventory_client.latest_time <= inventory_client.m_data.latest_time: continue # ---------- # 更新目标数据 inventory_client.t_id = inventory_client.m_data.pk_target current_time = datetime.utcnow() inventory_client.t_data = { 'production_id': production_client.t_data.id, 'production_brand': production_client.t_data.production_brand, 'production_model': production_client.t_data.production_model, 'production_sku': production_client.t_data.production_sku, 'warehouse_id': warehouse_client.t_data.id, 'warehouse_name': warehouse_client.t_data.name, 'rack_id': rack_client.t_data.id, 'rack_name': rack_client.t_data.name, 'stock_qty_initial': inventory_client.s_data.baseQuantity, 'stock_qty_current': inventory_client.s_data.baseQuantity, 'update_time': current_time, } # 删除条件 # if inventory_client.s_data.disabled: # inventory_client.t_data['status_delete'] = True # inventory_client.t_data['delete_time'] = current_time # ---------- inventory_client.t_update() inventory_client.m_update() # 没有历史数据 else: # 目标数据去重 inventory_client.t_data = inventory_client.t_api.get_row( production_id=production_client.t_data.id, warehouse_id=warehouse_client.t_data.id, rack_id=rack_client.t_data.id, ) if inventory_client.t_data: continue # ---------- # 创建目标数据 current_time = datetime.utcnow() inventory_client.t_data = { 'production_id': production_client.t_data.id, 'production_brand': production_client.t_data.production_brand, 'production_model': production_client.t_data.production_model, 'production_sku': production_client.t_data.production_sku, 'warehouse_id': warehouse_client.t_data.id, 'warehouse_name': warehouse_client.t_data.name, 'rack_id': rack_client.t_data.id, 'rack_name': rack_client.t_data.name, 'stock_qty_initial': inventory_client.s_data.baseQuantity, 'stock_qty_current': inventory_client.s_data.baseQuantity, 'create_time': current_time, 'update_time': current_time, } # 删除条件 # if inventory_client.s_data.disabled: # inventory_client.t_data['status_delete'] = True # inventory_client.t_data['delete_time'] = current_time # ---------- inventory_client.t_create() inventory_client.m_create() result = { '来源总数': inventory_client.s_api.count(), '目标总数': inventory_client.t_api.count(), } return result
def sync(): user_client = MigrationClient('user', SourceUser, TargetUser, MigrationContrast) while 1: s_rows = user_client.s_api.get_limit_rows_by_last_id( last_pk=user_client.s_id, limit_num=user_client.limit_num, ) if not s_rows: break for s_data in s_rows: user_client.s_data = s_data user_client.s_id = user_client.s_data.id user_client.latest_time = time_local_to_utc( user_client.s_data.updated) user_client.m_detail() # 存在历史数据 if user_client.m_data: if not user_client.s_data.updated: continue if user_client.latest_time <= user_client.m_data.latest_time: continue # ---------- # 更新目标数据 user_client.t_id = user_client.m_data.pk_target current_time = datetime.utcnow() user_client.t_data = { 'name': user_client.s_data.Code, 'mobile': user_client.s_data.name, 'update_time': current_time, } # 删除条件 if user_client.s_data.issystem or user_client.s_data.isStoped: user_client.t_data['status_delete'] = True user_client.t_data['delete_time'] = current_time # ---------- user_client.t_update() user_client.m_update() # 没有历史数据 else: # 目标数据去重 user_client.t_data = user_client.t_api.get_row( name=user_client.s_data.Code) if user_client.t_data: continue # ---------- # 创建目标数据 current_time = datetime.utcnow() user_client.t_data = { 'name': user_client.s_data.Code, 'mobile': user_client.s_data.name, 'create_time': current_time, 'update_time': current_time, } # 删除条件 if user_client.s_data.issystem or user_client.s_data.isStoped: user_client.t_data['status_delete'] = True user_client.t_data['delete_time'] = current_time # ---------- user_client.t_create() user_client.m_create() result = { '来源总数': user_client.s_api.count(), '目标总数': user_client.t_api.count(), } return result
def test(): utc_time = time_local_to_utc('2017-06-16 11:21:01') print(utc_time) print(type(utc_time))