def parse_article_detail_js(self, response):
"""
文章详情解析 js 版
:param response:
:return:
"""
article_detail_body = response.body_as_unicode()
article_detail_rule = r'<script>FM.view\({"ns":.*?"html":"(.*?)"}\)</script>'
article_detail_re_parse = re.compile(article_detail_rule,
re.S).findall(article_detail_body)
if not article_detail_re_parse:
return
article_detail_html = ''.join(article_detail_re_parse)
# 转义字符处理
article_detail_html = article_detail_html.replace('\\r', '')
article_detail_html = article_detail_html.replace('\\t', '')
article_detail_html = article_detail_html.replace('\\n', '')
article_detail_html = article_detail_html.replace('\\"', '"')
article_detail_html = article_detail_html.replace('\\/', '/')
article_detail_doc = fromstring(article_detail_html)
article_title_parse = article_detail_doc.xpath(
'//h1[@class="title"]/text()')
article_title = article_title_parse[0].strip(
) if article_title_parse else ''
article_pub_time_parse = article_detail_doc.xpath(
'//span[@class="time"]/text()')
article_pub_time = self.trans_time(article_pub_time_parse[0].strip(
)) if article_pub_time_parse else time.strftime('%Y-%m-%d %H:%M:%S')
article_content_parse = article_detail_doc.xpath(
'//div[@class="WBA_content"]')
article_content = tostring(
article_content_parse[0],
encoding='unicode').strip() if article_content_parse else ''
fetch_result_item = FetchResultItem()
fetch_result_item['task_id'] = response.meta['task_id']
fetch_result_item['platform_id'] = response.meta['platform_id']
fetch_result_item['platform_name'] = platform_name_map.get(
response.meta['platform_id'], '')
fetch_result_item['channel_id'] = response.meta['channel_id']
fetch_result_item['channel_name'] = channel_name_map.get(
response.meta['channel_id'], '')
fetch_result_item['article_id'] = response.meta['article_id']
fetch_result_item['article_title'] = article_title
fetch_result_item['article_author_id'] = response.meta['follow_id']
fetch_result_item['article_author_name'] = response.meta['follow_name']
fetch_result_item['article_pub_time'] = time_local_to_utc(
article_pub_time).strftime('%Y-%m-%d %H:%M:%S')
fetch_result_item['article_url'] = response.url
fetch_result_item['article_tags'] = ''
fetch_result_item['article_abstract'] = response.meta[
'article_abstract']
fetch_result_item['article_content'] = article_content
yield fetch_result_item
def test_local_to_utc(self):
"""
测试
:return:
"""
local_time_obj = datetime.datetime.strptime(self.local_time,
'%Y-%m-%d %H:%M:%S')
utc_time_obj = time_local_to_utc(self.local_time)
self.assertEqual(
utc_time_obj,
local_time_obj + datetime.timedelta(seconds=self.time_offset))
self.assertEqual(self.utc_time,
utc_time_obj.strftime('%Y-%m-%d %H:%M:%S'))
def parse_detail(self, response):
"""
详细页面
:param response:
:return:
"""
article_content = ''.join([
i.strip()
for i in response.xpath('//div[@id="js_content"]/*').extract()
])
# 原创内容处理（处理内容为空）
if not article_content:
share_source_url = response.xpath(
'//a[@id="js_share_source"]/@href').extract_first()
yield scrapy.Request(url=share_source_url,
callback=self.parse_detail,
meta=response.meta)
return
fetch_result_item = FetchResultItem()
fetch_result_item['task_id'] = response.meta['task_id']
fetch_result_item['platform_id'] = response.meta['platform_id']
fetch_result_item['platform_name'] = platform_name_map.get(
response.meta['platform_id'], '')
fetch_result_item['channel_id'] = response.meta['channel_id']
fetch_result_item['channel_name'] = channel_name_map.get(
response.meta['channel_id'], '')
fetch_result_item['article_id'] = response.meta['article_id']
fetch_result_item['article_title'] = response.meta['article_title']
fetch_result_item['article_author_id'] = response.meta['follow_id']
fetch_result_item['article_author_name'] = response.meta['follow_name']
fetch_result_item['article_pub_time'] = time_local_to_utc(
response.meta['article_pub_time']).strftime('%Y-%m-%d %H:%M:%S')
fetch_result_item['article_url'] = response.meta['article_url']
fetch_result_item['article_tags'] = ''
fetch_result_item['article_abstract'] = response.meta[
'article_abstract']
fetch_result_item['article_content'] = article_content
yield fetch_result_item
def parse_article_detail(self, response):
"""
文章详情
:param response:
:return:
"""
toutiao_body = response.body_as_unicode()
js_body = parse_toutiao_js_body(toutiao_body,
response.meta['detail_url'])
if not js_body:
return
pj = ParseJsTt(js_body=js_body)
article_id = pj.parse_js_item_id()
article_title = pj.parse_js_title()
article_abstract = pj.parse_js_abstract()
article_content = pj.parse_js_content()
article_pub_time = pj.parse_js_pub_time()
article_tags = pj.parse_js_tags()
fetch_result_item = FetchResultItem()
fetch_result_item['task_id'] = response.meta['task_id']
fetch_result_item['platform_id'] = response.meta['platform_id']
fetch_result_item['platform_name'] = platform_name_map.get(
response.meta['platform_id'], '')
fetch_result_item['channel_id'] = response.meta['channel_id']
fetch_result_item['channel_name'] = channel_name_map.get(
response.meta['channel_id'], '')
fetch_result_item['article_id'] = article_id
fetch_result_item['article_title'] = article_title
fetch_result_item['article_author_id'] = response.meta['follow_id']
fetch_result_item['article_author_name'] = response.meta['follow_name']
fetch_result_item['article_pub_time'] = time_local_to_utc(
article_pub_time).strftime('%Y-%m-%d %H:%M:%S')
fetch_result_item[
'article_url'] = response.url or response.meta['detail_url']
fetch_result_item['article_tags'] = article_tags
fetch_result_item['article_abstract'] = article_abstract
fetch_result_item['article_content'] = article_content
yield fetch_result_item
def parse_article_detail(self, response):
"""
文章详情
:param response:
:return:
"""
body = response.body_as_unicode()
result = json.loads(body)['data']
# print("==article body:", toutiao_body)
# fixme add 评论数，阅读数；
print('\n====result:', result)
impression_count = result[
'impression_count'] if 'impression_count' in result else 0
comment_count = result[
'comment_count'] if 'comment_count' in result else 0
fetch_result_item = FetchResultItem()
fetch_result_item['task_id'] = 0
fetch_result_item['platform_id'] = 0
fetch_result_item['platform_name'] = platform_name_map.get(3, '')
fetch_result_item['channel_id'] = 0
fetch_result_item['channel_name'] = '0'
fetch_result_item['article_id'] = response.meta['item_id']
fetch_result_item['article_title'] = response.meta['article_title']
fetch_result_item['article_pub_time'] = time_local_to_utc(
response.meta['article_pub_time']).strftime('%Y-%m-%d %H:%M:%S')
fetch_result_item['article_url'] = response.meta['article_url']
fetch_result_item['article_tags'] = response.meta.get('keywords')
fetch_result_item['article_abstract'] = ''
fetch_result_item['article_content'] = result['content']
fetch_result_item['impression_count'] = impression_count
fetch_result_item['comment_count'] = comment_count
print("===crawl url:", fetch_result_item['article_url'])
yield fetch_result_item
def sync():
rack_client = MigrationClient('rack', SourceRack, TargetRack,
MigrationContrast)
while 1:
s_rows = rack_client.s_api.get_limit_rows_by_last_id(
last_pk=rack_client.s_id,
limit_num=rack_client.limit_num,
)
if not s_rows:
break
for s_data in s_rows:
rack_client.s_data = s_data
rack_client.s_id = rack_client.s_data.id
rack_client.latest_time = time_local_to_utc(
rack_client.s_data.updated)
# ----------
m_data_warehouse = rack_client.m_api.get_row(
table_name='warehouse',
pk_source=rack_client.s_data.idwarehouse,
)
if not m_data_warehouse:
continue
# ----------
rack_client.m_detail()
# 存在历史数据
if rack_client.m_data:
if not rack_client.s_data.updated:
continue
if rack_client.latest_time <= rack_client.m_data.latest_time:
continue
# ----------
# 更新目标数据
rack_client.t_id = rack_client.m_data.pk_target
current_time = datetime.utcnow()
rack_client.t_data = {
'name': rack_client.s_data.name,
'warehouse_id': m_data_warehouse.pk_target,
'update_time': current_time,
}
# 删除条件
if rack_client.s_data.disabled:
rack_client.t_data['status_delete'] = True
rack_client.t_data['delete_time'] = current_time
# ----------
rack_client.t_update()
rack_client.m_update()
# 没有历史数据
else:
# 目标数据去重
rack_client.t_data = rack_client.t_api.get_row(
name=rack_client.s_data.name)
if rack_client.t_data:
continue
# ----------
# 创建目标数据
current_time = datetime.utcnow()
rack_client.t_data = {
'name': rack_client.s_data.name,
'warehouse_id': m_data_warehouse.pk_target,
'create_time': current_time,
'update_time': current_time,
}
# 删除条件
if rack_client.s_data.disabled:
rack_client.t_data['status_delete'] = True
rack_client.t_data['delete_time'] = current_time
# ----------
rack_client.t_create()
rack_client.m_create()
result = {
'来源总数': rack_client.s_api.count(),
'目标总数': rack_client.t_api.count(),
}
return result
def sync():
warehouse_client = MigrationClient('warehouse', SourceWarehouse,
TargetWarehouse, MigrationContrast)
while 1:
s_rows = warehouse_client.s_api.get_limit_rows_by_last_id(
last_pk=warehouse_client.s_id,
limit_num=warehouse_client.limit_num,
)
if not s_rows:
break
for s_data in s_rows:
warehouse_client.s_data = s_data
warehouse_client.s_id = warehouse_client.s_data.id
warehouse_client.latest_time = time_local_to_utc(
warehouse_client.s_data.updated)
warehouse_client.m_detail()
# 存在历史数据
if warehouse_client.m_data:
if not warehouse_client.s_data.updated:
continue
if warehouse_client.latest_time <= warehouse_client.m_data.latest_time:
continue
# ----------
# 更新目标数据
warehouse_client.t_id = warehouse_client.m_data.pk_target
current_time = datetime.utcnow()
warehouse_client.t_data = {
'name': warehouse_client.s_data.name,
'address': warehouse_client.s_data.address,
'update_time': current_time,
}
# 删除条件
if warehouse_client.s_data.disabled:
warehouse_client.t_data['status_delete'] = True
warehouse_client.t_data['delete_time'] = current_time
# ----------
warehouse_client.t_update()
warehouse_client.m_update()
# 没有历史数据
else:
# 目标数据去重
warehouse_client.t_data = warehouse_client.t_api.get_row(
name=warehouse_client.s_data.name)
if warehouse_client.t_data:
continue
# ----------
# 创建目标数据
current_time = datetime.utcnow()
warehouse_client.t_data = {
'name': warehouse_client.s_data.name,
'address': warehouse_client.s_data.address,
'create_time': current_time,
'update_time': current_time,
}
# 删除条件
if warehouse_client.s_data.disabled:
warehouse_client.t_data['status_delete'] = True
warehouse_client.t_data['delete_time'] = current_time
# ----------
warehouse_client.t_create()
warehouse_client.m_create()
result = {
'来源总数': warehouse_client.s_api.count(),
'目标总数': warehouse_client.t_api.count(),
}
return result
def sync():
purchase_client = MigrationClient('purchase', SourcePurchase,
TargetPurchase, MigrationContrast)
purchase_items_client = MigrationClient('purchase_items',
SourcePurchaseItems,
TargetPurchaseItems,
MigrationContrast)
user_client = MigrationClient('user', SourceUser, TargetUser,
MigrationContrast)
auditor_client = MigrationClient('user', None, None, MigrationContrast)
supplier_client = MigrationClient('supplier', None, TargetSupplier,
MigrationContrast)
supplier_contact_client = MigrationClient('supplier_contact', None,
TargetSupplierContact,
MigrationContrast)
warehouse_client = MigrationClient('warehouse', None, TargetWarehouse,
MigrationContrast)
rack_client = MigrationClient('rack', None, TargetRack, MigrationContrast)
production_client = MigrationClient('production', SourceProduction,
TargetProduction, MigrationContrast)
while 1:
s_rows = purchase_client.s_api.get_limit_rows_by_last_id(
last_pk=purchase_client.s_id,
limit_num=purchase_client.limit_num,
)
if not s_rows:
break
for s_data in s_rows:
purchase_client.s_data = s_data
purchase_client.s_id = purchase_client.s_data.id
purchase_client.latest_time = time_local_to_utc(
purchase_client.s_data.updated)
# ----------
# 制单人员
user_client.m_data = user_client.m_api.get_row(
table_name='user',
pk_source=purchase_client.s_data.makerid,
)
if not user_client.m_data:
continue
# 审核人员
auditor_client.m_data = user_client.m_api.get_row(
table_name='user',
pk_source=purchase_client.s_data.auditorid,
)
auditor_client.t_id = auditor_client.m_data.pk_target if auditor_client.m_data else 0
# 供应商信息
supplier_client.m_data = supplier_client.m_api.get_row(
table_name='supplier',
pk_source=purchase_client.s_data.idpartner,
)
if not supplier_client.m_data:
continue
supplier_client.t_data = supplier_client.t_api.get_row_by_id(
supplier_client.m_data.pk_target)
if not supplier_client.t_data:
continue
# 联系方式
supplier_contact_client.t_data = supplier_contact_client.t_api.get_row(
cid=supplier_client.m_data.pk_target, )
if not supplier_contact_client.t_data:
continue
# 仓库
warehouse_client.m_data = warehouse_client.m_api.get_row(
table_name='warehouse',
pk_source=purchase_client.s_data.idwarehouse,
)
if not warehouse_client.m_data:
continue
# ----------
purchase_client.m_detail()
# 存在历史数据
if purchase_client.m_data:
if not purchase_client.s_data.updated:
continue
if purchase_client.latest_time <= purchase_client.m_data.latest_time:
continue
# ----------
# 更新目标数据
purchase_client.t_id = purchase_client.m_data.pk_target
create_time = time_local_to_utc(
purchase_client.s_data.createdtime)
update_time = time_local_to_utc(purchase_client.s_data.updated)
type_tax = TYPE_TAX_HAS if purchase_client.s_data.totalTaxAmount > purchase_client.s_data.totalAmount else TYPE_TAX_NOT
purchase_client.t_data = {
'uid': user_client.m_data.pk_target,
'supplier_cid': supplier_client.t_data.id,
'supplier_company_name':
supplier_client.t_data.company_name,
'supplier_contact_id': supplier_contact_client.t_data.id,
'type_purchase': TYPE_PURCHASE_NORMAL,
'amount_production': purchase_client.s_data.totalTaxAmount,
'amount_purchase': purchase_client.s_data.totalTaxAmount,
'warehouse_id': warehouse_client.m_data.pk_target,
'note': purchase_client.s_data.memo,
'type_tax': type_tax,
'update_time': update_time, # 本地时间修改为UTC时间
}
# 删除条件
# if purchase_client.s_data.disabled:
# purchase_client.t_data['status_delete'] = True
# purchase_client.t_data['delete_time'] = current_time
# 审核条件
if auditor_client.t_id:
purchase_client.t_data['status_audit'] = True
purchase_client.t_data['audit_uid'] = auditor_client.t_id
purchase_client.t_data['audit_time'] = update_time
# ----------
purchase_client.t_update()
purchase_client.m_update()
# 明细数据
# 清空历史
purchase_items_history_rows = purchase_items_client.t_api.get_rows(
purchase_id=purchase_client.t_id, )
for purchase_items_history_data in purchase_items_history_rows:
purchase_items_client.t_api.delete(
purchase_items_history_data.id)
# 全部更新
s_i_rows = purchase_items_client.s_api.get_rows(
idPurchaseArrivalDTO=purchase_client.s_data.id)
for s_i_data in s_i_rows:
purchase_items_client.s_data = s_i_data
purchase_items_client.s_id = purchase_items_client.s_data.id
purchase_items_client.latest_time = update_time
# 产品
production_client.m_data = production_client.m_api.get_row(
table_name='production',
pk_source=purchase_items_client.s_data.idinventory,
)
if not production_client.m_data:
continue
production_client.t_data = production_client.t_api.get_row_by_id(
production_client.m_data.pk_target)
if not production_client.t_data:
continue
production_client.s_data = production_client.s_api.get_row_by_id(
production_client.m_data.pk_source)
if not production_client.s_data:
continue
# 仓位（产品编号 > 产品详情 > 产品仓位）
rack_client.m_data = rack_client.m_api.get_row(
table_name='rack',
pk_source=production_client.s_data.
idinvlocation, # Fixme
)
# 更新明细
purchase_items_client.t_data = {
'purchase_id':
purchase_client.t_id,
'uid':
user_client.m_data.pk_target,
'supplier_cid':
supplier_client.t_data.id,
'supplier_company_name':
supplier_client.t_data.company_name,
'production_id':
production_client.t_data.id,
'production_brand':
production_client.t_data.production_brand,
'production_model':
production_client.t_data.production_model,
'production_sku':
production_client.t_data.production_sku,
'warehouse_id':
warehouse_client.m_data.pk_target,
'rack_id':
rack_client.m_data.pk_target
if rack_client.m_data else 0,
'type_tax':
type_tax,
'quantity':
purchase_items_client.s_data.quantity,
'unit_price':
purchase_items_client.s_data.taxPrice,
'create_time':
create_time, # 本地时间修改为UTC时间
'update_time':
update_time, # 本地时间修改为UTC时间
}
# 删除条件 Fixme
# if purchase_client.s_data.disabled:
# current_time = datetime.utcnow()
# purchase_items_client.t_data['status_delete'] = True,
# purchase_items_client.t_data['delete_time'] = current_time,
purchase_items_client.t_create()
# 没有历史数据
else:
# 目标数据去重
# purchase_client.t_data = purchase_client.t_api.get_row(name=purchase_client.s_data.name)
# if purchase_client.t_data:
# continue
# ----------
# 创建目标数据
create_time = time_local_to_utc(
purchase_client.s_data.createdtime)
update_time = time_local_to_utc(purchase_client.s_data.updated)
type_tax = TYPE_TAX_HAS if purchase_client.s_data.totalTaxAmount > purchase_client.s_data.totalAmount else TYPE_TAX_NOT
purchase_client.t_data = {
'uid': user_client.m_data.pk_target,
'supplier_cid': supplier_client.t_data.id,
'supplier_company_name':
supplier_client.t_data.company_name,
'supplier_contact_id': supplier_contact_client.t_data.id,
'type_purchase': TYPE_PURCHASE_NORMAL,
'amount_production': purchase_client.s_data.totalTaxAmount,
'amount_purchase': purchase_client.s_data.totalTaxAmount,
'warehouse_id': warehouse_client.m_data.pk_target,
'note': purchase_client.s_data.memo,
'type_tax': type_tax,
'create_time': create_time, # 本地时间修改为UTC时间
'update_time': update_time, # 本地时间修改为UTC时间
}
# 删除条件
# if purchase_client.s_data.disabled:
# purchase_client.t_data['status_delete'] = True
# purchase_client.t_data['delete_time'] = current_time
# 审核条件
if auditor_client.t_id:
purchase_client.t_data['status_audit'] = True
purchase_client.t_data['audit_uid'] = auditor_client.t_id
purchase_client.t_data['audit_time'] = update_time
# ----------
purchase_client.t_create()
purchase_client.m_create()
# 明细数据
s_i_rows = purchase_items_client.s_api.get_rows(
idPurchaseArrivalDTO=purchase_client.s_data.id)
for s_i_data in s_i_rows:
purchase_items_client.s_data = s_i_data
purchase_items_client.s_id = purchase_items_client.s_data.id
purchase_items_client.latest_time = time_local_to_utc(
purchase_client.s_data.updated)
# 产品
production_client.m_data = production_client.m_api.get_row(
table_name='production',
pk_source=purchase_items_client.s_data.idinventory,
)
if not production_client.m_data:
continue
production_client.t_data = production_client.t_api.get_row_by_id(
production_client.m_data.pk_target)
if not production_client.t_data:
continue
production_client.s_data = production_client.s_api.get_row_by_id(
production_client.m_data.pk_source)
if not production_client.s_data:
continue
# 仓位（产品编号 > 产品详情 > 产品仓位）
rack_client.m_data = rack_client.m_api.get_row(
table_name='rack',
pk_source=production_client.s_data.
idinvlocation, # Fixme
)
# 插入明细
purchase_items_client.t_data = {
'purchase_id':
purchase_client.t_id,
'uid':
user_client.m_data.pk_target,
'supplier_cid':
supplier_client.t_data.id,
'supplier_company_name':
supplier_client.t_data.company_name,
'production_id':
production_client.t_data.id,
'production_brand':
production_client.t_data.production_brand,
'production_model':
production_client.t_data.production_model,
'production_sku':
production_client.t_data.production_sku,
'warehouse_id':
warehouse_client.m_data.pk_target,
'rack_id':
rack_client.m_data.pk_target
if rack_client.m_data else 0,
'type_tax':
type_tax,
'quantity':
purchase_items_client.s_data.quantity,
'unit_price':
purchase_items_client.s_data.taxPrice,
'create_time':
create_time, # 本地时间修改为UTC时间
'update_time':
update_time, # 本地时间修改为UTC时间
}
# 删除条件 Fixme
# if purchase_client.s_data.disabled:
# current_time = datetime.utcnow()
# purchase_items_client.t_data['status_delete'] = True,
# purchase_items_client.t_data['delete_time'] = current_time,
purchase_items_client.t_create()
result = {
'来源总数': purchase_client.s_api.count(),
'目标总数': purchase_client.t_api.count(),
}
return result
def sync():
production_client = MigrationClient('production', SourceProduction,
TargetProduction, MigrationContrast)
brand_client = MigrationClient('brand', s_model=SourceBrand)
while 1:
s_rows = production_client.s_api.get_limit_rows_by_last_id(
last_pk=production_client.s_id,
limit_num=production_client.limit_num,
)
if not s_rows:
break
for s_data in s_rows:
production_client.s_data = s_data
production_client.s_id = production_client.s_data.id
production_client.latest_time = time_local_to_utc(
production_client.s_data.updated)
# ----------
production_brand = ''
if s_data.productInfo:
s_data_brand = brand_client.s_api.get_row_by_id(
s_data.productInfo)
production_brand = s_data_brand.Name.strip().upper(
) if s_data_brand else ''
production_model = s_data.specification.strip().upper()
# ----------
production_client.m_detail()
# 存在历史数据
if production_client.m_data:
if not production_client.s_data.updated:
continue
if production_client.latest_time <= production_client.m_data.latest_time:
continue
# ----------
# 更新目标数据
production_client.t_id = production_client.m_data.pk_target
current_time = datetime.utcnow()
production_client.t_data = {
'production_brand': production_brand,
'production_model': production_model,
'note': production_client.s_data.name,
'cost_ref': production_client.s_data.invSCost,
'cost_new': production_client.s_data.latestCost,
'cost_avg': production_client.s_data.avagCost,
'update_time': current_time,
}
# 删除条件
if production_client.s_data.disabled:
production_client.t_data['status_delete'] = True
production_client.t_data['delete_time'] = current_time
# ----------
production_client.t_update()
production_client.m_update()
# 没有历史数据
else:
# 目标数据去重
production_client.t_data = production_client.t_api.get_row(
production_brand=production_brand,
production_model=production_model,
)
if production_client.t_data:
continue
# ----------
# 创建目标数据
current_time = datetime.utcnow()
production_client.t_data = {
'production_brand': production_brand,
'production_model': production_model,
'note': production_client.s_data.name,
'cost_ref': production_client.s_data.invSCost,
'cost_new': production_client.s_data.latestCost,
'cost_avg': production_client.s_data.avagCost,
'create_time': current_time,
'update_time': current_time,
}
# 删除条件
if production_client.s_data.disabled:
production_client.t_data['status_delete'] = True
production_client.t_data['delete_time'] = current_time
# ----------
production_client.t_create()
production_client.m_create()
result = {
'来源总数': production_client.s_api.count(),
'目标总数': production_client.t_api.count(),
}
return result
def sync():
supplier_client = MigrationClient(
'supplier',
SourceSupplier,
TargetSupplier,
MigrationContrast,
)
supplier_contact_client = MigrationClient(
'supplier_contact',
SourceSupplier,
TargetSupplierContact,
MigrationContrast,
)
while 1:
s_rows = supplier_client.s_api.get_limit_rows_by_last_id(
last_pk=supplier_client.s_id,
limit_num=supplier_client.limit_num,
)
if not s_rows:
break
for s_data in s_rows:
supplier_client.s_data = s_data
supplier_client.s_id = supplier_client.s_data.id
supplier_client.latest_time = time_local_to_utc(
supplier_client.s_data.updated)
# ----------
# 往来单位性质（[eap_EnumItem]）
# {{endpoint}}/sources/enum_items?page=1&per_page=20000&id_enum=165e3c70-718a-4938-8ca7-01bcaa30b814
# 0c90a0c3-960d-493a-869d-72d97189e4fc 客户
# 8e849744-d6f6-4a46-9428-7b438df6be54 供应商
# 45a62402-6aaf-42de-9a27-7ba96b9b9d2c 客户/供应商
partner_type_supplier = [
'8e849744-d6f6-4a46-9428-7b438df6be54',
'45a62402-6aaf-42de-9a27-7ba96b9b9d2c'
]
partner_type = str(
supplier_client.s_data.partnerType) # UUID 转 str
# 类型判断
if partner_type not in partner_type_supplier:
continue
# ----------
supplier_client.m_detail()
# 存在历史数据
if supplier_client.m_data:
if not supplier_client.s_data.updated:
continue
if supplier_client.latest_time <= supplier_client.m_data.latest_time:
continue
# ----------
# 更新目标数据
supplier_client.t_id = supplier_client.m_data.pk_target
current_time = datetime.utcnow()
# 提取基本数据
company_name = supplier_client.s_data.name.strip(
) if supplier_client.s_data.name else ''
company_address = supplier_client.s_data.ShipmentAddress.strip(
) if supplier_client.s_data.ShipmentAddress else ''
company_tel = supplier_client.s_data.TelephoneNo.strip(
) if supplier_client.s_data.TelephoneNo else ''
company_fax = supplier_client.s_data.Fax.strip(
) if supplier_client.s_data.Fax else ''
# owner_uid = supplier_client.s_data.idsaleman # todo
supplier_client.t_data = {
'company_name': company_name,
'company_address': company_address,
# 'company_site': '',
'company_tel': company_tel,
'company_fax': company_fax,
# 'company_email': '',
# 'company_type': '',
# 'owner_uid': owner_uid,
'update_time': current_time,
}
# 删除条件
if supplier_client.s_data.disabled:
supplier_client.t_data['status_delete'] = True
supplier_client.t_data['delete_time'] = current_time
# ----------
supplier_client.t_update()
# ----------
# 更新关联数据
t_data_supplier_contact = supplier_contact_client.t_api.get_row(
cid=supplier_client.t_id)
if not t_data_supplier_contact:
continue
supplier_contact_data = {
'cid':
supplier_client.t_id,
'name':
supplier_client.s_data.Contact.strip()
if supplier_client.s_data.Contact else '',
'mobile':
supplier_client.s_data.TelephoneNo.strip()
if supplier_client.s_data.TelephoneNo else '',
'address':
supplier_client.s_data.ShipmentAddress.strip()
if supplier_client.s_data.ShipmentAddress else '',
# 'status_default': True,
}
supplier_contact_client.t_api.edit(t_data_supplier_contact.id,
supplier_contact_data)
# ----------
supplier_client.m_update()
# 没有历史数据
else:
# 目标数据去重
supplier_client.t_data = supplier_client.t_api.get_row(
company_name=supplier_client.s_data.name.strip())
if supplier_client.t_data:
continue
# ----------
# 创建目标数据
current_time = datetime.utcnow()
# 提取基本数据
company_name = supplier_client.s_data.name.strip(
) if supplier_client.s_data.name else ''
company_address = supplier_client.s_data.ShipmentAddress.strip(
) if supplier_client.s_data.ShipmentAddress else ''
company_tel = supplier_client.s_data.TelephoneNo.strip(
) if supplier_client.s_data.TelephoneNo else ''
company_fax = supplier_client.s_data.Fax.strip(
) if supplier_client.s_data.Fax else ''
# owner_uid = supplier_client.s_data.idsaleman # todo
supplier_client.t_data = {
'company_name': company_name,
'company_address': company_address,
# 'company_site': '',
'company_tel': company_tel,
'company_fax': company_fax,
# 'company_email': '',
# 'company_type': '',
# 'owner_uid': owner_uid,
'create_time': current_time,
'update_time': current_time,
}
# 删除条件
if supplier_client.s_data.disabled:
supplier_client.t_data['status_delete'] = True
supplier_client.t_data['delete_time'] = current_time
# ----------
supplier_client.t_create()
# ----------
# 创建关联数据
supplier_contact_data = {
'cid':
supplier_client.t_id,
'name':
supplier_client.s_data.Contact.strip()
if supplier_client.s_data.Contact else '',
'mobile':
supplier_client.s_data.TelephoneNo.strip()
if supplier_client.s_data.TelephoneNo else '',
'address':
supplier_client.s_data.ShipmentAddress.strip()
if supplier_client.s_data.ShipmentAddress else '',
'status_default':
True,
}
supplier_contact_client.t_api.add(supplier_contact_data)
# ----------
supplier_client.m_create()
result = {
'来源总数': supplier_client.s_api.count(),
'目标总数': supplier_client.t_api.count(),
}
return result
def sync():
delivery_client = MigrationClient('delivery', SourceDelivery, TargetDelivery, MigrationContrast)
delivery_items_client = MigrationClient('delivery_items', SourceDeliveryItems, TargetDeliveryItems,
MigrationContrast)
user_client = MigrationClient('user', SourceUser, TargetUser, MigrationContrast)
auditor_client = MigrationClient('user', None, None, MigrationContrast)
customer_client = MigrationClient('customer', None, TargetCustomer, MigrationContrast)
customer_contact_client = MigrationClient('customer_contact', None, TargetCustomerContact, MigrationContrast)
warehouse_client = MigrationClient('warehouse', None, TargetWarehouse, MigrationContrast)
rack_client = MigrationClient('rack', None, TargetRack, MigrationContrast)
production_client = MigrationClient('production', SourceProduction, TargetProduction, MigrationContrast)
while 1:
s_rows = delivery_client.s_api.get_limit_rows_by_last_id(
last_pk=delivery_client.s_id,
limit_num=delivery_client.limit_num,
)
if not s_rows:
break
for s_data in s_rows:
delivery_client.s_data = s_data
delivery_client.s_id = delivery_client.s_data.id
delivery_client.latest_time = time_local_to_utc(delivery_client.s_data.updated)
# ----------
# 制单人员
user_client.m_data = user_client.m_api.get_row(
table_name='user',
pk_source=delivery_client.s_data.makerid,
)
if not user_client.m_data:
continue
# 审核人员
auditor_client.m_data = user_client.m_api.get_row(
table_name='user',
pk_source=delivery_client.s_data.auditorid,
)
auditor_client.t_id = auditor_client.m_data.pk_target if auditor_client.m_data else 0
# 客户信息
customer_client.m_data = customer_client.m_api.get_row(
table_name='customer',
pk_source=delivery_client.s_data.idsettleCustomer,
)
if not customer_client.m_data:
continue
customer_client.t_data = customer_client.t_api.get_row_by_id(customer_client.m_data.pk_target)
if not customer_client.t_data:
continue
# 联系方式
customer_contact_client.t_data = customer_contact_client.t_api.get_row(
cid=customer_client.m_data.pk_target,
)
if not customer_contact_client.t_data:
continue
# 仓库
warehouse_client.m_data = warehouse_client.m_api.get_row(
table_name='warehouse',
pk_source=delivery_client.s_data.idwarehouse,
)
if not warehouse_client.m_data:
continue
# ----------
delivery_client.m_detail()
# 存在历史数据
if delivery_client.m_data:
if not delivery_client.s_data.updated:
continue
if delivery_client.latest_time <= delivery_client.m_data.latest_time:
continue
# ----------
# 更新目标数据
delivery_client.t_id = delivery_client.m_data.pk_target
create_time = time_local_to_utc(delivery_client.s_data.createdtime)
update_time = time_local_to_utc(delivery_client.s_data.updated)
type_tax = TYPE_TAX_HAS if delivery_client.s_data.taxAmount > delivery_client.s_data.amount else TYPE_TAX_NOT
delivery_client.t_data = {
'uid': user_client.m_data.pk_target,
'customer_cid': customer_client.t_data.id,
'customer_company_name': customer_client.t_data.company_name,
'customer_contact_id': customer_contact_client.t_data.id,
'type_delivery': TYPE_DELIVERY_NORMAL,
'amount_production': delivery_client.s_data.taxAmount,
'amount_delivery': delivery_client.s_data.taxAmount,
'warehouse_id': warehouse_client.m_data.pk_target,
'note': delivery_client.s_data.memo,
'type_tax': type_tax,
'update_time': update_time, # 本地时间修改为UTC时间
}
# 删除条件
# if delivery_client.s_data.disabled:
# delivery_client.t_data['status_delete'] = True
# delivery_client.t_data['delete_time'] = current_time
# 审核条件
if auditor_client.t_id:
delivery_client.t_data['status_audit'] = True
delivery_client.t_data['audit_uid'] = auditor_client.t_id
delivery_client.t_data['audit_time'] = update_time
# ----------
delivery_client.t_update()
delivery_client.m_update()
# 明细数据
# 清空历史
delivery_items_history_rows = delivery_items_client.t_api.get_rows(
delivery_id=delivery_client.t_id,
)
for delivery_items_history_data in delivery_items_history_rows:
delivery_items_client.t_api.delete(delivery_items_history_data.id)
# 全部更新
s_i_rows = delivery_items_client.s_api.get_rows(idSaleDeliveryDTO=delivery_client.s_data.id)
for s_i_data in s_i_rows:
delivery_items_client.s_data = s_i_data
delivery_items_client.s_id = delivery_items_client.s_data.id
delivery_items_client.latest_time = time_local_to_utc(delivery_client.s_data.updated)
# 产品
production_client.m_data = production_client.m_api.get_row(
table_name='production',
pk_source=delivery_items_client.s_data.idinventory,
)
if not production_client.m_data:
continue
production_client.t_data = production_client.t_api.get_row_by_id(production_client.m_data.pk_target)
if not production_client.t_data:
continue
production_client.s_data = production_client.s_api.get_row_by_id(production_client.m_data.pk_source)
if not production_client.s_data:
continue
# 仓位（产品编号 > 产品详情 > 产品仓位）
rack_client.m_data = rack_client.m_api.get_row(
table_name='rack',
pk_source=production_client.s_data.idinvlocation, # Fixme
)
# 更新明细
delivery_items_client.t_data = {
'delivery_id': delivery_client.t_id,
'uid': user_client.m_data.pk_target,
'customer_cid': customer_client.t_data.id,
'customer_company_name': customer_client.t_data.company_name,
'production_id': production_client.t_data.id,
'production_brand': production_client.t_data.production_brand,
'production_model': production_client.t_data.production_model,
'production_sku': production_client.t_data.production_sku,
'warehouse_id': warehouse_client.m_data.pk_target,
'rack_id': rack_client.m_data.pk_target if rack_client.m_data else 0,
'type_tax': type_tax,
'quantity': delivery_items_client.s_data.quantity,
'unit_price': delivery_items_client.s_data.taxPrice,
'create_time': create_time, # 本地时间修改为UTC时间
'update_time': update_time, # 本地时间修改为UTC时间
}
# 删除条件 Fixme
# if delivery_client.s_data.disabled:
# current_time = datetime.utcnow()
# delivery_items_client.t_data['status_delete'] = True,
# delivery_items_client.t_data['delete_time'] = current_time,
delivery_items_client.t_create()
# 没有历史数据
else:
# 目标数据去重
# delivery_client.t_data = delivery_client.t_api.get_row(name=delivery_client.s_data.name)
# if delivery_client.t_data:
# continue
# ----------
# 创建目标数据
create_time = time_local_to_utc(delivery_client.s_data.createdtime)
update_time = time_local_to_utc(delivery_client.s_data.updated)
type_tax = TYPE_TAX_HAS if delivery_client.s_data.taxAmount > delivery_client.s_data.amount else TYPE_TAX_NOT
delivery_client.t_data = {
'uid': user_client.m_data.pk_target,
'customer_cid': customer_client.t_data.id,
'customer_company_name': customer_client.t_data.company_name,
'customer_contact_id': customer_contact_client.t_data.id,
'type_delivery': TYPE_DELIVERY_NORMAL,
'amount_production': delivery_client.s_data.taxAmount,
'amount_delivery': delivery_client.s_data.taxAmount,
'warehouse_id': warehouse_client.m_data.pk_target,
'note': delivery_client.s_data.memo,
'type_tax': type_tax,
'create_time': create_time, # 本地时间修改为UTC时间
'update_time': update_time, # 本地时间修改为UTC时间
}
# 删除条件
# if delivery_client.s_data.disabled:
# delivery_client.t_data['status_delete'] = True
# delivery_client.t_data['delete_time'] = current_time
# 审核条件
if auditor_client.t_id:
delivery_client.t_data['status_audit'] = True
delivery_client.t_data['audit_uid'] = auditor_client.t_id
delivery_client.t_data['audit_time'] = update_time
# ----------
delivery_client.t_create()
delivery_client.m_create()
# 明细数据
s_i_rows = delivery_items_client.s_api.get_rows(idSaleDeliveryDTO=delivery_client.s_data.id)
for s_i_data in s_i_rows:
delivery_items_client.s_data = s_i_data
delivery_items_client.s_id = delivery_items_client.s_data.id
delivery_items_client.latest_time = time_local_to_utc(delivery_client.s_data.updated)
# 产品
production_client.m_data = production_client.m_api.get_row(
table_name='production',
pk_source=delivery_items_client.s_data.idinventory,
)
if not production_client.m_data:
continue
production_client.t_data = production_client.t_api.get_row_by_id(production_client.m_data.pk_target)
if not production_client.t_data:
continue
production_client.s_data = production_client.s_api.get_row_by_id(production_client.m_data.pk_source)
if not production_client.s_data:
continue
# 仓位（产品编号 > 产品详情 > 产品仓位）
rack_client.m_data = rack_client.m_api.get_row(
table_name='rack',
pk_source=production_client.s_data.idinvlocation, # Fixme
)
# 插入明细
delivery_items_client.t_data = {
'delivery_id': delivery_client.t_id,
'uid': user_client.m_data.pk_target,
'customer_cid': customer_client.t_data.id,
'customer_company_name': customer_client.t_data.company_name,
'production_id': production_client.t_data.id,
'production_brand': production_client.t_data.production_brand,
'production_model': production_client.t_data.production_model,
'production_sku': production_client.t_data.production_sku,
'warehouse_id': warehouse_client.m_data.pk_target,
'rack_id': rack_client.m_data.pk_target if rack_client.m_data else 0,
'type_tax': type_tax,
'quantity': delivery_items_client.s_data.quantity,
'unit_price': delivery_items_client.s_data.taxPrice,
'create_time': create_time, # 本地时间修改为UTC时间
'update_time': update_time, # 本地时间修改为UTC时间
}
# 删除条件 Fixme
# if delivery_client.s_data.disabled:
# current_time = datetime.utcnow()
# delivery_items_client.t_data['status_delete'] = True,
# delivery_items_client.t_data['delete_time'] = current_time,
delivery_items_client.t_create()
result = {
'来源总数': delivery_client.s_api.count(),
'目标总数': delivery_client.t_api.count(),
}
return result
def sync():
inventory_client = MigrationClient('inventory', SourceInventory,
TargetInventory, MigrationContrast)
production_client = MigrationClient('production', SourceProduction,
TargetProduction, MigrationContrast)
warehouse_client = MigrationClient('warehouse', None, TargetWarehouse,
MigrationContrast)
rack_client = MigrationClient('rack', None, TargetRack, MigrationContrast)
while 1:
s_rows = inventory_client.s_api.get_limit_rows_by_last_id(
last_pk=inventory_client.s_id,
limit_num=inventory_client.limit_num,
)
if not s_rows:
break
for s_data in s_rows:
inventory_client.s_data = s_data
inventory_client.s_id = inventory_client.s_data.id
inventory_client.latest_time = time_local_to_utc(
inventory_client.s_data.updated)
# ----------
# 产品
production_client.m_data = production_client.m_api.get_row(
table_name='production',
pk_source=inventory_client.s_data.idinventory,
)
if not production_client.m_data:
continue
production_client.t_data = production_client.t_api.get_row_by_id(
production_client.m_data.pk_target)
# 仓库
warehouse_client.m_data = warehouse_client.m_api.get_row(
table_name='warehouse',
pk_source=inventory_client.s_data.idwarehouse,
)
if not warehouse_client.m_data:
continue
warehouse_client.t_data = warehouse_client.t_api.get_row_by_id(
warehouse_client.m_data.pk_target)
# 仓位
production_client.s_data = production_client.s_api.get_row_by_id(
inventory_client.s_data.idinventory)
if not production_client.s_data:
continue
rack_client.m_data = rack_client.m_api.get_row(
table_name='rack',
pk_source=production_client.s_data.idinvlocation,
)
if not rack_client.m_data:
continue
rack_client.t_data = rack_client.t_api.get_row_by_id(
rack_client.m_data.pk_target)
# ----------
inventory_client.m_detail()
# 存在历史数据
if inventory_client.m_data:
if not inventory_client.s_data.updated:
continue
if inventory_client.latest_time <= inventory_client.m_data.latest_time:
continue
# ----------
# 更新目标数据
inventory_client.t_id = inventory_client.m_data.pk_target
current_time = datetime.utcnow()
inventory_client.t_data = {
'production_id': production_client.t_data.id,
'production_brand':
production_client.t_data.production_brand,
'production_model':
production_client.t_data.production_model,
'production_sku': production_client.t_data.production_sku,
'warehouse_id': warehouse_client.t_data.id,
'warehouse_name': warehouse_client.t_data.name,
'rack_id': rack_client.t_data.id,
'rack_name': rack_client.t_data.name,
'stock_qty_initial': inventory_client.s_data.baseQuantity,
'stock_qty_current': inventory_client.s_data.baseQuantity,
'update_time': current_time,
}
# 删除条件
# if inventory_client.s_data.disabled:
# inventory_client.t_data['status_delete'] = True
# inventory_client.t_data['delete_time'] = current_time
# ----------
inventory_client.t_update()
inventory_client.m_update()
# 没有历史数据
else:
# 目标数据去重
inventory_client.t_data = inventory_client.t_api.get_row(
production_id=production_client.t_data.id,
warehouse_id=warehouse_client.t_data.id,
rack_id=rack_client.t_data.id,
)
if inventory_client.t_data:
continue
# ----------
# 创建目标数据
current_time = datetime.utcnow()
inventory_client.t_data = {
'production_id': production_client.t_data.id,
'production_brand':
production_client.t_data.production_brand,
'production_model':
production_client.t_data.production_model,
'production_sku': production_client.t_data.production_sku,
'warehouse_id': warehouse_client.t_data.id,
'warehouse_name': warehouse_client.t_data.name,
'rack_id': rack_client.t_data.id,
'rack_name': rack_client.t_data.name,
'stock_qty_initial': inventory_client.s_data.baseQuantity,
'stock_qty_current': inventory_client.s_data.baseQuantity,
'create_time': current_time,
'update_time': current_time,
}
# 删除条件
# if inventory_client.s_data.disabled:
# inventory_client.t_data['status_delete'] = True
# inventory_client.t_data['delete_time'] = current_time
# ----------
inventory_client.t_create()
inventory_client.m_create()
result = {
'来源总数': inventory_client.s_api.count(),
'目标总数': inventory_client.t_api.count(),
}
return result
def sync():
user_client = MigrationClient('user', SourceUser, TargetUser,
MigrationContrast)
while 1:
s_rows = user_client.s_api.get_limit_rows_by_last_id(
last_pk=user_client.s_id,
limit_num=user_client.limit_num,
)
if not s_rows:
break
for s_data in s_rows:
user_client.s_data = s_data
user_client.s_id = user_client.s_data.id
user_client.latest_time = time_local_to_utc(
user_client.s_data.updated)
user_client.m_detail()
# 存在历史数据
if user_client.m_data:
if not user_client.s_data.updated:
continue
if user_client.latest_time <= user_client.m_data.latest_time:
continue
# ----------
# 更新目标数据
user_client.t_id = user_client.m_data.pk_target
current_time = datetime.utcnow()
user_client.t_data = {
'name': user_client.s_data.Code,
'mobile': user_client.s_data.name,
'update_time': current_time,
}
# 删除条件
if user_client.s_data.issystem or user_client.s_data.isStoped:
user_client.t_data['status_delete'] = True
user_client.t_data['delete_time'] = current_time
# ----------
user_client.t_update()
user_client.m_update()
# 没有历史数据
else:
# 目标数据去重
user_client.t_data = user_client.t_api.get_row(
name=user_client.s_data.Code)
if user_client.t_data:
continue
# ----------
# 创建目标数据
current_time = datetime.utcnow()
user_client.t_data = {
'name': user_client.s_data.Code,
'mobile': user_client.s_data.name,
'create_time': current_time,
'update_time': current_time,
}
# 删除条件
if user_client.s_data.issystem or user_client.s_data.isStoped:
user_client.t_data['status_delete'] = True
user_client.t_data['delete_time'] = current_time
# ----------
user_client.t_create()
user_client.m_create()
result = {
'来源总数': user_client.s_api.count(),
'目标总数': user_client.t_api.count(),
}
return result
def test():
utc_time = time_local_to_utc('2017-06-16 11:21:01')
print(utc_time)
print(type(utc_time))
