def hct_main(cls):
# a = [8443674831, 6832892964, 8703057812]
sql_stat = (
'''select [ORD_NUM], [PACKAGE_NO] from [dbo].[LOGISTIC_STATUS]
where [SCT_DESC] = '新竹貨運' and [PACKAGE_STATUS] = 0 ''')
result = connection.db('AZURE').do_query(sql_stat)
threads = []
### 向隊列中放入任務, 真正使用時, 應該設置為可持續的放入任務
for task in result:
cls.SHARE_Q.put(task)
### 開啟_WORKER_THREAD_NUM個線程
for i in range(cls._WORKER_THREAD_NUM):
thread = MyThread(cls.worker)
thread.start() ### 線程開始處理任務
time.sleep(0.67)
threads.append(thread)
for thread in threads:
thread.join()
### 等待所有任務完成
cls.SHARE_Q.join()
def tcat_main(cls):
# a = [905244040160, 905244040145]
sql_stat = (
'''select [ORD_NUM], [PACKAGE_NO] from [dbo].[LOGISTIC_STATUS]
where [SCT_DESC] = '統一速達(黑貓宅急便)' and [PACKAGE_STATUS] = 0 '''
)
result = connection.db('AZURE').do_query(sql_stat)
threads = []
for task in result:
cls.SHARE_Q.put(task)
for i in range(cls._WORKER_THREAD_NUM):
thread = MyThread(cls.worker)
# time.sleep(0.27)
# thread.setDaemon(True)
thread.start()
threads.append(thread)
for thread in threads:
thread.join()
cls.SHARE_Q.join()
def ecan_main(cls):
# a = [778014468840, 777039297694, 777039300520]
sql_stat = (
'''select [ORD_NUM], [PACKAGE_NO] from [dbo].[LOGISTIC_STATUS]
where [SCT_DESC] = '台灣宅配通' and [PACKAGE_STATUS] = 0 '''
) # and [PACKAGE_STATUS] = 0 and [package_no] = '777060026116'
result = connection.db('AZURE').do_query(sql_stat)
threads = []
for task in result:
cls.SHARE_Q.put(task)
for i in range(cls._WORKER_THREAD_NUM):
thread = MyThread(cls.worker)
thread.start()
threads.append(thread)
for thread in threads:
thread.join()
cls.SHARE_Q.join()
def maple_main(cls):
# a = ['010145523083']
sql_stat = (
'''select [ORD_NUM], [PACKAGE_NO] from [dbo].[LOGISTIC_STATUS]
where [SCT_DESC] = '豐業物流(便利帶)' and [PACKAGE_STATUS] = 0 '''
) #
result = connection.db('AZURE').do_query(sql_stat)
threads = []
for task in result:
cls.SHARE_Q.put(task)
for i in range(cls._WORKER_THREAD_NUM):
thread = MyThread(cls.worker)
thread.start()
time.sleep(0.3)
threads.append(thread)
for thread in threads:
thread.join()
cls.SHARE_Q.join()
def pstmail_main(cls):
# a = ['00898360203218', '00487830300816']
sql_stat = (
'''select [ORD_NUM], [PACKAGE_NO] from [dbo].[LOGISTIC_STATUS]
where [SCT_DESC] = '郵局' and [PACKAGE_STATUS] = 0 ''')
result = connection.db('AZURE').do_query(sql_stat)
threads = []
for task in result:
cls.SHARE_Q.put(task)
for i in range(cls._WORKER_THREAD_NUM):
thread = MyThread(cls.worker)
# time.sleep(0.37)
# thread.setDaemon(True)
thread.start()
# thread.join()
threads.append(thread)
for thread in threads:
thread.join()
cls.SHARE_Q.join()
def update(status, pack_no):
sql_stat = ('''update [dbo].[LOGISTIC_STATUS]
set [PACKAGE_STATUS] = %s, [PARSE_DATE] = CONVERT(VARCHAR(19), GETDATE(), 120)
where [PACKAGE_NO] = %s''')
connection.db('AZURE').do_query(sql_stat, (status, str(pack_no)))
def parse_page_to_db(data):
def replace_string(product_name):
replace_string = [
'/', ':', '【', '】', '《', '》', '+', '※', "'", '$', '(', ')', '_',
'<', '>', '★', '?', '-', '-', '#', '“', '紅', '橙', '黃', '綠', '藍',
'紫', '白', '灰', '黑', '銀', '停', '用', '[', ']'
]
for i in product_name:
if i in replace_string:
product_name = product_name.replace(i, ' ')
return product_name.replace('福利網獨享', '').replace('單一規格', '')
def len_verify(obj_list, length):
while len(obj_list) < length:
obj_list.append('')
while len(obj_list) > length:
del obj_list[-1]
return obj_list
query_name = replace_string(data[0])
terms = urllib.parse.quote(query_name)
result = connection.request.get_page(
'https://www.google.com.tw/search?&q=' + terms + '&oq=' + terms)
### 目前顯示的是以下字詞的搜尋結果
correct_word = list()
if result.find_all('a', {'class': 'spell'}):
for i in result.find_all('a', {'class': 'spell'}):
correct_word.append(i.text)
else:
correct_word.append('')
correct_word = len_verify(correct_word, 1)
### rec_index
rec_index_idx = dict()
for i in result.find_all('em'):
if i.text in rec_index_idx:
count = rec_index_idx.get(i.text) + 1
rec_index_idx[i.text] = count
else:
rec_index_idx[i.text] = 1
rec_index_lst = [
i[0] for i in sorted(
rec_index_idx.items(), key=lambda x: x[1], reverse=True)[:3]
]
rec_index_lst = len_verify(rec_index_lst, 3)
### pin_yin
pin_yin_modify1 = [
i.replace(re.sub(r'[^a-zA-Z\d+]', '', i), '') for i in query_name
if i.isalpha()
]
pin_yin_modify2 = [i for i in pin_yin_modify1 if i != '']
pin_yin = [
i[0] for i in pypinyin.pinyin(''.join(pin_yin_modify2),
style=pypinyin.BOPOMOFO)
]
pin_yin_last = [''.join(pin_yin)]
pin_yin_last = len_verify(pin_yin_last, 1)
### 相關搜尋
rec_word = list()
if result.find_all('p', {'class': '_e4b'}):
for i in result.find_all('p', {'class': '_e4b'}):
rec_word.append(i.text)
else:
rec_word = ['' for _ in range(10)]
rec_word = len_verify(rec_word, 10)
### search_url
search_url = [
'https://www.google.com.tw/search?&q=' + terms + '&oq=' + terms
]
### date
date = [datetime.datetime.today().strftime("%Y-%m-%d")]
to_db = [
data[0], data[1]
] + correct_word + rec_index_lst + pin_yin_last + rec_word + search_url + date
sql_stat = ("""insert into [dbo].[BI_GA_USERS_SEARCH_NULL]
values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"""
)
try:
connection.db('NP').do_query(sql_stat, tuple(to_db))
except Exception as e:
connection.log.WRITE('solr', e, 'solr')