def reset_shutdown_status():
try:
file_name = Configs().system_shutdown_flag_filename
FileHelper.create(file_name, "0")
except:
LogGo.error("error while reseting shutdown flag file!!!")
def send_to_queue(result):
"""
传送
code: 0 到达最大访问频率
1 正常结果
:param request:
:param result:
:return:
"""
global target_mutex, target_count, target_transported_count, all_target_transported
code, value = result
target, detail_page_bundle_list, content_ruler, encode = value
if code == 1:
for detail_page_bundle in detail_page_bundle_list:
target_producer.target_queue.queue.put(
(target, detail_page_bundle, content_ruler, encode))
ScrabingTarget.set_last_access_date(target.id)
else:
LogGo.error("List Page Error:" + str(target.data_key) + " Code: " +
str(code))
ScrabingTarget.set_elog(target.id, "error code: " + str(code))
if target_mutex.acquire():
if target_count == target_transported_count:
all_target_transported = True
else:
target_transported_count += 1
LogGo.debug('target_transported_count: ' +
str(target_transported_count))
target_mutex.release()
def set_shutdown_status():
try:
file_name = Configs().system_shutdown_flag_filename
FileHelper.create(file_name, "1")
except:
LogGo.error("something wrong at setting shutdown flag file !!!")
def store_program(self, ids, programs):
status = False
if len(ids) != len(programs):
LogGo.error("ids count unmatch programs count")
return False
count = len(ids)
element = 0
if count > 0:
for id, program in zip(ids, programs):
try:
self.program.save(program, id)
element += 1
except Exception as e:
import traceback
msg = traceback.format_exc()
LogGo.warning(msg)
LogGo.info('Total :' + str(count) + ' / ' + str(element) +
' elements Saved!')
if element == 0:
status = False
else:
status = True
else:
LogGo.info("0 Element!")
return status
def check_shutdown_status():
try:
file_name = Configs().system_shutdown_flag_filename
status = int(StringHelper.trim(FileHelper.read(file_name)))
if status == 1:
somebody_help.reset_shutdown_status()
return True
else:
return False
except:
LogGo.error("system_shutdown_flag_file unavailable!")
return False
def store(self, news_list: [], article_list: [], heavy_list: [] = None):
status = False
if len(article_list) != len(news_list):
LogGo.error("news count unmatch article count")
return False
count = len(news_list)
element = 0
if count > 0:
if heavy_list is not None:
for news, article, heavy in zip(news_list, article_list,
heavy_list):
try:
id = genUUID()
self.news.save(news, id)
self.article.save(article, id)
self.heavy.save_with_news_id(heavy, id)
element += 1
except Exception as e:
LogGo.warning(repr(e))
else:
for news, article in zip(news_list, article_list):
try:
id = genUUID()
self.news.save(news, id)
self.article.save(article, id)
element += 1
except Exception as e:
LogGo.warning(repr(e))
LogGo.info('Total :' + str(count) + ' / ' + str(element) +
' elements Saved!')
if element == 0:
status = False
else:
status = True
# return True
else:
LogGo.info("0 Element!")
# return False
return status
def fetch_detail(self, value):
"""value: (target, original dic, ruler, encode )
"""
LogGo.debug("in detail :" + str(value))
target, detail_page_dic, content_ruler, encode = value
delegate = None
if target.type == 'ulweb' or target.type == 'jsweb':
delegate = self.web.scrape_detail
elif target.type == 'newrank':
delegate = self.wechat.newrank_detail
elif target.type == 'gsdata':
delegate = self.wechat.gs_detail
elif target.type == 'weibo':
delegate = self.weibo.weibo_detail
code, result_dic = delegate(target, detail_page_dic, content_ruler,
encode)
if code is 1:
self.upload_queue.queue.put(result_dic)
else:
LogGo.error("Detail Error: " + str(detail_page_dic))
"requests==2.12.4\r\n"
"shellescape==3.4.1\r\n"
"six==1.10.0\r\n"
"stem==1.6.0\r\n"
"tzlocal==1.3\r\n"
"Werkzeug==0.11.15\r\n")
print(" ")
print("------ ---- --- ---getIndexNumber--- ------ ---- --")
print(" ")
scan = Scanner()
while True:
LogGo.info(">>> New Loop -->")
try:
scan.start()
except Exception as e:
import traceback
msg = traceback.format_exc()
LogGo.error(msg)
if Configs.infinity:
while not somebody_help.isthataworkday(Configs().work_sequence):
time.sleep(60 * 60)
else:
break
LogGo.info("-- GH Offline --")
print("---------------------------------------------")
