class WorkProc(SProcess):
_logger = None
_check_service = None
_browser = None
_main_dev = None
_mutex = threading.Lock()
def __init__(self, mainDev):
super(WorkProc, self).__init__()
self._logger = logging.getLogger(str(__name__))
self._check_service = CheckService(self)
self.registerService(self._check_service)
self._main_dev = mainDev
self._browser = Browser()
pass
def getMainDev(self):
return self._main_dev
pass
def checkData(self,numiid):
infoUrl = "http://hws.m.taobao.com/cache/mtop.wdetail.getItemFullDesc/4.1/?data=%7B%22item_num_id%22%3A%22"+numiid+"%22%7D"
data = self._browser.openurl(infoUrl)
m = re.search(r'SYSTEM_ERROR',data.read())
if m:
self._mutex.acquire()
self._main_dev.getDb().query("delete from ftxia_items where num_iid=%s" % (numiid))
self._main_dev.getDb().commit()
self._mutex.release()
self._logger.debug("SYSTEM_ERROR:%s"%(numiid))
def __init__(self, mainDev):
super(WorkProc, self).__init__()
self._logger = logging.getLogger(str(__name__))
self._check_service = CheckService(self)
self.registerService(self._check_service)
self._main_dev = mainDev
self._browser = Browser()
pass
def __init__(self, data_file_path):
self.data_file_path = data_file_path
data = self.load_data(data_file_path)
if not data:
raise Exception('Invalid data or no data file specified.')
self.data = data
self.jobs = data['jobs']
self.username = data['credentials']['username']
self.password = data['credentials']['password']
self.host = data['host']['url']
self.browser = Browser()
def suite_setup():
# 创建 浏览器
browser = Browser()
browser.open_browser()
browser.openUrl()
def __init__(self, code):
super(SolidotCmder, self).__init__(code)
self._logger = logging.getLogger(str(__name__))
self._brower = Browser()
pass
class SolidotCmder(BaseCmder):
_brower = None
_article_url = None
_cuid = None
def __init__(self, code):
super(SolidotCmder, self).__init__(code)
self._logger = logging.getLogger(str(__name__))
self._brower = Browser()
pass
def getItems(self):
urls = SysConfig.config['solidot']['urls']
cuids = SysConfig.config['solidot']['cuid']
for j in range(len(urls)):
url = urls[j]
html = self._brower.openurl(url).read().decode(
SysConfig.config['solidot']['charset']).encode('utf-8')
center = "<div id=\"center\">(.*?)<div class=\"page\">"
center_str = ReHelper.findAll(center, html)
item = "<a href=\"(/story\?sid=\d+?)\">.*?</a>"
items = ReHelper.findAll(item, center_str[0])
items = list(set(items))
for i in items:
new_url = url + i
print new_url
if not AppList.UrlList.contains(new_url):
cmder = SolidotCmder(OpCode.GET_ARTICLE)
cmder.setArticleUrl(new_url)
cmder.setCuid(cuids[j])
AppQueue.Cmders.put(cmder)
AppList.UrlList.append(new_url)
self._logger.debug("############:%s" % self._code)
pass
def getArticle(self):
html = self._brower.openurl(self._article_url).read().decode(
SysConfig.config['solidot']['charset']).encode('utf-8')
re_title = "<div class=\"bg_htit\">[\s]+<h2>([^<|^:]+)</h2>"
title = ReHelper.findAll(re_title, html)
re_conter = "<div class=\"p_mainnew\">(.*?)</div>[\s]+<div class=\"e_reply a_bold\">"
conter = ReHelper.findAll(re_conter, html)
conter = ReHelper.sub(
"(\t|\n| |<a.*?>|<img.*?>|</a>|</div>|<div.*?>|<u.*?>|</u>|<i.*?>|</i>)",
conter[0])
muid = SysConfig.config['solidot']['muid']
account = SysConfig.config['solidot']['account']
params = {
'cuid': self._cuid,
'muid': muid,
'title': title[0],
'content': conter,
'account': account
}
post_url = SysConfig.config['solidot']['postUrl']
cmder = HttpCmder(post_url, OpCode.HTTP_POST)
cmder.setParams(params)
AppQueue.HttpCmders.put(cmder)
self._logger.debug("############:%s" % self._code)
pass
def setCuid(self, cuid):
self._cuid = cuid
pass
def setArticleUrl(self, url):
self._article_url = url
pass