def extract(system: str, kinetics: str): log_data = { "UID": CURRENT_JOB, "System": system, "Kinetics": kinetics, "TotalLipid": total_lipid_concentration, "Analysis": "Converting Old Parameters", "version": "3.0" } LOG.info(json.dumps(log_data, sort_keys=True)) progress_counter = 0 all_para = get_parameters() for para in all_para: initial_con = get_random_concentrations(total_lipid_concentration, system) sanity_counter = 0 update_progress(progress_counter / len(all_para), "Extracting Old Parameters") while sanity_counter < 10000: para[E_SOURCE].k = np.random.uniform(min_k, max_k) output = get_concentration_profile(system, initial_con, para, ode_end_time, ode_slices) e = calculate_wild_type_error(output[-1]) if e < save_cutoff: save_para(para, e) break sanity_counter += 1 progress_counter += 1
def parse_acct_search(self, content="", save=True): _content = content soup = BeautifulSoup(_content, 'html') wechat_acct = soup.find("label", {"name": "em_weixinhao"}).text wechat_name = soup.find("em").text self.wechat_acct = wechat_acct self.wechat_name = wechat_name res = soup.find("div", {"class": "results"}) login_param = urlparse(res.div["href"]).query self.login_info = login_param if self.wechat_acct and self.login_info and save: with MongodbUtils(config.WECHAT_DB_IP, config.WECHAT_DB_PORT, config.WECHAT_COLLECTION, config.WECHAT_ACCOUNT_TABLE) as connect_db: _res = connect_db.find_one({"account": self.wechat_acct}) # acct_param = _res.get("param", "") if not _res: connect_db.update({"account": self.wechat_acct}, { "$set": { "keyword": self.keyword, "account": self.wechat_acct, "url_param": login_param, "account_name": self.wechat_name, "real_acct": self.wechat_acct, "category": "", "tag": [], } }, upsert=True) LOG.info("%s is add to the database" % (self.wechat_acct)) else: LOG.info("%s is already in the database" % (self.wechat_acct)) return login_param
def sync_verify_token(token): """ verify request token :param token: request token :return: verify result """ try: verify_token_body = { "auth": { "token": { "id": token }, "tenantName": options.tenant_name } } resp = sync_request(url=options.auth_url, body=verify_token_body, method=METHOD_POST) if resp: session = __get_session(resp) else: raise exception.OpenStackError except Exception as e: LOG.error(e) raise e return session
def parse_acct_search(self, content="", save=True): _content = content soup = BeautifulSoup(_content, 'html') wechat_acct = soup.find("label", {"name": "em_weixinhao"}).text wechat_name = soup.find("em").text self.wechat_acct = wechat_acct self.wechat_name = wechat_name res = soup.find("div", {"class": "results"}) login_param = urlparse(res.div["href"]).query self.login_info = login_param if self.wechat_acct and self.login_info and save: with MongodbUtils(config.WECHAT_DB_IP, config.WECHAT_DB_PORT, config.WECHAT_COLLECTION, config.WECHAT_ACCOUNT_TABLE) as connect_db: _res = connect_db.find_one({"account": self.wechat_acct}) # acct_param = _res.get("param", "") if not _res: connect_db.update({"account": self.wechat_acct}, {"$set": { "keyword": self.keyword, "account": self.wechat_acct, "url_param": login_param, "account_name": self.wechat_name, "real_acct": self.wechat_acct, "category": "", "tag": [], }}, upsert=True) LOG.info("%s is add to the database" % (self.wechat_acct)) else: LOG.info("%s is already in the database" % (self.wechat_acct)) return login_param
def getPhoneInfo(devices): """获取设备的一些基本信息""" cmd = "adb -s " + devices + " shell cat /system/build.prop" LOG.info("执行命令:%s" % cmd) phone_info = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).stdout.readlines() release = "ro.build.version.release=" # 版本 model = "ro.product.model=" # 型号 brand = "ro.product.brand=" # 品牌 device = "ro.product.device=" # 设备名 result = {"release": release} for line in phone_info: for i in line.split(): temp = i.decode() if temp.find(release) >= 0: result["release"] = temp[len(release):] break if temp.find(model) >= 0: result["model"] = temp[len(model):] break if temp.find(brand) >= 0: result["brand"] = temp[len(brand):] break if temp.find(device) >= 0: result["device"] = temp[len(device):] break LOG.info("移动设备信息:%s" % result) return result
def sync_ops_request(token, service_type, url, method, response_key=None, body=None, interface=INTF_TYPE_ADMIN): """ sync request OpenStack REST API :param token: OpenStack request session: Session :param response_key: :param service_type: service type :param url: request url :param method:request method :param body:request body :param interface: 'admin' - default, 'public', 'internal' :return: the dict of response """ interface_key = interface + 'URL' if service_type in token.urls \ and interface_key in token.urls[service_type]: service_url = token.urls[service_type][interface_key] else: raise Exception() try: url = "%s%s" % (service_url, url) resp = sync_request(url=url, method=method, body=body, token=token.id, response_key=response_key) except Exception as e: LOG.error("ops_request failed: %s" % e) raise e else: return resp
def __get_session(token): """get the session of OpenStack service link. :param token: request token :return: the session of request """ try: token_id = token.get("token").get("id") services = token.get("serviceCatalog") tenant_id = token.get("token").get("tenant").get("id") user_id = token.get("user").get("id") urls = {} for item in services: if item.get("type") in REQUEST_TYPES: for endpoint in item.get("endpoints"): if endpoint.get('region') == options.region_name: urls[item.get("type")] = endpoint session = Session(id=token_id, urls=urls, token=token, tenant_id=tenant_id, user_id=user_id) except Exception as e: LOG.error(e) raise e return session
def __init__(self): """检测待测试APK文件是否存在""" if not target_app: raise BaseErorr("未指定待测试的APK文件!") target_app_path = os.path.join(app_path, target_app) if not os.path.exists(target_app_path): raise BaseErorr("未找到指定测试的APK文件!") else: LOG.info("开始测试APP:%s..." % target_app) """检测是否连接移动设备""" self.l_devices = [] devices = AndroidDebugBridge().attached_devices() if len(devices) > 0: for dev in devices: app = {"devices": dev, "port": "4723"} self.l_devices.append(app) else: raise BaseErorr("没有可用的安卓设备!") """获取Apk基本信息""" self.app_info = getApkBaseInfo(target_app_path) """向移动设备中安装Apk文件""" pool_app_list = [] for i in range(0, len(self.l_devices)): pool_app_list.append({ "device": self.l_devices[i]["devices"], "info": self.app_info, "path": target_app_path }) pool = Pool(len(pool_app_list)) pool.map(installApp, pool_app_list) pool.close() pool.join() """启动appium""" appium_server = AppiumServer(self.l_devices) appium_server.start_server()
def runnerPool(self): """ 根据连接的设备生成不同的dict 然后放到设备的list里面 设备list的长度产生进程池大小 """ devices_pool = [] for i in range(0, len(self.l_devices)): _initApp = { "deviceName": self.l_devices[i]["devices"], "udid": self.l_devices[i]["devices"], "platformVersion": getPhoneInfo(devices=self.l_devices[i]["devices"])["release"], "platformName": "android", "port": self.l_devices[i]["port"], "appPackage": self.app_info["packageName"], "appActivity": self.app_info["appActivity"] } devices_pool.append(_initApp) LOG.info("进程数量:%s" % len(devices_pool)) pool = Pool(len(devices_pool)) # """运行测试用例""" pool.map(runnerCaseApp, devices_pool) pool.close() pool.join()
def runner_pool(getDevices): """ 根据链接的设备生成不同的dict 然后放到设备的list里面 设备list的长度产生进程池大小 """ devices_Pool = [] for device in getDevices: _pool = [] _initApp = { "udid": device["devices"], "port": device["port"], "deviceName": device["devices"], "platformVersion": getPhoneInfo(devices=device["devices"])["release"], "platformName": "android", "appPackage": 'com.aixuetang.online', "appActivity": 'com.aixuetang.mobile.activities.LaunchActivity' } _pool.append(_initApp) devices_Pool.append(_initApp) pools = Pool(processes=1) # 定义CPU核数量为3 res = pools.map(runner_case_app, devices_Pool) # 把测试用例放到设置到进程池 LOG.info(res) pools.close() pools.join()
def send_email(self, to_list, cc_list, sub=None, content=None, file_path=None, subtype='html'): msg = MIMEMultipart() if sub is None: msg['Subject'] = '无标题邮件' else: msg['Subject'] = sub msg['From'] = self._me msg['To'] = ";".join(to_list) msg['Cc'] = ";".join(cc_list) if content is None: puretext = MIMEText('该邮件未输入任何内容') msg.attach(puretext) else: puretext = MIMEText(content, _subtype=subtype, _charset='utf-8') msg.attach(puretext) if file_path is not None: for tuple_path in tuple(file_path): mail_file = MIMEApplication(open(tuple_path, 'rb').read()) mail_file.add_header('Content-Disposition', 'attachment', filename='%s' % tuple_path.split('/')[-1]) msg.attach(mail_file) try: self._server.sendmail(self._me, to_list, msg.as_string()) return True except Exception as e: LOG.error(str(e)) return False
def test_web(url): time.sleep(10) flag = False LOG.info("测试访问:%s..." % url) response = urllib.request.urlopen(url, timeout=5) if str(response.getcode()).startswith("2"): flag = True return flag
def _check_process(self): try: out = os.popen(self._cmd, 'r').read() return [tuple(line.split()) for line in out.splitlines()] except KeyboardInterrupt as e: pass except Exception as e: LOG.info(str(e))
def reg(self,**kwargs): f = self.open.exce_case(**kwargs) if f['code']==1: LOG.info('无法获取断言') return else: beijing = f['data'] return beijing
def test_swipe(self): # 打印屏幕高和宽 window_size = self.driver.get_window_size() x = window_size["width"] y = window_size["height"] LOG.info("印屏幕大小 window={}".format(window_size)) self.driver.swipe(6 / 7 * x, 1 / 2 * y, 1 / 7 * x, 1 / 2 * y, 100)
def open_da(path): try: with open('{}'.format(path), 'r', encoding='utf-8') as file: data = yaml.load(file, Loader=yaml.FullLoader) return {'code': 0, 'data': data} except Exception as e: LOG.info('yaml文档解析失败!原因:{}'.format(e)) return {'code': 1, 'data': e}
def _wrap(*args, **kwargs): try: return f(*args, **kwargs) except UnicodeEncodeError as e: raise e except Exception as e: LOG.exception('DB exception wrapped.') raise DBError(e)
def write_recording(cpu, neicun, devices): try: with open(recording, 'a', encoding='utf-8') as f: m = '%s:cpu:%s,内存:%s' % (devices, cpu, neicun) f.write(m + '\n') f.close() except Exception as e: LOG.info('写入性能数据失败!失败原因:%s' % e)
def delete(self, obj_typ, id=None, obj=None): if not (id or obj): LOG.error('Give either net_id or net_obj') if obj: id = obj.get('id') return self.sendjson('delete', '%(obj_typ)s/%(id)s' % {'obj_typ': obj_typ, 'id': id})
def on_submitBtn_clicked(self): caseName = self.caseNameEdit.text() testPoint = self.testPointEdit.text() httpStatus = self.httpStatusEdit.text() #前置处理 preDict = {} rowCount = self.preTableView.rowCount() for i in range(rowCount): key = self.preTableView.cellWidget(i,0).text() value = self.preTableView.cellWidget(i,1).text() preDict.setdefault(key,value) #请求参数 reqestDict = {} rowCount = self.reTableView.rowCount() for i in range(rowCount): key = self.reTableView.cellWidget(i,1).text() value = self.reTableView.cellWidget(i,2).text() reqestDict.setdefault(key,value) #预期结果 expected = self.expectedEdit.toPlainText() #后置处理 postposition = self.postpositionEdit.toPlainText() #获取签名 temp = [] signvalue="" if len(keyList)>0: for mykey in keyList: val = reqestDict.get(mykey) temp.append(val) signvalue = md5Encode(temp) reqestDict.setdefault("sign", signvalue) LOG.info('签名:%s,%s' % (keyList, signvalue)) caseObj = Cases() caseObj.casename = caseName caseObj.faceid = self.faceid caseObj.userid = self.userid caseObj.testpoint = testPoint caseObj.httpstatus = httpStatus caseObj.sign = signvalue caseObj.pretreat = preDict caseObj.request = reqestDict caseObj.expected = expected caseObj.postposition = postposition caseObj.updatetime = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') caseObj.keylist = keyList count = self.casedb.createCase(caseObj) if count > 0: QMessageBox.information(self, "提示", "用例添加成功", QMessageBox.Yes, QMessageBox.Yes) self.add_case_success_signal.emit() self.close() self.clearEdit()
def getTotalRecordCount(self, id=None): '''获取总记录数''' sqlBuilder = [] sqlBuilder.append("select * from tc_interface where 1=1") if id is not None: sqlBuilder.append(" and projectId=%s" % id) recordCount = self.mydb.executeQuery_count(''.join(sqlBuilder)) LOG.info(''.join(sqlBuilder)) return recordCount
def catch_error(*args, **kwargs): try: res = func(*args, **kwargs) except Exception as e: LOG.error(traceback.format_exc()) # print traceback.format_exc() return None return res
def catch_error(*args, **kwargs): try: res = func(*args, **kwargs) except Exception as e: LOG.error(traceback.format_exc()) # print traceback.format_exc() return None return res
def exec_rt(self, cmd, out_func=None): ''' 实时显示远端信息 Usage:: >>> out, err = ssh.exec_rt('top -b -n 5', self.write_message) # tornado websocket ''' LOG.info('SSH RT_CMD: %s' % cmd) stdin, stdout, stderr = self._client.exec_command(cmd, get_pty=True) channel = stdout.channel pending = err_pending = None if not out_func: out_func = print out, err = [], [] while not channel.closed or channel.recv_ready( ) or channel.recv_stderr_ready(): readq, _, _ = select.select([channel], [], [], 1) for c in readq: if c.recv_ready(): chunk = c.recv(len(c.in_buffer)) if pending is not None: chunk = pending + chunk lines = chunk.splitlines() if lines and lines[-1] and lines[-1][-1] == chunk[-1]: pending = lines.pop() else: pending = None for line in lines: out_func(line) line = line.decode('utf-8') out.append(line) if c.recv_stderr_ready(): chunk = c.recv_stderr(len(c.in_stderr_buffer)) if err_pending is not None: chunk = err_pending + chunk lines = chunk.splitlines() if lines and lines[-1] and lines[-1][-1] == chunk[-1]: err_pending = lines.pop() else: err_pending = None for line in lines: out_func(line) line = line.decode('utf-8') err.append(line) if err == ['[', ']']: err = [] if err: self._log(err, 'RT_ERR') else: self._log(out, 'RT_OUT') return out, err
def main(): try: app = Application() app.listen(address=options.address, port=options.port, max_body_size=TORNADO_MAX_BODY_SIZE) LOG.info('Sever Listen {port}...'.format(port=options.port)) tornado.ioloop.IOLoop.instance().start() except: LOG.error(traceback.format_exc())
def query(sql): try: conn = get_conn() cur = conn.cursor() cur.execute(sql) rows = cur.fetchall() return rows except Exception as e: LOG.error("get mysql connection: %s" % e) raise None
def tearDown(self): """ tearDown """ LOG.info("tearDown....") if self.driver.is_app_installed(AppPackage): self.driver.remove_app(AppPackage) LOG.info('测试用例执行完毕,测试环境正在还原!') time.sleep(5) self.driver.quit()
def runner_case_app(devices): """ 利用unittest的testsuite来组织测试用例 """ LOG.info(devices) test_suit = unittest.TestSuite() test_suit.addTest(Parmer.parametrize(Regtest, param=devices)) #扩展的其他的测试用例均这样添加 unittest.TextTestRunner(verbosity=2).run(test_suit)
def get_url_content(self, url, head={}): LOG.info("Visit URL: %s" % url) _headers = head if head else self.head req = urllib2.Request(url, None, _headers) response = urllib2.urlopen(req) # req = requests.get(url) _content = response.read() # req = self.opener.open(url) # _content = req.read() return _content
def get_url_content(self, url, head={}): LOG.info("Visit URL: %s" % url) _headers = head if head else self.head req = urllib2.Request(url, None, _headers) response = urllib2.urlopen(req) # req = requests.get(url) _content = response.read() # req = self.opener.open(url) # _content = req.read() return _content
def make_case_files(case_name, desc, funtion_name): LOG.info("开始生成测试用例文件") file_path = os.path.join(base_dir, 'cases/{}_case_test.py'.format(case_name)) if not os.path.exists(file_path): with open(file_path, 'w', encoding='utf-8') as file: file.write(read_header().format(case_name, case_name)) file.write(reader_conntent().format(funtion_name, desc)) else: pass
def run(): try: all_test = AllTest() all_test.runnerPool() except Exception as e: LOG.error("运行程序失败,原因:%s" % e) finally: """关闭Appium服务""" appium_server = AppiumServer(all_test.l_devices) appium_server.stop_server() sys.exit()
def get_conn(): try: connection_dict = url.make_url(options.sql_conn) conn = MySQLdb.connect(host=connection_dict.host, user=connection_dict.username, passwd=connection_dict.password, db=connection_dict.database) except Exception as e: LOG.error("get mysql connection: %s" % e) raise e return conn
def save_result(data): if os.path.exists(recording): with open(recording,'w+',encoding='utf-8') as f: f.write(data+'\n') else: # os.system(r"touch {}".format(recording)) with open(recording,'w+') as f: f.write(data+'\n') LOG.info('记录测试结果完毕')
def browser_search(self, keyword="data"): self.keyword = keyword url = "http://weixin.sogou.com/weixin?type=1&query={keyword}&ie=utf8".format(keyword=keyword) LOG.info("Search Keyword [%s]: %s" % (keyword, url)) # _headers = self.head # req = urllib2.Request(url, None, _headers) # response = urllib2.urlopen(req) # _content = response.read() _content = self.browser.get_url_content(url) return _content
def download_article_content(self, item): article_url = item.get("url", "") refer_url = item.get("site", "") uuid = item.get("uuid", "") if not refer_url: LOG.info("There is not refered URL. Try to download the content through original url") _url = self.domain + article_url article_content = self.browser.get_url_content(_url) with open(config.APACHE_DIR + uuid + ".html", "wb") as f: f.write(article_content) res = "download" else: LOG.info("Article is refered. No need to download the content") res = "refer" return res
def load_account_info(self, account): url_generator_js = None url_generator = None with MongodbUtils(config.WECHAT_DB_IP, config.WECHAT_DB_PORT, config.WECHAT_COLLECTION, config.WECHAT_ACCOUNT_TABLE) as connect_db: _res = connect_db.find_one({"account": account}) if _res: _acct_param = _res.get("url_param", "") wechat_acct = _res.get("account") wechat_name = _res.get("account_name") if _acct_param: # _param_ = "&cb=sogou.weixin_gzhcb&gzhArtKeyWord=%s&tsn=3&t=&_=&page=" _param_ = "&cb=sogou.weixin_gzhcb&tsn=3&page=" LOG.info("Infomation of account [%s] was loaded." % (account)) url_generator_js = lambda x: self.domain + "/gzhjs?" + _acct_param + _param_ + str(x) url_generator = lambda x: self.domain + "/gzh?" + _acct_param + _param_ + str(x) else: LOG.info("Account [%s] does not have login param." % (account)) else: LOG.info("Account [%s] was not found." % (account)) self.wechat_acct = wechat_acct self.wechat_name = wechat_name self.url_generator = url_generator self.url_generator_js = url_generator_js
def sendjson(self, method, urlpath, obj=None): """Send json to the OpenDaylight controller.""" headers = {'Content-Type': 'application/json'} data = jsonutils.dumps(obj, indent=2) if obj else None url = '/'.join([self.url, urlpath]) LOG.debug("Sending METHOD (%(method)s) URL (%(url)s) JSON (%(obj)s)" % {'method': method, 'url': url, 'obj': obj}) r = requests.request(method, url=url, headers=headers, data=data, auth=self.auth, timeout=self.timeout) try: r.raise_for_status() except Exception as ex: LOG.error("Error Sending METHOD (%(method)s) URL (%(url)s)" "JSON (%(obj)s) return: %(r)s ex: %(ex)s rtext: " "%(rtext)s" % {'method': method, 'url': url, 'obj': obj, 'r': r, 'ex': ex, 'rtext': r.text}) import pdb;pdb.set_trace() return r try: return json.loads(r.content) except Exception: LOG.debug("%s" % r) return
def catch_error(*args, **kwargs): try: res = func(*args, **kwargs) except Exception as e: LOG.error(traceback.format_exc()) LOG.info("================content start================") _content = kwargs.get("content") _content = _content if _content else args[0] print _content LOG.info("================content end================") return None return res
def get_articles_on_page(self, page=1): _end_of_process = False url = self.url_generator_js(page) print url _content = self.get_url_content(url) print _content # soup = BeautifulSoup(req.text, 'html') _tmp = _content.replace("sogou.weixin_gzhcb(", "") _tmp = _tmp.replace(")\n\n\n\n\n\n\n\n\n", "") _tmp = _tmp.replace("\\/", "/") _res_dict = eval(_tmp) # print _content total_pages = _res_dict.get("totalPages") page = _res_dict.get("page") _items = _res_dict.get("items") LOG.info("Page: %s/%s" % (str(page), str(total_pages))) # total_pages = 3 articles = [] num_of_articles_downloaded = 0 for _item in _items: xmlp = ET.XMLParser(encoding="utf-8") _item_xml = ET.fromstring(_item, parser=xmlp) ref_url = _item_xml.find("item").find("display").find("url").text _uuid = make_uuid() # print _item_xml.find("item").find("display").text print "--------start test------" print self.get_xml_node(_item_xml, "item/display/title") print self.get_xml_node(_item_xml, "item/display/site") print "--------end test------" _res = dict( uuid=_uuid, title=self.get_xml_node(_item_xml, "item/display/title"), content=_item_xml.find("item").find("display").find("content168").text, site=self.get_xml_node(_item_xml, "item/display/site"), url=self.get_xml_node(_item_xml, "item/display/url"), sourcename=self.get_xml_node(_item_xml, "item/display/sourcename"), release_date=self.get_xml_node(_item_xml, "item/display/date"), wechat_acct=self.wechat_acct, timestamp=date_helper.current_timestamp(), unread=True, tag=[] ) # print _res.get("title") with MongodbUtils(config.WECHAT_DB_IP, config.WECHAT_DB_PORT, config.WECHAT_COLLECTION, config.WECHAT_ARTICLE_SUMMARY_TABLE) as connect_db: _query = dict(wechat_acct=self.wechat_acct, title=_res.get("title")) exist_record = connect_db.find_one(_query) if not exist_record or self.mode == 'override': if self.mode == 'override': _old_uuid = exist_record.get("uuid", "") _res["uuid"] = _old_uuid connect_db.remove(_query) connect_db.insert(_res) # _article_url = self.get_article_url(_res.get("site"), _uuid) # if _article_url: # num_of_articles_downloaded += 1 else: if self.mode in ["init"]: _end_of_process = False LOG.info("Already exist") else: _end_of_process = True LOG.info("Meet the last article in DB") break LOG.info("%s pages are downloaded on this page" % num_of_articles_downloaded) if (page < total_pages) and not _end_of_process: LOG.info("Go to next page") self.get_articles_on_page(page + 1) else: return articles
def run_wechat_crawler(self): acct = self.random_select_account() LOG.info("Select account: %s" % acct) crawler = WechatCrawler( mode=self.mode) crawler.run_crawl_account_articles(acct)
def browser_account_page(self, page=1): url = self.url_generator_js(page) LOG.info("Visit account page: %s " % (url)) _content = self.browser.get_url_content(url) return _content
def simulate_sleep(self, m=5, offset=10): sleep_time = round(np.random.chisquare(m), 2) + offset LOG.info("I'm gona to sleep for %s sec." % (str(sleep_time))) time.sleep(sleep_time)
def get_wechat_articles(self, keyword=""): LOG.info("Start to get data of account [%s]" % (keyword)) login_info = self.login(keyword) LOG.info("Finished getting login param of account [%s]" % (keyword)) self.get_articles(1, login_info) LOG.info("End process for account [%s]" % (keyword))
def parse_account_page(self, content): _end_of_process = False _tmp = content.replace("sogou.weixin_gzhcb(", "") _tmp = _tmp.replace(")\n\n\n\n\n\n\n\n\n", "") _tmp = _tmp.replace("\\/", "/") _res_dict = eval(_tmp) # Get pagination infomation total_pages = _res_dict.get("totalPages") page = _res_dict.get("page") _end_of_process = True if page >= total_pages else False LOG.info("Page: %s/%s" % (str(page), str(total_pages))) # Get articles on page _items = _res_dict.get("items") num_of_articles_downloaded = 0 for _item in _items: xmlp = ET.XMLParser(encoding="utf-8") _item_xml = ET.fromstring(_item, parser=xmlp) _uuid = make_uuid() _res = dict( uuid=_uuid, title=self.browser.get_xml_node(_item_xml, "item/display/title"), content=_item_xml.find("item").find("display").find("content168").text, site=self.browser.get_xml_node(_item_xml, "item/display/site"), url=self.browser.get_xml_node(_item_xml, "item/display/url"), sourcename=self.browser.get_xml_node(_item_xml, "item/display/sourcename"), release_date=self.browser.get_xml_node(_item_xml, "item/display/date"), wechat_acct=self.wechat_acct, timestamp=date_helper.current_timestamp(), unread=True, tag=[] ) LOG.info("Reading article: %s" % (_res.get("title"))) LOG.info("Source site: %s" % (_res.get("site"))) LOG.info("From: %s" % (_res.get("wechat_acct"))) with MongodbUtils(config.WECHAT_DB_IP, config.WECHAT_DB_PORT, config.WECHAT_COLLECTION, config.WECHAT_ARTICLE_SUMMARY_TABLE) as connect_db: _query = dict(wechat_acct=self.wechat_acct, title=_res.get("title")) exist_record = connect_db.find_one(_query) if not exist_record or self.mode == 'init': if self.mode == 'init' and exist_record: _old_uuid = exist_record.get("uuid", "") _res["uuid"] = _old_uuid connect_db.remove(_query) connect_db.insert(_res) if not _res.get("site"): self.download_article_content(_res) # _article_url = self.get_article_url(_res.get("site"), _uuid) # if _article_url: # num_of_articles_downloaded += 1 else: _end_of_process = True break return _end_of_process
def record_time(*args, **kwargs): res = func(*args, **kwargs) sleep_time = round(np.random.chisquare(seed), 2) + offset LOG.info("I'm gona to sleep for %s sec." % (str(sleep_time))) time.sleep(sleep_time) return res