def callback(self, ch, method, properties, body): """Action performed when recreiving messages.""" log.info("Received task with body {}".format(body)) try: # Ignore messages that are redelivered if method.redelivered == True: log.info("Task discarded as copy") ch.basic_ack(delivery_tag=method.delivery_tag) else: log.info("Start processing task") # Execute crawling task log.info("Execute crawl with spec: {}".format(body)) finished = run_crawl(body, worker_flag=True) log.info("Scrapy worker finished: {}".format(finished)) # Unsubscribe before sending ack to avoid accepting a task on closing log.info("Unsubscribing task queue.") self.channel.basic_cancel(self.consumer_tag) # send acc before closing ch.basic_ack(delivery_tag=method.delivery_tag) log.info("Finished processing task") exit() except Exception as e: log.exception(e) # ch.basic_ack(delivery_tag=method.delivery_tag) # log.info("Failed processing task") raise
def wait_element_clickable(self, loc, img_info, timeout=15, poll_frequency=0.5): """ 等待元素可点击 :param loc:定位元素表达式 :param img_info:发生错误时截图文件名 :param timeout:等待超时时间 :param poll_frequency:查询频率 :return:超时错误或者元素 """ # 获取当前时间 start_time = time() try: ele = WebDriverWait(self.driver, timeout, poll_frequency).until(EC.element_to_be_clickable(loc)) except Exception as e: # 输出错误日志 log.error("元素{}等待可点击超时".format(loc)) log.exception(e) # 对当前错误页面截图 self.screen_shot(img_info) raise e else: # 打印等待时间,返回元素 end_time = time() log.info('元素{}可点击,等待时间{}秒'.format(loc, start_time - end_time)) return ele
def main(): try: app = QApplication(sys.argv) if app_is_run(): ex = ErrorUI() else: ex = MainUI() sys.exit(app.exec_()) except Exception as e: log.exception(e)
def explicit_wait_presence(self, locator: tuple): """ 显示等待--直到元素出现 :param locator: 元素定位器 :return: WebElement 对象 """ try: return self.wait.until(ec.presence_of_element_located(locator)) except TimeoutException as e: log.exception(e) self.screen_shot()
def explicit_wait_clickable(self, locator: tuple): """ 显示等待--直到元素可被点击 :param locator: 元素定位器 :return: WebElement 对象 """ try: return self.wait.until(ec.element_to_be_clickable(locator)) except TimeoutException as e: log.exception(e) self.screen_shot()
def close(self): """Close consumer connection to rmq server.""" log.info( "Stop consuming messages on queue {} with routing key {}.".format( self.queue_name, self.routing_key)) # For unknown reasons sometimes StreamLostError: ("Stream connection lost: IndexError('pop from an empty deque') occures while closing try: self.channel.basic_cancel(self.consumer_tag) self.connection.close() except Exception as ex: log.exception("Error while closing consumer: {}".format(ex)) pass
def get_element(self, loc, img_info): """ 封装find_element :param loc: :param img_info: :return: """ try: ele = self.driver.find_element(*loc) except Exception as e: log.error("定位{}元素失败".format(loc)) log.exception(e) self.screen_shot(img_info) else: log.info("定位{}元素成功".format(loc)) return ele
def click_element(self, loc, img_info): """ click封装 :param loc: :param img_info: :return: """ try: self.driver.find_element(*loc).click() except Exception as e: log.error("元素{}点击失败".format(loc)) log.exception(e) self.screen_shot(img_info) raise e else: log.info("元素{}点击成功".format(loc))
def op_xml(file_path_: str = None, update=False): """ 1 读取xml文件 2 获取element_tree 3 写xml :return: element_tree """ try: if not file_path_: file_path_ = "{}{}".format(CONST.PROJECT_PATH, CONST.DB_PATH) element_tree = read_xml(file_path_) yield element_tree if update: write_xml(element_tree, file_path_) except Exception as e: log.exception(e)
def input_text(self, loc, text, img_info): """ 文本输入 :param loc: :param text:文本内容 :param img_info: :return: """ try: self.driver.find_element(*loc).send_keys(text) except Exception as e: log.error("文本{}输入失败".format(text)) log.exception(e) self.screen_shot(img_info) raise e else: log.info("文本{}输入成功".format(text))
def get_element_text(self, loc, img_info): """ 获取元素文本信息 :param loc:定位器 :param img_info:错误截图信息 :return:text """ try: text = self.driver.find_element(*loc).text except Exception as e: log.error("获取{}元素文本失败".format(loc)) log.exception(e) self.screen_shot(img_info) raise e else: log.info("获取{}元素文本成功".format(loc)) return text
def get_element_attr(self, loc, attr_name, img_info): """ 获取元素属性 :param loc:定位器 :param attr_name:属性名称 :param img_info:错误截图信息 :return: """ try: attr_value = self.driver.find_element(*loc).get_attribute(attr_name) except Exception as e: log.error("获取{}元素属性失败".format(loc)) log.exception(e) self.screen_shot(img_info) raise e else: log.info("获取{}元素属性成功".format(loc)) return attr_value
def login(self): try: info = None already_login = False if not already_login: # email = self.get_edit_text(EMAIL_TAG) # password = self.get_edit_text(PW_TAG) # captcha = self.get_edit_text(CAPTCHA_TAG) # TODO 登录 # already_login, err_code, info = login(email, password, captcha) already_login = True if already_login: self.main_ui.hide() self.remove_saas_page_modules() self.add_second_page_modules() self.main_ui.show() else: QMessageBox.information(self.main_ui, '登录失败', info) update_captcha(self) except Exception as err: log.exception(err)
class TestLogin: """登陆测试用例""" @pytest.mark.parametrize('case', LoginData.success_data) def test_login_success(self, case, login_fixture): """登陆成功测试""" login_page, index_page = login_fixture login_page.login(case['username'], case['password']) res = index_page.if_login_success() try: assert "登陆成功" == res except AssertionError as e: log.error("成功登陆用例执行失败") log.exception(e) raise e else: log.info("登陆成功用例执行成功") index_page.logout()
def app_is_run(): try: return False except Exception as err: log.exception(err)
def div(a, b): try: a / b except Exception as e: log.exception(e)
def handle(self, *args, **options): start_time = int(time.time()) for board in boards: read_path = base_path + board files = os.listdir(read_path) # TODO 1.写入项目文件 2.拆分函数 for file in files: try: with open(f"{read_path}/{file}", "r", encoding="GB2312") as f: first_line = f.readline() iuid = first_line.split(" ")[0] tail = first_line.split(" ").index("日线") name = "".join(first_line.split(" ")[1:tail]) tick_data = pd.read_csv(f"{read_path}/{file}", sep="\s+", encoding="gbk", header=1)[:-1] if tick_data.isnull().any().values[0]: print(f"{file}, 存在空数据") size_in_file = tick_data.shape[0] # 根据收盘价算出涨跌幅,并对成交额进行处理/10000 close_prices = list(tick_data["收盘"]) if tick_data.shape[0] != len(close_prices): print(f"{file}, 收盘价计算涨跌幅有问题") volatilies = [0] for i in range(1, len(close_prices)): rate = (close_prices[i] - close_prices[i - 1]) / close_prices[i - 1] rate = rate * 1000 rate = float('%.2f' % rate) volatilies.append(rate) tick_data["涨跌幅"] = volatilies tick_data["成交额"] = tick_data["成交额"].map( self.volume_dividen) # 计算绝对值平均波动率 计算平均交易额 avg_volatilty = sum([abs(i) for i in volatilies ]) / len(volatilies) avg_volatilty = float('%.2f' % avg_volatilty) avg_volume = sum(list(tick_data["成交额"])) / len( list(tick_data["成交额"])) avg_volume = float('%.8f' % avg_volume) avg_volume_rates = [ float('%.2f' % (i / (10000 * avg_volume))) for i in list(tick_data["成交额"]) ] tick_data["与年成交量比率"] = avg_volume_rates # 保存到数据库中 params = { "iuid": iuid, "name": name, "board": board, "base_volatility": avg_volatilty, "base_volume": avg_volume, "size_in_file": size_in_file } _ = Instruments.objects.create(**params) # 保存到文件中 csv_name = file.split(".")[0] + ".csv" csv_name = csv_name[2:] save_path = f"{settings.BASE_DIR}/historically/{board}/{csv_name}" tick_data.to_csv(save_path, sep=",", header=True, index=True) except Exception as ex: print(f"{iuid} failed because {ex}") log.exception(f"{iuid} failed because {ex}") if ex.args == 1062: continue following_path = f"{settings.BASE_DIR}/historically/data_man/lack_iuid" with open(following_path, "a+") as f: f.write(f"{iuid} {ex}") f.write("\n") duration_time = int(time.time()) - start_time print(f"{duration_time}s")
def remove_module_many(self, name_type_list): try: for key, module_type in name_type_list: self.remove_module(key, module_type) except Exception as err: log.exception(err)