def callback(self, ch, method, properties, body):
"""Action performed when recreiving messages."""
log.info("Received task with body {}".format(body))
try:
# Ignore messages that are redelivered
if method.redelivered == True:
log.info("Task discarded as copy")
ch.basic_ack(delivery_tag=method.delivery_tag)
else:
log.info("Start processing task")
# Execute crawling task
log.info("Execute crawl with spec: {}".format(body))
finished = run_crawl(body, worker_flag=True)
log.info("Scrapy worker finished: {}".format(finished))
# Unsubscribe before sending ack to avoid accepting a task on closing
log.info("Unsubscribing task queue.")
self.channel.basic_cancel(self.consumer_tag)
# send acc before closing
ch.basic_ack(delivery_tag=method.delivery_tag)
log.info("Finished processing task")
exit()
except Exception as e:
log.exception(e)
# ch.basic_ack(delivery_tag=method.delivery_tag)
# log.info("Failed processing task")
raise
def wait_element_clickable(self, loc, img_info, timeout=15, poll_frequency=0.5):
"""
等待元素可点击
:param loc:定位元素表达式
:param img_info:发生错误时截图文件名
:param timeout:等待超时时间
:param poll_frequency:查询频率
:return:超时错误或者元素
"""
# 获取当前时间
start_time = time()
try:
ele = WebDriverWait(self.driver, timeout, poll_frequency).until(EC.element_to_be_clickable(loc))
except Exception as e:
# 输出错误日志
log.error("元素{}等待可点击超时".format(loc))
log.exception(e)
# 对当前错误页面截图
self.screen_shot(img_info)
raise e
else:
# 打印等待时间，返回元素
end_time = time()
log.info('元素{}可点击，等待时间{}秒'.format(loc, start_time - end_time))
return ele
def main():
try:
app = QApplication(sys.argv)
if app_is_run():
ex = ErrorUI()
else:
ex = MainUI()
sys.exit(app.exec_())
except Exception as e:
log.exception(e)
def explicit_wait_presence(self, locator: tuple):
"""
显示等待--直到元素出现
:param locator: 元素定位器
:return: WebElement 对象
"""
try:
return self.wait.until(ec.presence_of_element_located(locator))
except TimeoutException as e:
log.exception(e)
self.screen_shot()
def explicit_wait_clickable(self, locator: tuple):
"""
显示等待--直到元素可被点击
:param locator: 元素定位器
:return: WebElement 对象
"""
try:
return self.wait.until(ec.element_to_be_clickable(locator))
except TimeoutException as e:
log.exception(e)
self.screen_shot()
def close(self):
"""Close consumer connection to rmq server."""
log.info(
"Stop consuming messages on queue {} with routing key {}.".format(
self.queue_name, self.routing_key))
# For unknown reasons sometimes StreamLostError: ("Stream connection lost: IndexError('pop from an empty deque') occures while closing
try:
self.channel.basic_cancel(self.consumer_tag)
self.connection.close()
except Exception as ex:
log.exception("Error while closing consumer: {}".format(ex))
pass
def get_element(self, loc, img_info):
"""
封装find_element
:param loc:
:param img_info:
:return:
"""
try:
ele = self.driver.find_element(*loc)
except Exception as e:
log.error("定位{}元素失败".format(loc))
log.exception(e)
self.screen_shot(img_info)
else:
log.info("定位{}元素成功".format(loc))
return ele
def click_element(self, loc, img_info):
"""
click封装
:param loc:
:param img_info:
:return:
"""
try:
self.driver.find_element(*loc).click()
except Exception as e:
log.error("元素{}点击失败".format(loc))
log.exception(e)
self.screen_shot(img_info)
raise e
else:
log.info("元素{}点击成功".format(loc))
def op_xml(file_path_: str = None, update=False):
"""
1 读取xml文件
2 获取element_tree
3 写xml
:return: element_tree
"""
try:
if not file_path_:
file_path_ = "{}{}".format(CONST.PROJECT_PATH, CONST.DB_PATH)
element_tree = read_xml(file_path_)
yield element_tree
if update:
write_xml(element_tree, file_path_)
except Exception as e:
log.exception(e)
def input_text(self, loc, text, img_info):
"""
文本输入
:param loc:
:param text:文本内容
:param img_info:
:return:
"""
try:
self.driver.find_element(*loc).send_keys(text)
except Exception as e:
log.error("文本{}输入失败".format(text))
log.exception(e)
self.screen_shot(img_info)
raise e
else:
log.info("文本{}输入成功".format(text))
def get_element_text(self, loc, img_info):
"""
获取元素文本信息
:param loc:定位器
:param img_info:错误截图信息
:return:text
"""
try:
text = self.driver.find_element(*loc).text
except Exception as e:
log.error("获取{}元素文本失败".format(loc))
log.exception(e)
self.screen_shot(img_info)
raise e
else:
log.info("获取{}元素文本成功".format(loc))
return text
def get_element_attr(self, loc, attr_name, img_info):
"""
获取元素属性
:param loc:定位器
:param attr_name:属性名称
:param img_info:错误截图信息
:return:
"""
try:
attr_value = self.driver.find_element(*loc).get_attribute(attr_name)
except Exception as e:
log.error("获取{}元素属性失败".format(loc))
log.exception(e)
self.screen_shot(img_info)
raise e
else:
log.info("获取{}元素属性成功".format(loc))
return attr_value
def login(self):
try:
info = None
already_login = False
if not already_login:
# email = self.get_edit_text(EMAIL_TAG)
# password = self.get_edit_text(PW_TAG)
# captcha = self.get_edit_text(CAPTCHA_TAG)
# TODO 登录
# already_login, err_code, info = login(email, password, captcha)
already_login = True
if already_login:
self.main_ui.hide()
self.remove_saas_page_modules()
self.add_second_page_modules()
self.main_ui.show()
else:
QMessageBox.information(self.main_ui, '登录失败', info)
update_captcha(self)
except Exception as err:
log.exception(err)
class TestLogin:
"""登陆测试用例"""
@pytest.mark.parametrize('case', LoginData.success_data)
def test_login_success(self, case, login_fixture):
"""登陆成功测试"""
login_page, index_page = login_fixture
login_page.login(case['username'], case['password'])
res = index_page.if_login_success()
try:
assert "登陆成功" == res
except AssertionError as e:
log.error("成功登陆用例执行失败")
log.exception(e)
raise e
else:
log.info("登陆成功用例执行成功")
index_page.logout()
def app_is_run():
try:
return False
except Exception as err:
log.exception(err)
def div(a, b):
try:
a / b
except Exception as e:
log.exception(e)
def handle(self, *args, **options):
start_time = int(time.time())
for board in boards:
read_path = base_path + board
files = os.listdir(read_path)
# TODO 1.写入项目文件 2.拆分函数
for file in files:
try:
with open(f"{read_path}/{file}", "r",
encoding="GB2312") as f:
first_line = f.readline()
iuid = first_line.split(" ")[0]
tail = first_line.split(" ").index("日线")
name = "".join(first_line.split(" ")[1:tail])
tick_data = pd.read_csv(f"{read_path}/{file}",
sep="\s+",
encoding="gbk",
header=1)[:-1]
if tick_data.isnull().any().values[0]:
print(f"{file}, 存在空数据")
size_in_file = tick_data.shape[0]
# 根据收盘价算出涨跌幅,并对成交额进行处理/10000
close_prices = list(tick_data["收盘"])
if tick_data.shape[0] != len(close_prices):
print(f"{file}, 收盘价计算涨跌幅有问题")
volatilies = [0]
for i in range(1, len(close_prices)):
rate = (close_prices[i] -
close_prices[i - 1]) / close_prices[i - 1]
rate = rate * 1000
rate = float('%.2f' % rate)
volatilies.append(rate)
tick_data["涨跌幅"] = volatilies
tick_data["成交额"] = tick_data["成交额"].map(
self.volume_dividen)
# 计算绝对值平均波动率 计算平均交易额
avg_volatilty = sum([abs(i) for i in volatilies
]) / len(volatilies)
avg_volatilty = float('%.2f' % avg_volatilty)
avg_volume = sum(list(tick_data["成交额"])) / len(
list(tick_data["成交额"]))
avg_volume = float('%.8f' % avg_volume)
avg_volume_rates = [
float('%.2f' % (i / (10000 * avg_volume)))
for i in list(tick_data["成交额"])
]
tick_data["与年成交量比率"] = avg_volume_rates
# 保存到数据库中
params = {
"iuid": iuid,
"name": name,
"board": board,
"base_volatility": avg_volatilty,
"base_volume": avg_volume,
"size_in_file": size_in_file
}
_ = Instruments.objects.create(**params)
# 保存到文件中
csv_name = file.split(".")[0] + ".csv"
csv_name = csv_name[2:]
save_path = f"{settings.BASE_DIR}/historically/{board}/{csv_name}"
tick_data.to_csv(save_path,
sep=",",
header=True,
index=True)
except Exception as ex:
print(f"{iuid} failed because {ex}")
log.exception(f"{iuid} failed because {ex}")
if ex.args == 1062:
continue
following_path = f"{settings.BASE_DIR}/historically/data_man/lack_iuid"
with open(following_path, "a+") as f:
f.write(f"{iuid} {ex}")
f.write("\n")
duration_time = int(time.time()) - start_time
print(f"{duration_time}s")
def remove_module_many(self, name_type_list):
try:
for key, module_type in name_type_list:
self.remove_module(key, module_type)
except Exception as err:
log.exception(err)
