def parse_url(url):
"""
Parse RSS XML URL and get the basic details
:param url: RSS XML URL
:param provider: Name of the Article provider
:return: item
"""
try:
items = []
column_list = [
'title', 'summary', 'id', 'language', 'link', 'description',
'published', 'media_content', 'image'
]
elements = feedparser.parse(url)
elements = (element for element in elements.entries)
if not elements:
return []
logger.info("Please wait, parsing through the RSS feeds........")
for element_res in elements:
item = {}
for element in element_res:
if element in column_list:
item[element] = element_res.get(element)
if 'media_content' in element_res:
item['image_url'] = element_res.media_content[0]['url']
else:
item['image_url'] = element_res.get('image')
item = get_metadata_newspaper(item)
# yield item
if item:
items.append(item)
return iter(items)
except Exception as error:
logger.error(error)
def checkFabricServiceStatus(yamlContent):
logger.info("Waiting for BoK to start up ...")
clusterName = yamlContent["clusterName"]
peerOrgs = yamlContent["crypto-config.yaml"]["PeerOrgs"]
ifReady=0
timeCount=0
config.load_kube_config()
v1 = client.CoreV1Api()
while True:
ret = v1.list_pod_for_all_namespaces(watch=False)
for peerOrg in peerOrgs:
peerOrgName=peerOrg["Name"]
namespace=peerOrgName.lower() + "-" + clusterName.lower()
for i in ret.items:
if namespace == i.metadata.namespace:
if i.status.phase != "Running":
ifReady=0
break
else:
ifReady=1
if ifReady == 0:
break
if ifReady == 1:
logger.info("BoK is up and running.")
break
else:
timeCount = timeCount + INTERVAL
time.sleep(INTERVAL)
if timeCount > TIMEOUT:
logger.error("Error: Failed to start BoK service.")
sys.exit(1)
def load_paperdata(distance_f):
'''
Load distance from data
Args:
distance_f : distance file, the format is column1-index 1, column2-index 2, column3-distance
Returns:
distances dict, max distance, min distance, max continues id
'''
log.info("PROGRESS: load data")
distances = {}
min_dis, max_dis = sys.float_info.max, 0.0
max_id = 0
with open(distance_f, 'r') as fp:
for line in fp:
x1, x2, d = line.strip().split(' ')
x1, x2 = int(x1), int(x2)
max_id = max(max_id, x1, x2)
dis = float(d)
min_dis, max_dis = min(min_dis, dis), max(max_dis, dis)
distances[(x1, x2)] = float(d)
distances[(x2, x1)] = float(d)
for i in range(max_id):
distances[(i, i)] = 0.0
logger.info("PROGRESS: load end")
return distances, max_dis, min_dis, max_id
def evaluate(args, model, eval_dataloader, metrics):
# Eval!
logger.info(" Num examples = %d", len(eval_dataloader))
logger.info(" Batch size = %d", args.eval_batch_size)
eval_loss = AverageMeter()
metrics.reset()
preds = []
targets = []
pbar = ProgressBar(n_total=len(eval_dataloader), desc='Evaluating')
for bid, batch in enumerate(eval_dataloader):
model.eval()
batch = tuple(t.to(args.device) for t in batch)
with torch.no_grad():
inputs = {'input_ids': batch[0],
'attention_mask': batch[1],
'labels': batch[3]}
inputs['token_type_ids'] = batch[2]
outputs = model(**inputs)
loss, logits = outputs[:2]
eval_loss.update(loss.item(), n=1)
preds.append(logits.cpu().detach())
targets.append(inputs['labels'].cpu().detach())
pbar(bid)
preds = torch.cat(preds, dim=0).cpu().detach()
targets = torch.cat(targets, dim=0).cpu().detach()
metrics(preds, targets)
eval_log = {"eval_acc": metrics.value(),
'eval_loss': eval_loss.avg}
return eval_log
def uploadspeech(localpath):
"""
upload file to qiniu, if upload failed, it will retry 3 times.
if upload still failed after retry, it return "None"
"""
access_key = '_D2Iavhr-DRKHHhW0BTT7-liQ2jO-1cC_lqKn0eF'
secret_key = 'E3QKF99mgA8HAyGF1nMlKWVVaKlIxRpTZvEb1CiO'
global qiniuAuth
if isinstance(qiniuAuth, Auth) is False:
qiniuAuth = Auth(access_key, secret_key)
bucket_name = 'pipixia'
key = "audio_" + str(int(time.time())) + ".mp3"
uptoken = qiniuAuth.upload_token(bucket_name, key, 700000)
ret, info = put_file(uptoken, key, localpath)
global uploadspeechRetryCount
if ret is None:
info(info)
if uploadspeechRetryCount < 4:
uploadspeechRetryCount += 1
return uploadspeech(localpath)
else:
return None
else:
uploadspeechRetryCount = 0
return "http://oty38yumz.bkt.clouddn.com/" + key
def _download(self, url):
download_dir = global_define.XML_DIR
if not os.path.exists(download_dir):
os.mkdir(download_dir)
day = global_define.TODAY
download_dir = os.path.abspath(os.path.join(download_dir, str(day)))
if not os.path.exists(download_dir):
os.mkdir(download_dir)
#(head,fname)=os.path.split(url)
#file_path=os.path.join(file_dir,fname)
file_name = url.replace(self.__url_prefix,"")
file_name = file_name.replace("/","")
file_path = os.path.join(download_dir, file_name)
wgetcmd = "wget %s --timeout=10 --quiet -O %s" %(url,file_path)
if not subprocess.call(wgetcmd,shell=True):
logger.debug('wget url[%s] finished.' %url)
else:
logger.critical('cmd[%s] failed.' %wgetcmd)
return None
logger.info('download [%s] successfully' %(url))
return os.path.abspath(file_path)
def setup(self, fname):
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
logger.addHandler(self.file_handler)
logger.addHandler(self.stream_handler)
logger.info('Begin logging to {}'.format(fname))
return logger
def test_persons_get(self):
for setting in [
'minimal', 'normal', 'detailed', 'extensive', 'default'
]:
for mode in [False, True]:
query = urllib.parse.urlencode({
'select': setting,
'dict': str(mode).lower()
})
check = self.app.get('/persons?{}'.format(query),
follow_redirects=True)
logger.info(check.data)
if setting == 'extensive' and mode:
comparison = check.data
selected = random.choice(json.loads(comparison.decode())['content'])
for setting in ['full', 'default']:
query = urllib.parse.urlencode({'select': setting})
check = self.app.get('/persons/{}?{}'.format(
selected['uuid'], query),
follow_redirects=True)
self.assertEqual(
json.loads(check.data.decode())['content'], selected)
print('\nfinished persons')
def send_email(setting: dict, file_path):
"""
入参一个字典
:param user: 发件人邮箱
:param password: 邮箱授权码
:param host: 发件人使用的邮箱服务 例如：smtp.163.com
:param contents: 内容
:param addressees: 收件人列表
:param title: 邮件标题
:param enclosures: 附件列表
:param file_path: 需要压缩的文件夹
:return:
"""
EmailServe.zip_report(
file_path=file_path,
out_path=setting['enclosures'])
yag = yagmail.SMTP(
setting['user'],
setting['password'],
setting['host'])
# 发送邮件
yag.send(
setting['addressees'],
setting['title'],
setting['contents'],
setting['enclosures'])
# 关闭服务
yag.close()
logger.info("邮件发送成功！")
def simulate(self):
target = Plant.get(Plant.localhost == True)
source = Plant.get(Plant.name == 'marta')
for sensor in Sensor.select():
logger.info(sensor.name)
PlantSimulate().run(target, sensor, source)
logger.info('finished')
def run():
if os.path.exists('report/'):
shutil.rmtree(path='report/')
logger.add(file_path['log'], enqueue=True, encoding='utf-8')
logger.info("""
_ _ _ _____ _
__ _ _ __ (_) / \ _ _| |_ __|_ _|__ ___| |_
/ _` | '_ \| | / _ \| | | | __/ _ \| |/ _ \/ __| __|
| (_| | |_) | |/ ___ \ |_| | || (_) | | __/\__ \ |_
\__,_| .__/|_/_/ \_\__,_|\__\___/|_|\___||___/\__|
|_|
Starting ... ... ...
""")
pytest.main(
args=['test/test_api.py', f'--alluredir={file_path["report"]}/data'])
# 自动以服务形式打开报告
# os.system(f'allure serve {report}/data')
# 本地生成报告
os.system(
f'allure generate {file_path["report"]}/data -o {file_path["report"]}/html --clean'
)
logger.success('报告已生成')
# 发送邮件带附件报告
EmailServe.send_email(email, file_path['report'])
# 删除本地附件
os.remove(email['enclosures'])
def delete_all_data_in_tables():
Product.objects.all().delete()
Category.objects.all().delete()
logger.info("Toutes les données ont étaient éffacés")
# def bulk_insert_product_category():
# """
# insert all product and ur categories
# """
# # Creation de deux objet (product & category) pour bulk insert
#
# p1 = Product.objects.create(name="Pizza fromage",
# image_product="https://image.fr",
# stores="OpenClassrooms",
# url=None,
# nutriscore="D",
# image_reperes_nutrionnels="https://image_repere.fr")
#
# p1.save()
# c1 = Category.objects.create(name='Pizza')
# c1.save()
# p1.category.add(c1)
# c2 = Category.objects.create(name='Fromage')
# c2.save()
# p1.category.add(c2)
#
# get = Product.objects.filter(pk=221)
# print(get)
"""
def bert_epoch_step(self, state, current):
'''
适合bert类型模型，适合pytorch_transformer模块
:param state:
:param current:
:return:
'''
model_to_save = state['model']
if self.save_best_only:
if self.monitor_op(current, self.best):
logger.info(
f"\nEpoch {state['epoch']}: {self.monitor} improved from {self.best:.5f} to {current:.5f}"
)
self.best = current
state['best'] = self.best
model_to_save.save_pretrained(str(self.base_path))
output_config_file = self.base_path / 'configs.json'
with open(str(output_config_file), 'w') as f:
f.write(model_to_save.config.to_json_string())
state.pop("model")
torch.save(state, self.base_path / 'checkpoint_info.bin')
else:
if state['epoch'] % self.epoch_freq == 0:
save_path = self.base_path / f"checkpoint-epoch-{state['epoch']}"
save_path.mkdir(exist_ok=True)
logger.info(f"\nEpoch {state['epoch']}: save model to disk.")
model_to_save.save_pretrained(save_path)
output_config_file = save_path / 'configs.json'
with open(str(output_config_file), 'w') as f:
f.write(model_to_save.config.to_json_string())
state.pop("model")
torch.save(state, save_path / 'checkpoint_info.bin')
def run(cls):
if os.path.exists('../report'):
shutil.rmtree(path='../report')
logger.add(logfile, enqueue=True, encoding='utf-8')
logger.info('开始测试...')
pytest.main(args=[f'--alluredir={report}/data'])
os.system(f'allure generate {report}/data -o {report}/html --clean')
logger.success('报告已生成')
def save_response(cls, key: str, value: object) -> None:
"""
保存实际响应
:param key: 保存字典中的key，一般使用用例编号
:param value: 保存字典中的value，使用json响应
"""
cls.response_dict[key] = value
logger.info(f'添加key: {key}, 对应value: {value}')
def stopFabricExplorer(yamlContent):
clusterName = yamlContent["clusterName"]
mountPoint = yamlContent["nfsServer"]["mountPoint"]
explorerYaml = mountPoint + "/" + clusterName + "/resources/explorer-artifacts/fabric_1_0_explorer.yaml"
command = "kubectl delete --grace-period=0 --force -f " + explorerYaml
re=os.system(command)
if re != 0:
logger.info(command + " exec failed")
def notifyDFgeneration(deploymentYaml, template):
# Notification of deployment file generation
deploymentYaml = deploymentYaml.rsplit("deployment/")[-1]
# Only show the path under deployment/
template = template.rsplit("templates/")[-1]
# Only show the path under template/
raw = "Generating {} from template: {}".format(deploymentYaml, template)
# should replace by logger in the furture
logger.info(raw)
def execute_cmd(self, cmd: str):
"""
:param cmd: 服务器下对应的命令, 可以是list，或者str
"""
stdin, stdout, stderr = self.ssh.exec_command(cmd)
error = stderr.read().decode()
logger.info(f"输入命令: {cmd} -> 输出结果: {stdout.read().decode()}")
logger.error(f"异常信息: {error}")
return error
def showAddress(yamlContent):
urls = getUrls(yamlContent)
dashboard ="You can view Kubernetes dashboard at: " + urls["k8sDashboardUrl"]
logger.info(dashboard)
if urls["fabricExplorerUrl"]:
explorerMsg = "You can view Fabric Explorer at: " + urls["fabricExplorerUrl"]
else:
explorerMsg = "Can not get Ingress Controller IP. Please check your Ingress controller settings."
logger.info(explorerMsg)
def init_db():
'''
Crete db with table 'flt'
Or do nothing if they exist
'''
try:
Base.metadata.create_all(bind=engine)
logger.info('Init DB - OK')
except OperationalError:
logger.info('Init DB - Error')
raise MyLocalException
def run():
if os.path.exists('report/'):
shutil.rmtree(path='report/')
logger.add(logfile, enqueue=True, encoding='utf-8')
logger.info('开始测试...')
pytest.main(args=['test/test_api.py', f'--alluredir={report}/data'])
# 自动以服务形式打开报告
# os.system(f'allure serve {report}/data')
# 本地生成报告
os.system(f'allure generate {report}/data -o {report}/html --clean')
logger.success('报告已生成')
def set(self):
result = VariousTools.offline_check('display', hardware=False)
if result is True:
execute = False
sensor = SensorHardware.get(label='display')
if sensor.last_execution is not None:
offset = datetime.datetime.now() - sensor.last_execution
if offset.seconds >= 30 * 60:
execute = True
else:
execute = True
if execute is True:
sensor.last_execution = datetime.datetime.now()
sensor.save()
logger.debug('display: updated')
bus = 1
gpio_count = 16
address = 0x20
self.get()
self.calculate()
# Create MCP230xx GPIO adapter.
mcp = MCP230XX_GPIO(bus, address, gpio_count)
print('creating text')
# Create LCD, passing in MCP GPIO adapter.
lcd = Adafruit_CharLCD(pin_rs=8,
pin_e=9,
pins_db=[10, 11, 12, 13],
GPIO=mcp)
lcd.clear()
lcd.message(self.data['display']['text'])
else:
logger.debug('display: not updated')
# print self.data['display']['text']
else:
logger.info('display: in offline timeframe - clearing')
bus = 1
gpio_count = 16
address = 32
mcp = MCP230XX_GPIO(bus, address, gpio_count)
lcd = Adafruit_CharLCD(pin_rs=8,
pin_e=9,
pins_db=[10, 11, 12, 13],
GPIO=mcp)
lcd.clear()
def run(self, cmd, sudo=True):
if sudo:
cmd = 'sudo %s' % cmd
logger.info('\nRunning %s' % cmd)
_, stdout, stderr = self.connection.exec_command(cmd, get_pty=True)
out = {
'stdout': stdout.read().decode('utf-8'),
'stderr': stderr.read().decode('utf-8'),
'es': stdout.channel.recv_exit_status()
}
if out['stderr']:
logger.info(out['stderr'])
return out
def files_action(self, post: bool, local_path: str = os.getcwd(), remote_path: str = "/root"):
"""
:param post: 动作 为 True 就是上传， False就是下载
:param local_path: 本地的文件路径， 默认当前脚本所在的工作目录
:param remote_path: 服务器上的文件路径，默认在/root目录下
"""
if post: # 上传文件
self.ftp_client.put(localpath=local_path, remotepath=f"{remote_path}{os.path.split(local_path)[1]}")
logger.info(f"文件上传成功: {local_path} -> {self.host}:{remote_path}{os.path.split(local_path)[1]}")
else: # 下载文件
file_path = local_path + os.path.split(remote_path)[1]
self.ftp_client.get(remotepath=remote_path, localpath=file_path)
logger.info(f"文件下载成功: {self.host}:{remote_path} -> {file_path}")
def main(conf_file_name):
pid = os.getpid()
logger = tools.logger.Logger('mainlog', str(pid) + 'main.log')
work_path = os.path.abspath(os.curdir)
config_path = os.path.join(work_path, conf_file_name)
conf = configparser.ConfigParser()
conf.read(config_path)
redis_ip = conf.get('conf', 'redis_ip')
redis_port = conf.get('conf', 'redis_port')
platform = conf.get('conf', 'platform')
key = conf.get('conf', 'key')
is_send_message = conf.get('conf', 'is_send_message')
currency_list = json.loads(conf.get('conf', 'currency_list'))
pool = redis.ConnectionPool(
host=redis_ip, port=redis_port,
decode_responses=True) # host是redis主机，需要redis服务端和客户端都起着 redis默认端口是6379
r = redis.Redis(connection_pool=pool)
#启动货币处理进程池
p = Pool(len(currency_list))
path = os.path.abspath(os.curdir) + '/data' + '/paths_result.dat'
logger.info(path)
f = open(path, 'r')
path_list = []
for line in f.readlines():
line = line.strip()
a = json.loads(line)
path_list.append(a)
for x in currency_list:
currency_path = (x, path_list, platform, redis_ip, redis_port)
p.apply_async(calc_core.calc_fork, args=(currency_path, ))
logger.info('Waiting for all subprocesses done...')
p.close()
logger.info('All subprocesses done.')
# #recalc process pool
# p_recalc = Pool(4)
# for i in range(4):
# p.apply_async(calc_core.recalc, args=((redis_ip,redis_port),))
while 1:
try:
main_process(r, key, platform, currency_list, is_send_message,
logger, redis_ip, redis_port)
except Exception as e:
logger.info(e)
msg = traceback.format_exc()
logger.info(msg)
time.sleep(15)
def connect(self):
client = SSHClient()
client.set_missing_host_key_policy(AutoAddPolicy)
logger.info('\nConnecting to %s' % self.host)
for _ in range(self.tries):
try:
client.connect(hostname=self.host,
username='******',
key_filename=os.path.expanduser(
conf.get('environment', 'ssh_key')))
return client
except Exception as e:
logger.info('%s occurred') % e
time.sleep(10)
def send_api(
cls,
url,
method,
parametric_key,
header=None,
data=None,
file=None) -> dict:
"""
:param method: 请求方法
:param url: 请求url
:param parametric_key: 入参关键字， params(查询参数类型，明文传输，一般在url?参数名=参数值), data(一般用于form表单类型参数)
json(一般用于json类型请求参数)
:param data: 参数数据，默认等于None
:param file: 文件对象
:param header: 请求头
:return: 返回res对象
"""
session = cls.get_session()
if parametric_key == 'params':
res = session.request(
method=method,
url=url,
params=data,
headers=header)
elif parametric_key == 'data':
res = session.request(
method=method,
url=url,
data=data,
files=file,
headers=header)
elif parametric_key == 'json':
res = session.request(
method=method,
url=url,
json=data,
files=file,
headers=header)
else:
raise ValueError(
'可选关键字为params, json, data')
response = res.json()
logger.info(
f'\n最终请求地址:{res.url}\n请求方法:{method}\n请求头:{header}\n请求参数:{data}\n上传文件:{file}\n响应数据:{response}')
allure_step_no(f'响应耗时(s): {res.elapsed.total_seconds()}')
allure_step('响应结果', response)
return response
def assert_result(cls, response: dict, expect_str: str):
""" 预期结果实际结果断言方法
:param response: 实际响应字典
:param expect_str: 预期响应内容，从excel中读取
return None
"""
expect_dict = convert_json(expect_str)
index = 0
for k, v in expect_dict.items():
actual = extractor(response, k)
index += 1
logger.info(
f'第{index}个断言,实际结果:{actual} | 预期结果:{v} \n断言结果 {actual == v}')
allure_step(f'第{index}个断言', f'实际结果:{actual} = 预期结果:{v}')
assert actual == v
def make_json(self):
'''
Expects abspath to .csv file
Returns json data
'''
flt = self.parse_filename()
passengers = self.csv_to_dict()
flt['prl'] = passengers
try:
json_data = json.dumps(flt)
logger.info(f'{self.file_name}; Convert to json - OK')
return json_data
except (AttributeError, TypeError):
logger.error(f'{self.file_name}; Convert to json - Error')
raise MyLocalException
def parseChannel(self, response):
info("-----------------kejiliechannels url:" + response.url)
title = response.xpath(
"//div[@class='am_news_list_all']//div[@class='am-titlebar am-titlebar-default mt-0']/h1/text()"
).extract_first()
logo = response.xpath(
"//div[@class='am_news_list_all']//div[@class='mt-10']/div[@class='am-fl']/img/@src"
).extract_first()
self.redis_db.sadd(
"kejiliechannels",
json.dumps({
'url': response.url,
'title': title,
'logo': logo
}))
def handle_sql(cls, sql: str, db: object):
"""处理sql，并将结果写到响应字典中"""
if sql not in ['no', '']:
sql = rep_expr(sql, DataProcess.response_dict)
else:
sql = None
allure_step('运行sql', sql)
logger.info(sql)
if sql is not None:
# 查后置sql
result = db.fetch_one(sql)
allure_step('sql执行结果', {"sql_result": result})
logger.info(f'结果：{result}')
if result is not None:
# 将查询结果添加到响应字典里面，作用在，接口响应的内容某个字段 直接和数据库某个字段比对，在预期结果中
# 使用同样的语法提取即可
DataProcess.response_dict.update(result)
def ret_func():
logger.info("func [%s.%s] begin ..." %(func.__module__, func.__name__))
begin_t = datetime.datetime.now()
func()
end_t = datetime.datetime.now()
logger.info("func [%s.%s] end ... " %(func.__module__, func.__name__))
logger.info("consume time %s" %(end_t - begin_t))
