def __init__(self, params):
self._debug = params.debug
self._subsite = params.subsite
self._logger = Logger("spuppy", "logs", params.debug)
self._path = params.directory
self._files = params.files
self._out = params.out
def upload_confirm():
"""
***确认上传***
*** 路径 ***
:
path: "/api/v1.0.0/upload/upload_confirm"
**参数 request.json**
: {
upload_type: requirement
project_id: 100 # 项目id
requirement_id: 8 # 需求id
file_name: [name, name, name, ..., name]
}
**返回值 **
:
{
"code": 200,
"msg": "处理中"
}
"""
from config import Logger
try:
res = UploadController.confirm_upload(request.json)
except Exception as e:
Logger.error(e)
return res
def __init__(self, connection_params: Config, logger: Logger):
self._connection_params = Config(
) if connection_params is None else connection_params
self._logger = Logger() if logger is None else logger
self._domain = 'esceer.com'
self._dns_type = 'A'
self._dns_record_name = '@'
def upload_page():
"""
***上传文件, 文件分页***
***可以根据upload_type 进行逻辑判断.***
*** 路径 /api/v1.0.0/upload/uploads/page***
:
path: ""
**参数 request.json**
:
upload_type: project # 上传类型
project_id: 100 # 项目id
requirement_id: 8 # 需求id
page: 1
per_page: 20
**返回值 **
:
{
"code": 200,
"msg": "处理中"
}
"""
from config import Logger
try:
res = UploadController.upload_page(request.json)
except Exception as e:
Logger.error(e)
return res
def model_file_check():
if os.path.isfile(Config.IMAGE_MODEL_FILE):
return
Logger.info('正在下载模型文件...')
with requests.get(Config.OCR.get('image_model_url'), stream=True) as r:
with open(Config.IMAGE_MODEL_FILE, 'wb') as f:
shutil.copyfileobj(r.raw, f)
Logger.info('下载成功\n')
def model_file_check():
try:
if os.path.isfile(Config.IMAGE_MODEL_FILE):
model_hash_verify()
return
Logger.info('正在下载模型文件...')
with requests.get(Config.OCR.get('image_model_url'), stream=True) as r:
with open(Config.IMAGE_MODEL_FILE, 'wb') as f:
shutil.copyfileobj(r.raw, f)
Logger.info('下载成功\n')
model_hash_verify()
except InvalidModelException:
return model_file_check()
def get_coordinate(self, img_str):
# 储存最终坐标结果
result = ''
try:
# 读取并预处理验证码
img = cv2.imdecode(np.fromstring(img_str, np.uint8),
cv2.IMREAD_COLOR)
text = self.get_text(img)
images = np.array(list(self._get_imgs(img)))
images = self.preprocess_input(images)
label = self.model.predict(text)
label = label.argmax()
text = self.texts[label]
# list放文字
titles = [text]
position = []
# 获取下一个词
# 根据第一个词的长度来定位第二个词的位置
if len(text) == 1:
offset = 27
elif len(text) == 2:
offset = 47
else:
offset = 60
text2 = self.get_text(img, offset=offset)
if text2.mean() < 0.95:
label = self.model.predict(text2)
label = label.argmax()
text2 = self.texts[label]
titles.append(text2)
labels = self.code_model.predict(images)
labels = labels.argmax(axis=1)
for pos, label in enumerate(labels):
if self.texts[label] in titles:
position.append(pos + 1)
# 没有识别到结果
if len(position) == 0:
return result
result = position
Logger.info('识别结果: %s' % result)
except:
pass
return result
def upload_func():
"""
***文件上传***
***可以根据upload_type 进行逻辑判断.***
*** 路径 /api/v1.0.0/upload/uploads***
:
path: ""
**参数 body.from-data**
:
upload_type: project # 上传类型
project_id: 100 # 项目id
requirement_id: 8 # 需求id
files: binary_file # 上传文件
file_type: png
**返回值 **
:
{
"code": 200,
"msg": "处理中"
}
"""
try:
req_dict = request.form.to_dict()
if req_dict.get("upload_type") in ("project", "requirement"):
res = UploadController.start_upload(request.files, request.form)
else:
headers = {'Referer': 'http://127.0.0.1:8008/15'}
api_url = request.base_url.split(VERSION)[-1]
base_url = BASE_API_URL + VERSION
url = base_url + api_url
files = {}
for req_file in request.files.getlist("files"):
save_path = os.path.join('upload', req_file.filename)
req_file.save(save_path)
files = {'files': open(save_path, 'rb')}
resp = requests.post(url=url,
cookies=request.cookies,
data=request.form,
files=files,
headers=headers)
resp_data = json.loads(resp.content)
return jsonify(resp_data)
except Exception as e:
Logger.error(e)
return res
def model_hash_verify():
"""
验证模型文件是否完整
:return:
"""
mode_md5 = Config.OCR.get('image_model_md5')
if not mode_md5:
# 跳过检测
return
md5 = file_md5(Config.IMAGE_MODEL_FILE)
if md5 != mode_md5:
os.remove(Config.IMAGE_MODEL_FILE)
Logger.error('模型文件校验失败，正在重新下载')
raise InvalidModelException()
return True
def __init__(self, base_path, sources):
self.sources = list(
map(
lambda source: sources[source]
(base_path / source,
Logger(logging.getLogger(f"test_assistant_crawler.{source}"),
ALERT_FUNCTION, ALERT_SETTINGS)), sources))
class IpUtils:
def __init__(self, config: Config, logger: Logger):
self._config = Config() if config is None else config
self._logger = Logger() if logger is None else logger
def get_external_ip(self) -> str:
external_ip = requests.get(self._config.get_ip_provider_url()).text
self._logger.info('External ip address: %s' % external_ip)
return external_ip
@staticmethod
def gather_ip_from_dns_response(dns_response_content: str) -> str:
match = re.search(r'\"data\":\"(\d{1,4}\.\d{1,4}\.\d{1,4}\.\d{1,4})\"',
dns_response_content)
if match:
return match.group(1)
else:
raise ValueError('Ip cannot be gathered from dns response: %s' %
dns_response_content)
def _add_task_thread(url, headers, cookies, project_id, obj_info):
with app.app_context():
pos_obj_info = {}
for _index, _info in enumerate(obj_info):
info = eval(_info)
info["object_url"] = ''
info["object_text"] = ''
pos_obj_info[_index] = str(info)
data = {"project_id": project_id, "objects_info": pos_obj_info}
# print(data)
import time
# from datetime import datetime
# print("请求开始时间: %s" % datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
# begin_time =time.time()
res = requests.post(url=url,
headers=headers,
cookies=cookies,
json=data)
# pivot_time = time.time()
# print("请求结束时间: %s" % datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
# print("请求远端链接, 花费时间: %s" % (pivot_time-begin_time))
content = eval(res.text)
code = content.get('code')
if code == 200:
data = content.get('data')
for task_id, obj_url in zip(data, obj_info):
obj_url = eval(obj_url)
Task.create(
**{
"task_id": task_id,
"object_url": obj_url.get("object_url", "")
})
# print("写入数据库话费时间: %s" % (time.time()-pivot_time))
return True
else:
Logger.error("存在上传失败的数据段.")
return False
class GoDaddyConnector:
def __init__(self, connection_params: Config, logger: Logger):
self._connection_params = Config(
) if connection_params is None else connection_params
self._logger = Logger() if logger is None else logger
self._domain = 'esceer.com'
self._dns_type = 'A'
self._dns_record_name = '@'
def fetch_ip_from_dns(self) -> str:
self._logger.debug('Fetching current ip set in dns...')
response = requests.get(self._get_url(), headers=self._get_headers())
self._logger.debug(response.content)
return IpUtils.gather_ip_from_dns_response(
response.content.decode('utf-8'))
def update_dns(self, target_ip: str) -> str:
self._logger.debug('Updating dns information...')
response = requests.put(self._get_url(),
data=self._build_new_dns_info(target_ip),
headers=self._get_headers())
self._logger.debug(response.content)
return response.content
def _get_url(self) -> str:
return '%s/v1/domains/%s/records/%s/%s' % (
self._connection_params.get_godaddy_url_base(), self._domain,
self._dns_type, self._dns_record_name)
def _get_headers(self):
return {
'Authorization':
'sso-key %s:%s' % (self._connection_params.get_api_key(),
self._connection_params.get_api_secret()),
'accept':
'application/json',
'Content-Type':
'application/json'
}
def _build_new_dns_info(self, target_ip: str):
return '[{ "data": "%s", "ttl": 3600 }]' % target_ip
def uploaded_check(uploaded_files, req_form):
"""
上传文件
成功数, 失败数, 错误数.
"""
req_dict = req_form.to_dict()
up_type = req_dict.get("upload_type")
project_id = req_dict.get("project_id")
requirement_id = req_dict.get("requirement_id")
# 判断需求状态是否允许上传
# task = PlatformTask.get_instance(requirement_id)
# status_list = LocalStatus.query.filter_by(category="task").all()
# accept_status = [item.id for item in status_list if item.en_name in ("active", "nonactivated")]
#
# if requirement_id:
# if not task:
# return jsonify(code=PARAMS_NOT_PROVIDED, msg="无效的需求id")
# if task.status not in accept_status:
# return jsonify(code=DATA_NOT_FOUND, msg="需求状态, 不允许上传文件")
# if task.creator_id != current_user.id:
# return jsonify(code=OPERATOR_ERROR, msg="操作非法")
# local_status = LocalStatus.query.filter_by(category='task', name='delete').first()
# if task.status == local_status:
# return jsonify(code=OPERATOR_ERROR, msg="需求已删除")
#
# if project_id:
# project = Project.get_instance(project_id)
# if not project:
# return jsonify(code=PARAMS_NOT_PROVIDED, msg="无效的参数")
# if current_user.platform != project.platform:
# return jsonify(code=OPERATOR_ERROR, msg="非法操作")
req_dict["check_type"] = "uploaded_check"
# res = requests.post("check", json=req_dict)
# 文件缓存隔离级别为: 单个项目/需求, 每一次上传.
# 合法检查
from flask import request
from config.config import VERSION, BASE_API_URL
headers = get_headers()
# api_url = request.base_url.split(VERSION)[-1]
base_url = BASE_API_URL + VERSION
url = os.path.join(base_url, 'upload/check_legal')
res = requests.post(url=url,
cookies=request.cookies,
json=req_dict,
headers=headers)
print('-----------------------------------')
print(res.text, type(res.text))
print('-----------------------------------')
res_json = json.loads(res.text)
if res_json.get("code") != 200:
return res
upload_tmp_file = UPLOAD_TMP_FILE.format(up_type=up_type,
id=(project_id or requirement_id))
upload_tmp_url = UPLOAD_TMP_URLS.format(up_type=up_type,
id=(project_id or requirement_id))
upload_tmp_url_index = UPLOAD_TMP_URLS_INDEX.format(up_type=up_type,
id=(project_id
or requirement_id))
# 图片临时存放位置
local_tmp_path = get_local_tmp_path(up_type, project_id, requirement_id)
# 文件合法格式
img_type, file_type, compress_type, _ = get_upload_file_type()
# 文件成功数, 失败数, 重复数.
success_file_num, failure_file_num, repeat_file_num = 0, 0, 0
error_content = []
repeat_content = []
for f in uploaded_files:
file_name = f.filename
_name, _type = os.path.splitext(file_name)
ext = _type.strip('.').lower()
# 名称检查
is_legal, res_info = check_name(_name)
if not is_legal:
error_content.append({"file": file_name, "msg": '文件名称不合法'})
Logger.error("%s 文件名称不合法" % file_name)
failure_file_num += 1
continue
# 类型检查
if ext not in (img_type + file_type + compress_type):
error_content.append({"file": file_name, "msg": '文件后缀不合规'})
Logger.error("%s 文件后缀不合规" % file_name)
failure_file_num += 1
continue
file_info = xredis.hgetall(upload_tmp_file)
# 重名检查
if file_name in list(file_info.keys()):
error_content.append({"file": file_name, "msg": '文件名重复'})
Logger.error("%s 文件已存在, 文件名称重复" % file_name)
repeat_file_num += 1
continue
# 存放路径.
save_path = os.path.join(local_tmp_path, f.filename)
# 保存文件, 计算md5值, 判断文件是否已存在
if ext in img_type:
md5_code = save_ret_img_md5(f, save_path)
if ext in file_type:
md5_code = save_ret_file_md5(f, save_path)
if md5_code not in list(file_info.values()):
xredis.hset(upload_tmp_file, file_name, md5_code)
else:
os.remove(save_path)
error_content.append({"file": file_name, "msg": '文件已存在, 内容相同'})
Logger.error("%s 文件已存在, 内容相同" % file_name)
repeat_file_num += 1
continue
# 图片校验
if ext in img_type:
res = check_img_format(save_path)
if res is False:
os.remove(save_path)
xredis.hdel(upload_tmp_file, file_name)
Logger.error("%s 图片过大不合法" % file_name)
failure_file_num += 1
continue
id_ = requirement_id or project_id
cdn_path = get_cdn_path(req_dict={
"upload_type": up_type,
"id": id_
},
filename=file_name)
with open(save_path, 'rb') as pic_f:
upload_img(pic_f, cdn_path)
cdn_info = {'object_url': cdn_path, 'object_type': "img"}
xredis.hset(upload_tmp_url, cdn_path, str(cdn_info))
l_list = {
"object_type": "img",
"object_url": cdn_path,
"object_text": "",
"object_key": cdn_path
}
xredis.lpush(upload_tmp_url_index, str(l_list))
# 文件校验
if ext in file_type:
res, _, error_urls, repeat_urls = check_file_format(
save_path, upload_tmp_url, upload_tmp_url_index)
if res is True:
error_content.extend(error_urls)
repeat_content.extend(repeat_urls)
success_file_num += 1
else:
error_content.extend(error_urls)
repeat_content.extend(repeat_urls)
failure_file_num += 1
xredis.hdel(upload_tmp_file, file_name)
file_list = xredis.hgetall(upload_tmp_file)
object_info = xredis.lrange(upload_tmp_url_index, 0, 9)
ret_info = []
pos = 0
for info in list(object_info):
info = eval(info)
object_type = info.get("object_type")
if object_type in ("tracking", ):
object_url = eval(info.get("object_url"))
main = object_url.get("main", {})
base_url = main.get("base_url", '')
pic_1 = main.get("picture", [''])[0]
object_url = os.path.join(base_url, pic_1)
info["object_url"] = os.path.join(get_endpoint, object_url)
else:
object_url = info.get("object_url")
if object_url and ("http" not in object_url):
abs_url = "{0}/{1}".format(get_endpoint, object_url)
info["object_url"] = abs_url
info["id"] = pos
pos += 1
ret_info.append(info)
total = xredis.llen(upload_tmp_url_index)
data = {
"file_list": list(file_list.keys()),
"object_urls": ret_info,
"error_content": error_content,
"repeat_content": repeat_content,
"success_file_num": success_file_num,
"failure_file_num": failure_file_num,
"repeat_file_num": repeat_file_num
}
# 设定缓存过期时间, 1小时.
xredis.expire(upload_tmp_url, 60 * 60)
xredis.expire(upload_tmp_url_index, 60 * 60)
xredis.expire(upload_tmp_file, 60 * 60)
return jsonify(code=200,
msg="文件上传成功",
content={
"data": data,
"total": total
})
# encoding=utf-8
import pymysql
from com.entity.Article import Article
from config import Logger
log = Logger('all.log', level='debug').logger
class ArticleDao:
# 查找数据
@staticmethod
def select_by_article_id(article_id):
db = pymysql.connect(host="localhost",
user="******",
password="******",
db="keqiao",
charset="utf8mb4")
cursor = db.cursor()
select_sql = "select sid, article_id, name, url, read_count, discuss_count, spread_count from keqiao_article where article_id = '{}' ".format(
article_id)
try:
cursor.execute(select_sql)
result = cursor.fetchone()
if result is None:
return None
else:
sid = result[0]
article_id = result[1]
def confirm_process(req_json):
"""确认上传"""
filename = req_json.get("file_name")
upload_type = req_json.get("upload_type")
project_id = req_json.get('project_id')
requirement_id = req_json.get('requirement_id')
# if upload_type == "requirement":
# obj = PlatformTask.get_instance(id_=requirement_id)
# if not obj:
# return jsonify(code=DATA_NOT_FOUND, msg="无效的需求id")
# if obj.creator_id != current_user.id:
# return jsonify(code=OPERATOR_ERROR, msg="非法操作")
#
# if upload_type == "project":
# obj = Project.get_instance(_id=project_id)
# if not obj:
# return jsonify(code=DATA_NOT_FOUND, msg="无效的项目id")
# if current_user.platform != obj.platform:
# return jsonify(code=OPERATOR_ERROR, msg="非法操作")
# 合法检查
upload_tmp_file = UPLOAD_TMP_FILE.format(up_type=upload_type,
id=(project_id or requirement_id))
upload_tmp_url = UPLOAD_TMP_URLS.format(up_type=upload_type,
id=(project_id or requirement_id))
upload_tmp_url_index = UPLOAD_TMP_URLS_INDEX.format(up_type=upload_type,
id=(project_id
or requirement_id))
req_json["check_type"] = "confirm_process"
req_json["counts"] = xredis.llen(upload_tmp_url_index)
# res = requests.post("check", json=req_json)
# res_json = json.loads(res.text())
# if res_json.get("code") != 200:
# return res
from flask import request
from config.config import VERSION, BASE_API_URL
headers = get_headers()
# api_url = request.base_url.split(VERSION)[-1]
base_url = BASE_API_URL + VERSION
url = os.path.join(base_url, 'upload/check_legal')
res = requests.post(url=url,
cookies=request.cookies,
json=req_json,
headers=headers)
print('-----------------------------------')
print(res.text, type(res.text))
print('-----------------------------------')
res_json = json.loads(res.text)
if res_json.get("code") != 200:
return res
local_path = get_local_tmp_path(upload_type, project_id, requirement_id)
# 类型检查
img_type, file_type, compress_type, _ = get_upload_file_type()
# # 确认提交文件存在.
# if not check_file_exist(local_path, filename):
# return jsonify(code=FILE_NOT_EXIST, msg='文件不存在')
# # 删除不在指定名称列表内文件.
# delete_not_exist_file(local_path, filename)
base_dir = os.path.dirname(local_path)
if upload_type == "requirement":
# 生成批次, 重命名tmp文件
# batch = ImgBatch.create(**{"desc": req_json.get("desc", ""),
# "status": BatchImgStatus.WAITING_PUBLISH.value,
# "img_list": "",
# "requirement_id": requirement_id}
# )
# 更新批次数量
# success_count = xredis.llen(upload_tmp_url_index)
# batch.update(**{"counts": success_count})
# 重命名tmp文件为批次id
batch_id = res_json.get("data").get("batch_id")
os.rename(local_path, os.path.join(base_dir, str(batch_id)))
xredis.delete(upload_tmp_file)
xredis.delete(upload_tmp_url_index)
upload_tmp_batch_urls = UPLOAD_TMP_BATCH_URLS.format(
up_type=upload_type,
id=(project_id or requirement_id),
batch_id=batch_id)
xredis.rename(upload_tmp_url, upload_tmp_batch_urls)
if upload_type == "project":
try:
obj_urls = xredis.hgetall(upload_tmp_url)
publish_task(project_id, obj_urls)
xredis.delete(upload_tmp_file)
xredis.delete(upload_tmp_url)
xredis.delete(upload_tmp_url_index)
except Exception as e:
Logger.error("发布任务异常 %s" % e)
return jsonify(code=1, msg="发布任务异常")
# 文件保存, 图片及压缩包删除.
for name in filename:
_type = name.split('.')[-1]
if _type in file_type:
os.rename(local_path,
os.path.join(base_dir, date2str(datetime.now())))
if os.path.exists(local_path):
shutil.rmtree(local_path)
return jsonify(code=200, msg="确认成功")
import json
import logging.config
from datetime import datetime
from traceback import format_exc as trace
from config import ALERT_FUNCTION, ALERT_SETTINGS, DB_INFO_ALERT_SETTINGS, MONGODB_CONNECTION, LOG_PATH
from config import LOG_CONFIG, Logger
logging.config.fileConfig(LOG_CONFIG)
errors_logger = Logger(logging.getLogger("test_assistant_crawler.collector"),
ALERT_FUNCTION, ALERT_SETTINGS)
db_update_logger = Logger(
logging.getLogger("test_assistant_crawler.db_updater"), ALERT_FUNCTION,
DB_INFO_ALERT_SETTINGS)
class Collector:
def __init__(self, base_path, sources):
self.sources = list(
map(
lambda source: sources[source]
(base_path / source,
Logger(logging.getLogger(f"test_assistant_crawler.{source}"),
ALERT_FUNCTION, ALERT_SETTINGS)), sources))
def load_all(self):
for module in self.sources:
db_update_logger.info(
f'Выполняется загрузка данных из источника {module.__name__}',
def publish_batch(req_json):
batch_id = req_json.get("img_batch_id")
# batch_obj = ImgBatch.query.filter_by(id=batch_id).first()
# if not batch_obj:
# return jsonify(code=PARAMS_NOT_PROVIDED, msg="无效的批次id")
#
# published_id = LocalStatus.query.filter_by(category="batch",
# en_name="published").first().id
# if batch_obj.status == published_id:
# return jsonify(code=PARAMS_NOT_PROVIDED, msg="批次已发布")
#
# local_tmp_path = get_local_tmp_path("requirement", project_id=None,
# requirement_id=batch_obj.requirement_id)
#
# task = PlatformTask.get_instance(batch_obj.requirement_id)
# if task.creator_id != current_user.id:
# return jsonify(code=OPERATOR_ERROR, msg="操作非法")
#
# active_status = LocalStatus.query.filter_by(category="task",
# name="active").first()
# if task.status != active_status.id:
# return jsonify(code=PARAMS_NOT_PROVIDED, msg="需求未审核通过, 不允许发布")
#
# # 需求已发布项目, 更改批次状态为发布.
# if batch_obj.project_id:
# batch_obj.update(**{"status": published_id})
# return jsonify(code=200, msg="更新状态成功")
#
# # 创建项目
# protect_status = LocalStatus.query.filter_by(category="project",
# name="protect").first()
# if not current_user.platform:
# platform = User.query.filter(User.id==current_user.id).first().platform
# else:
# platform = current_user.platform
# request.json["name"] = task.name # 项目名称
# request.json["status"] = protect_status.id # 项目状态 默认保护
# request.json["finish_time"] = date2str(datetime.now() + timedelta(7))
# request.json["requirement_name"] = task.creator.username or task.creator.email
# request.json["batch_id"] = batch_obj.id
# request.json["temp_id"] = task.temp_id
# request.json["platform"] = platform
# request.json["demand_user_id"] = current_user.id
#
# del request.json["img_batch_id"]
# 创建项目
# pro_obj = sql_create(Project, ProjectCreateForm) # 同步执行.
#
# data = json.loads(pro_obj.data)
# project_id = data["data"]["id"]
#
# active_status = LocalStatus.query.filter_by(category="batch",
# name="published").first()
# batch_obj.update(**{"project_id": project_id, "status": active_status.id})
from config.config import VERSION, BASE_API_URL
headers = get_headers()
base_url = BASE_API_URL + VERSION
url = os.path.join(base_url, 'upload/check_legal')
req_json["check_type"] = "publish_batch"
res = requests.post(url, json=req_json, headers=headers)
res_json = json.loads(res.text)
if res_json.get("code") != 200:
return res
try:
project_id = res_json.get("data").get("project_id")
requirement_id = res_json.get("data").get("requirement_id")
upload_tmp_batch_url = UPLOAD_TMP_BATCH_URLS.format(
up_type="requirement", id=requirement_id, batch_id=batch_id)
obj_urls = xredis.hgetall(upload_tmp_batch_url)
publish_task(project_id, obj_urls)
xredis.delete(upload_tmp_batch_url)
except Exception as e:
Logger.error("发布任务异常 %s" % e)
return jsonify(code=1, msg="发布任务异常")
return jsonify(code=200, msg="批次发布成功")
def __init__(self, config: Config, logger: Logger):
self._config = Config() if config is None else config
self._logger = Logger() if logger is None else logger
def delete_path():
"""
***删除路径***
***可以根据upload_type 进行逻辑判断.***
*** 路径 /api/v1.0.0/upload/uploads/delete_path***
:
path: ""
**参数 request.json**
:
upload_type: project # 上传类型
project_id: 100 # 项目id
requirement_id: 8 # 需求id
obj_urls: []
**返回值 **
:
{
"code": 200,
"msg": "处理成功"
}
"""
from config import Logger
try:
res = UploadController.delete_path(request.json)
except Exception as e:
Logger.error(e)
return res
# # todo 保留.
# @upload_bp.route('/upload/upload_delete', methods=['POST'])
# @verify_perm(code='upload_delete_doc')
# def upload_delete():
# """
# ***上传(文档/logo)删除***
#
# *** 路径 ***
# :
# path: "/api/v1.0.0/upload/upload_delete"
#
# **参数 request.json**
# : {
# path: "requirement_doc/2019/05/2275/状态码统计_1559202980.xls"
# requirement_id: 100
# platform_id:100
# }
# **返回值 **
# :
# {
# "code": 200,
# "msg": "处理中"
# }
# """
# from config import Logger
# try:
# res = UploadController.delete_cdn_path(request.json)
# except Exception as e:
# Logger.error(e)
# return res
import sys
from config import Config, Logger
from utils import IpUtils
from web_connector import GoDaddyConnector
if __name__ == '__main__':
if len(sys.argv) != 2:
print('Invalid arguments')
print('Usage:')
print('update_dns.py <dev|prod>')
sys.exit(1)
environment = sys.argv[1]
config = Config(environment)
logger = Logger(config)
ip_utils = IpUtils(config, logger)
external_ip = ip_utils.get_external_ip()
go_daddy_connector = GoDaddyConnector(config, logger)
if external_ip != go_daddy_connector.fetch_ip_from_dns():
go_daddy_connector.update_dns(external_ip)
default=5,
help='Save model checkpoints every k epochs.')
parser.add_argument('--early_stop', type=bool, default=True)
parser.add_argument('--patience', type=int, default=10)
parser.add_argument('--resume', type=bool, default=False)
parser.add_argument('--resume_path',
type=str,
default='./saved_models/model_best.pt')
parser.add_argument('--log_step', type=int, default=20)
# other
parser.add_argument('--cuda', type=bool, default=torch.cuda.is_available())
parser.add_argument('--config_file', type=str, default='./config.json')
parser.add_argument('--seed', type=int, default=1234)
logger = Logger()
cfg = Config(logger=logger, args=parser.parse_args())
cfg.print_config()
cfg.save_config(cfg.config['config_file'])
torch.manual_seed(cfg.config['seed'])
torch.cuda.manual_seed(cfg.config['seed'])
torch.backends.cudnn.enabled = False
np.random.seed(cfg.config['seed'])
# vocab
vocab = load_vocab('dataset/vocab.txt')
tokenizer = Tokenizer(vocab)
# data_loader
return
if datetime.now().hour == self.timer_list[0]:
self.num = self.timer_list.pop(0)
logger.info('Emiter {0} task start...'.format(self.num))
alert = self.emiter()
alert.append(self.emiter_item(alert[0]))
logger.info('Emiter a {0}:{1} {2} alert'.format(
alert[0], alert[1], alert[2]))
return alert
if datetime.now().hour > self.timer_list[0]:
self.timer_list.pop(0)
logger.info('Poped one point...')
def action(self):
tasks = self.actions
for task in tasks:
target = task['action']['host']
item = task['action']['item']
stime = task['action']['start_time']
rtime = task['action']['recovery_time']
ack = task['action']['ack']
state = task['action']['state']
if ack:
pass
if __name__ == '__main__':
Logger()
a = Emitor('config.yml')
a.start()
import argparse
import logging.config
import sys
from config import ELASTICSEARCH_CONNECTION, BASE_PATH, DATA_SOURCES, ALERT_FUNCTION, DB_INFO_ALERT_SETTINGS
from config import LOG_CONFIG, Logger
from sources.collector import Collector
logging.config.fileConfig(LOG_CONFIG)
logger = Logger(logging.getLogger("test_assistant_crawler.runner"),
ALERT_FUNCTION,
DB_INFO_ALERT_SETTINGS)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--load', action='store_const', const=True)
parser.add_argument('--save', action='store_const', const=True)
parser.add_argument('--drop_elastic', action='store_const', const=True)
parser.add_argument('--migrate', action='store_const', const=True)
options = parser.parse_args(sys.argv[1:])
col = Collector(base_path=BASE_PATH, sources=DATA_SOURCES)
if options.load:
logger.info('Action --load triggered', alert=True)
col.load_all() # TODO добавить какой-нибудь report об окончании загрузки с информацией о результатах
logger.info('Action --load completed', alert=True)
'''
Created on Aug 9, 2016
@author: root
'''
import config.Logger as logging
log = logging.getLogger()
def extractValueFromStock(stock, value):
if not value in stock.map:
raise ValueError("The value " + value + " for symbol " + stock.symbol + " is not valid.")
else:
return float(stock.map[value])
class Uploader:
"""
Uploads files to Sharepoint.
"""
def __init__(self, params):
self._debug = params.debug
self._subsite = params.subsite
self._logger = Logger("spuppy", "logs", params.debug)
self._path = params.directory
self._files = params.files
self._out = params.out
def main(self):
"""
Orchestrate upload of a file using given configurations.
"""
spconf = SharepointConfig(self._subsite)
runtime = RuntimeConfig(self._debug)
fconf = FilesConfig(self._path, self._files, self._out)
verify = fconf.verify()
if self._debug:
self._logger.debug(spconf)
self._logger.debug(runtime)
self._logger.debug(fconf)
if verify:
for v in verify:
self._logger.error(v)
return
if self._debug:
self._logger.debug(fconf)
try:
sp = Sharepoint(fconf, spconf, runtime, self._logger)
sp.get_token()
if self._debug:
self._logger.debug(sp)
if not sp.verify_url():
return
if fconf.out_folder and not sp.add_folder():
return
sp.upload_files()
except Exception as e: # pylint: disable=broad-except
# This `except` is intentionally broad to capture it in
# the log for posterity. main exists anyway.
self._logger.exception(e)
