def write_json(mills, path):
try:
with open(path, 'w') as f:
f.write(json.dumps(mills))
logger.info('Completed writing %s' % path)
except Exception as e:
logger.error('Failed writing %s' % path)
def webhook():
"""
接收git提交参数，默认提交处理
:return:
"""
# try:
ret = request.data.decode('utf-8')
# print(ret, type(ret))
data = json.loads(ret)
# git提交类型
event = data['object_kind']
print("event:", event)
branch = data['ref']
print(branch)
branch_name = branch if branch.find('/') < 0 else branch[branch.rfind('/') + 1:]
print('branch_name:', branch_name)
git_url = data['project']['url']
pro_name = "CI__" \
+ git_url[git_url.rfind(':') + 1:git_url.rfind("/")] + "__" \
+ git_url[git_url.rindex('/') + 1:git_url.rindex(".git")] + "__" + branch_name
if event == 'push':
if branch_name.startswith('release-'):
create_build(data)
elif branch_name.startswith('develop') and (request.args.get('dev') == '1' or request.args.get('dev') == 'true'):
create_build(data)
else:
logger.error('未处理分支类型：[%s]！' % branch_name)
elif event == 'tag_push':
create_build(data)
else:
logger.error("未处理类型: [%s]！" % event)
logger.info('已经完成对[%s]项目的创建！' % pro_name + '\n' + '分支/标签: [%s]' % branch_name)
return 'Hello Webhook!'
def create_build(data, project_type='default'):
"""
构建默认提交类型
:param data:
:param project_type:
:return:
"""
git_url = data['project']['url']
branch = data['ref']
branch_name = branch if branch.find(
'/') < 0 else branch[branch.rfind('/') + 1:]
pro_name = "CI__" \
+ git_url[git_url.rfind(':') + 1:git_url.rfind("/")] + "__" \
+ git_url[git_url.rindex('/') + 1:git_url.rindex(".git")] + "__" + branch_name
if project_type == 'jdk7':
build_name = 'JDK_7u79'
config = get_simple_maven_config(url=git_url,
branch=branch,
jdk_version=build_name)
elif project_type == 'jdk8':
build_name = 'JDK_8u112'
config = get_simple_maven_config(url=git_url,
branch=branch,
jdk_version=build_name)
elif project_type == 'npm':
config = get_simple_npm_config(url=git_url, branch=branch)
elif not project_type or project_type == 'default':
config = get_simple_default_config(url=git_url, branch=branch)
else:
logger.error('项目类型有误: [%s]' % project_type)
# raise Exception("Project Type Error: [%s]" % project_type)
s = 0
while s < 20:
s += 1
try:
if not server.get_job_name(pro_name):
server.create_job(pro_name, config)
logger.info('新的项目，开始创建[%s]项目名称！' % pro_name)
else:
logger.info('[%s]项目名称已存在，准备开始构建！' % pro_name)
break
except jenkins.JenkinsException as e:
continue
while s < 30:
s += 1
try:
log.info("项目 [%s] 构建中..." % pro_name)
server.build_job(pro_name)
log.info("项目 [%s] 构建完成!" % pro_name)
break
except Exception as e:
log.error("项目 [%s] 构建失败！" + pro_name + '\n' +
traceback.format_exc())
continue
logger.info('构建项目 [%s] 信息:' % pro_name + '分支/标签: [%s]' % branch_name +
'构建重试次数：[%s]' % (s - 2))
def webhook():
"""
接收git提交参数，默认提交处理
:return:
"""
try:
# print("\n\n\n\n1111111111")
# print('method:', request.methodd)
# print("ret:", type(ret), retaa)
ret = request.data.decode('utf-8')
data = json.loads(ret)
print('data:', type(data), data)
# for k, v in data.items():
# print('k:', k, 'v:', v)
# git提交类型
event = data['object_kind']
# print('event:', event)
branch = data['ref']
branch_name = branch if branch.find(
'/') < 0 else branch[branch.rfind('/') + 1:]
# print('branch_name:', branch_name)
# if event == 'push':
# if branch_name.startswith('release-'):
# default_push(data)
# elif branch_name.startswith('develop') and request.args.get('dev') == '1' or request.args.get('dev') == 'true':
# default_push(data)
# elif event == 'tag_push':
# default_push(data)
# if event == 'push':
# default_push(data)
# elif event == 'tag_push':
# default_push(data)
if branch_name != 'master' and event == 'push' or event == 'tag_push':
if branch_name.startswith('release-') or branch_name.startswith(
'develop') and request.args.get(
'dev') == '1' or request.args.get('dev') == 'true':
default_push(data)
return 'Hello Webhook!'
except Exception as e:
logger.error(traceback.format_exc())
def webhook_jdk_npm(j_n):
"""
接收git提交参数，处理以分支和url是jdk和npm类型
:param j_n:
:return:
"""
# 获取JDK版本
# git提交信息
ret = request.data.decode('utf-8')
data = json.loads(ret)
# git提交类型
event = data['object_kind']
branch = data['ref']
branch_name = branch if branch.find(
'/') < 0 else branch[branch.rfind('/') + 1:]
git_url = data['project']['url']
pro_name = "CI__" \
+ git_url[git_url.rfind(':') + 1:git_url.rfind("/")] + "__" \
+ git_url[git_url.rindex('/') + 1:git_url.rindex(".git")] + "__" + branch_name
if event == 'push':
if branch_name.startswith('release-') or branch_name.startswith(
'dev-debug-'):
create_build(data, j_n)
elif branch_name.startswith('develop') and (
request.args.get('dev') == '1'
or request.args.get('dev') == 'true'):
create_build(data, j_n)
else:
for i, v in request.args.items():
if (v == '1' or v == 'true') and branch_name == i:
create_build(data, j_n)
break
else:
logger.error('未处理分支类型：[%s]！' % branch_name)
elif event == 'tag_push':
if j_n.startswith('jdk') or j_n.startswith('npm'):
create_build(data, j_n)
else:
logger.error("未处理类型: [%s]！" % event)
return 'Hello Webhook!'
def build_uml_data(output_path, mills_api_url, request_params):
res = {}
if os.path.exists(output_path):
logger.info("Reading UML mills from local file.")
try:
with open(output_path, 'r') as f:
res = json.load(f)
except Exception as e:
logger.error("Failed to read UML file.")
pass
else:
try:
mills_dict = {}
# Request mills from opendata.arcgis.com
req = requests.get(mills_api_url, params=request_params)
res_json = json.loads(req.text)
# Handle empty response or missing mills data
if 'features' not in res_json or len(res_json['features']) == 0:
logger.error('Missing mills data')
pass
# Extract mills properties from response JSON
mills = res_json['features']
mills_dict = {
x["properties"]["objectid"]: x["properties"]
for x in mills
}
column_mapper = {'Group_Name': 'group_name', 'id': 'umlid'}
for k, v in mills_dict.items():
if v['country'] in request_params['country']:
for old, new in column_mapper.items():
v[new] = v.pop(old)
res[k] = v
write_json(res, output_path)
except Exception as e:
print(e)
logger.error("Failed to read UML mills from API.")
return pd.DataFrame.from_dict(res, orient='index')
def write_geojson(gdf, path):
try:
gdf.to_file(path, driver='GeoJSON')
logger.info('Completed writing %s' % path)
except Exception as e:
logger.error('Failed writing %s' % path)
def write_df(df, path, index=False):
try:
df.to_csv(path, index=index)
logger.info('Completed writing %s' % path)
except Exception as e:
logger.error('Failed writing %s' % path)
def local_datetime(data):
'''
把data转换为日期时间，时区为东八区北京时间，能够识别：今天、昨天、5分钟前等等，如果不能成功识别，则返回datetime.datetime.now()
'''
dt = datetime.datetime.now()
# html实体字符转义
# data = HTMLParser.HTMLParser().unescape(data)
data = data.strip()
if not data:
return dt
try:
data = text_to_str(data)
except Exception as e:
logger.error("utc_datetime() error: data is not utf8 or unicode : %s" % data)
# 归一化
data = data.replace("年", "-").replace("月", "-").replace("日", " ").replace("/", "-").strip()
data = re.sub("\s+", " ", data)
year = dt.year
regex_format_list = [
# 2013年8月15日 22:46:21
("(\d{4}-\d{1,2}-\d{1,2} \d{1,2}:\d{1,2}:\d{1,2})", "%Y-%m-%d %H:%M:%S", ""),
# "2013年8月15日 22:46"
("(\d{4}-\d{1,2}-\d{1,2} \d{1,2}:\d{1,2})", "%Y-%m-%d %H:%M", ""),
# "2014年5月11日"
("(\d{4}-\d{1,2}-\d{1,2})", "%Y-%m-%d", ""),
# "2014年5月"
("(\d{4}-\d{1,2})", "%Y-%m", ""),
# "13年8月15日 22:46:21",
("(\d{2}-\d{1,2}-\d{1,2} \d{1,2}:\d{1,2}:\d{1,2})", "%y-%m-%d %H:%M:%S", ""),
# "13年8月15日 22:46",
("(\d{2}-\d{1,2}-\d{1,2} \d{1,2}:\d{1,2})", "%y-%m-%d %H:%M", ""),
# "8月15日 22:46:21",
("(\d{1,2}-\d{1,2} \d{1,2}:\d{1,2}:\d{1,2})", "%Y-%m-%d %H:%M:%S", "+year"),
# "8月15日 22:46",
("(\d{1,2}-\d{1,2} \d{1,2}:\d{1,2})", "%Y-%m-%d %H:%M", "+year"),
# "8月15日",
("(\d{1,2}-\d{1,2})", "%Y-%m-%d", "+year"),
# "3 秒前",
("(\d+)\s*秒前", "", "-seconds"),
# "3 分钟前",
("(\d+)\s*分钟前", "", "-minutes"),
# "3 秒前",
("(\d+)\s*小时前", "", "-hours"),
# "3 秒前",
("(\d+)\s*天前", "", "-days"),
# 今天 15:42:21
("今天\s*(\d{1,2}:\d{1,2}:\d{1,2})", "%Y-%m-%d %H:%M:%S", "date-0"),
# 昨天 15:42:21
("昨天\s*(\d{1,2}:\d{1,2}:\d{1,2})", "%Y-%m-%d %H:%M:%S", "date-1"),
# 前天 15:42:21
("前天\s*(\d{1,2}:\d{1,2}:\d{1,2})", "%Y-%m-%d %H:%M:%S", "date-2"),
# 今天 15:42
("今天\s*(\d{1,2}:\d{1,2})", "%Y-%m-%d %H:%M", "date-0"),
# 昨天 15:42
("昨天\s*(\d{1,2}:\d{1,2})", "%Y-%m-%d %H:%M", "date-1"),
# 前天 15:42
("前天\s*(\d{1,2}:\d{1,2})", "%Y-%m-%d %H:%M", "date-2"),
# 前天
]
for regex, dt_format, flag in regex_format_list:
m = re.search(regex, data)
if m:
if not flag:
dt = datetime.datetime.strptime(m.group(1), dt_format)
elif flag == "+year":
# 需要增加年份
dt = datetime.datetime.strptime("%s-%s" % (year, m.group(1)), dt_format)
elif flag in ("-seconds", "-minutes", "-hours", "-days"):
flag = flag.strip("-")
if flag == 'seconds':
dt = dt - datetime.timedelta(seconds = int(m.group(1)))
elif flag=='minutes':
dt = dt - datetime.timedelta(minutes=int(m.group(1)))
elif flag =="hours":
dt = dt - datetime.timedelta(hours=int(m.group(1)))
elif flag=='days':
dt = dt - datetime.timedelta(days=int(m.group(1)))
# exec("dt = dt - datetime.timedelta(%s = int(m.group(1)))" % flag)
elif flag.startswith("date"):
del_days = int(flag.split('-')[1])
_date = dt.date() - datetime.timedelta(days=del_days)
_date = _date.strftime("%Y-%m-%d")
dt = datetime.datetime.strptime("%s %s" % (_date, m.group(1)), dt_format)
return dt
else:
logger.error("unknow datetime format: %s" % data)
return dt
return jsonify(dal.get_mills_aggregate_stats())
@app.errorhandler(404)
@cross_origin()
def not_found(error):
'''
Generates an error response with a "404 - Not Found" status code
if the API receives a request for a non-existent resource/endpoint.
Parameters:
error (Exception): The exception caught by the error handler
Returns:
(flask.Flask.response_class): the response message
'''
return make_response(jsonify({'error': 'Not found'}), 404)
if __name__ == "__main__":
if len(sys.argv) == 2:
if sys.argv[1] == 'docker':
input_path = './'
elif sys.argv[1] == 'local':
input_path = '../data/'
dal = DAL(input_path)
app.run(host=config.get('host'), port=config.get('port'))
else:
logger.error('Invalid run arguments. Example: python app.py local')
sys.exit(1)