def create_docker_sync_job(self, obj):
# s = {r'/': r'...'}
# jn = utils.multiple_replace(jobname, s)
s = utils.randomstr(8)
jn = '__docker_sync_job_' + s
cid = obj['sync_cloud_id']
jsonUtil = JsonUtil()
c = jsonUtil.parseJsonString(config.CLOUD_CONFIG)
j = Jenkins(c[cid]['jenkins_url'],
username=c[cid]['jenkins_name'],
password=c[cid]['jenkins_token'])
re = postimagesync()
re.time = datetime.now()
re.sync_cloud_id = obj['sync_cloud_id']
re.image_name = obj['image_name']
re.post_callback_url = obj['post_callback_url']
re.tag = obj['tag']
re.status = 'started'
try:
if j.job_exists(jn):
jn = jn + utils.randomstr(4)
j.create_job(jn, self.edit_docker_sync_job_config(obj))
yield gen.sleep(0.5)
j.build_job(jn)
j.create_job(jn, self.edit_docker_sync_job_config(obj))
yield gen.sleep(0.5)
j.build_job(jn)
except Exception as e:
print e.message
re.status = 'error'
raise gen.Return(re)
def post_docker_sync_hook(self, obj, cloudid):
jn = obj['name']
# bid = str(obj['build']['number'])
jsonUtil = JsonUtil()
c = jsonUtil.parseJsonString(config.CLOUD_CONFIG)
j = Jenkins(c[cloudid]['jenkins_url'],
username=c[cloudid]['jenkins_name'],
password=c[cloudid]['jenkins_token'])
re = postimagesync()
try:
if j.job_exists(jn):
ss = xmltodict.parse(j.get_job_config(jn))
jsonpickle.set_preferred_backend('json')
if isinstance(jsonpickle.decode(ss['project']['description']),
callback):
desobj = jsonpickle.decode(ss['project']['description'])
re.image_name = desobj.image_name
re.status = self.getstatus(obj['build']['status'])
re.sync_cloud_id = desobj.repo_name
re.tag = desobj.tag
re.time = datetime.now()
re.post_callback_url = desobj.callback_url
if re.status != 'error' and config.JENKINS_IMAGEOPTJOB_DELETE == 'true':
j.delete_job(jn)
except Exception as e:
print e.message
re = None
raise gen.Return(re)
def write_json(self, data,status_code=200, msg='success.'):
jsonUtil = JsonUtil()
self.finish(
jsonUtil.parseJsonObj(
{
'error_info':data,
'error_code': status_code,
'error_reason': msg,
}
))
def write_httperror(self,data,**kwargs):
jsonUtil = JsonUtil()
self.finish(
jsonUtil.parseJsonObj(
{
'error_info':data,
'error_code': kwargs['status_code'],
'error_reason': kwargs['reason'],
}
))
def edit_docker_sync_job_config(self, obj):
try:
# {docker_login} && docker pull {oldimage} && docker tag {oldimage} {newimage} && docker push {newimage}
ss = xmltodict.parse(self.getdocker_sync_config())
jsonUtil = JsonUtil()
c = jsonUtil.parseJsonString(config.CLOUD_CONFIG)
cid = obj['sync_cloud_id']
desobj = callback()
desobj.des = ""
desobj.callback_url = obj['post_callback_url']
desobj.build_id = ''
desobj.duration = ''
desobj.namespace = ""
desobj.repo_name = obj['sync_cloud_id'] # 把cloudid 临时存在 这
desobj.image_name = obj['image_name']
desobj.status = ''
desobj.tag = obj['tag']
desobj.time = ''
jsonpickle.set_preferred_backend('json')
ss['project']['description'] = jsonpickle.encode(desobj)
ss['project']['properties']['com.tikal.hudson.plugins.notification.HudsonNotificationProperty']['endpoints'] \
['com.tikal.hudson.plugins.notification.Endpoint']['url'] = config.JOBHOOKURL+'?cloudid='+obj['sync_cloud_id']
tempstr = str(
ss['project']['builders']['hudson.tasks.Shell']['command'])
s = {
r'{docker_login}':
c[cid]['login_common'],
r'{oldimage}':
config.REGISTRYNAME + '/' + obj['image_name'] + ':' +
obj['tag'],
r'{newimage}':
c[cid]['registry_name'] + '/' + obj['image_name'] + ':' +
obj['tag']
}
ss['project']['builders']['hudson.tasks.Shell'][
'command'] = utils.multiple_replace(tempstr, s)
return xmltodict.unparse(ss)
except Exception as e:
print e.message
def main():
worldcup_url = 'http://www.fifa.com/fifa-tournaments/statistics-and-records/worldcup/'
champions_league_url = 'http://www.uefa.com/uefachampionsleague/season=2016/statistics/round=2000634/clubs/index.html'
scraper = Scraper()
cwd = os.getcwd()
csv_util = CsvUtil()
json_util = JsonUtil()
# Scrape World Cup data then write to csv and json format
worldcup_edition_list = scraper.scrape_worldcup_data(worldcup_url)
# Write data to file
print 'Writing World Cup data to csv file!'
csv_util.write_world_cup_data_to_file(
cwd + os.sep + "data" + os.sep + 'world_cup_data.csv',
worldcup_edition_list)
print 'Writing World Cup data to json file!'
json_util.write_world_cup_data_to_file(
cwd + os.sep + "data" + os.sep + 'world_cup_data.json',
worldcup_edition_list)
# Scrape Champions League data then write to csv and json format
champions_league_teams = scraper.scrape_champions_league_data(
champions_league_url)
# Write data to file
print 'Writing Champions League data to csv file!'
csv_util.write_champions_league_data_to_file(
cwd + os.sep + "data" + os.sep + 'champions_league_data.csv',
champions_league_teams)
print 'Writing Champions League data to json file!'
json_util.write_champions_league_data_to_file(
cwd + os.sep + "data" + os.sep + 'champions_league_data.json',
champions_league_teams)
def execute(self, casedata):
#logging.debug(casedata.keys())
if '指定header' in casedata.keys() and casedata['指定header'].upper(
) == 'Y':
if 'header内容' in casedata.keys():
header_value = casedata['header内容']
else:
header_value = {}
# 如果header内容为{...}格式则直接解析为json内容，否则根据header_file中对应标签提取json内容
#Todo: header内容直接写在Excel中也应支持表达式替换
if re.findall("^{.+}$", header_value.replace(
"\n", "")) != []: # 含\n时匹配为空，why?
header = json.loads(header_value)
else:
jutil = JsonUtil(get_header_file())
header = jutil.get_data(header_value)
#header_str = eval_from_string(repr(header),return_str=True,json_str=True)
#header = json.loads(json.dumps(eval(header_str))) #直接用json.loads(header_str)会报json.decoder.JSONDecodeError: Expecting property name enclosed in double quotes: line 1 column 2 (char 1)
header = eval_from_string(repr(header))
else:
header = None
data = []
if '请求数据' in casedata.keys() and casedata['请求数据'] != '':
request_data = casedata['请求数据'].strip()
# 如果请求数据为{...}格式则直接解析为json内容，否则根据data_file中对应标签提取json内容
#if re.findall("^{.+}$",request_data.replace("\n",""))!=[]: #含\n时匹配为空，why?
try:
data = json.loads(request_data)
data_str = request_data
except:
if ';' in request_data: # 分号分隔的等式，针对json格式数据中调用函数返回值非字符串型，原有方法难以处理的问题
items = request_data.split(';')
data = {}
for item in items:
key, value = tuple(
ele.strip('\n') for ele in item.split('=', 1))
data[key] = eval(eval_from_string(value))
data_str = repr(data)
elif not '=' in request_data: #request_data为data文件中标签名
jutil = JsonUtil(get_data_file())
data = jutil.get_data(request_data) #json格式数据
data_str = repr(data)
else: #形如url=/xxx&type=POST¶ms={"id":"123","type":"1","paths":{}}&contentType=1的接口
data_str = request_data.strip()
# result = eval_from_string(data_str,return_str=True,json_str=True)
# if not isinstance(result,str):
# result = repr(result)
# #logging.debug("data_str: %s"%data_str)
# #将函数执行后的结果再重新构造成json对象
# data = json.loads(json.dumps(eval(result))) #json.dumps(eval(data_str))的作用是兼容单引号内容
# 解析表达式
data = eval_from_string(data_str)
if isinstance(data, str):
try:
# 如果data是json格式字符串则转换为json对象
data = json.loads(data) # json.dumps(eval(xxx))的作用是兼容单引号内容
except:
pass
#获取case依赖数据
if 'case依赖' in casedata.keys(
) and casedata['case依赖'] != '': #casedata已先期将nan替换为''
depend_data = DependentData(casedata['case依赖'])
#依赖的返回数据可能不止一处，返回的json数据，返回的html页面的一部分(authenticity_token， 如果要动态操作repository，则url也依赖返回的html页面元素)
value = depend_data.get_data_for_case(casedata)
if '数据依赖字段' in casedata.keys():
field = casedata['数据依赖字段']
data[field] = value
else:
return "未指定数据依赖字段"
if 'Root_URL' in casedata:
root_url = casedata['Root_URL']
else:
root_url = ""
#root_url = eval_from_string(root_url, return_str=True, json_str=False)
root_url = eval_from_string(root_url)
if 'relative_URL' in casedata.keys():
#relative_url = eval_from_string(casedata['relative_URL'], return_str=True, json_str=False)
relative_url = eval_from_string(casedata['relative_URL'])
else:
relative_url = ""
# if 'encode_URL' in casedata.keys() and casedata['encode_URL'].upper()=='Y':
# relative_url = quote(relative_url, 'utf-8')
url = root_url + relative_url
if '新会话' in casedata.keys() and casedata['新会话'].upper() == 'Y':
#print("新会话~~~~~~~~~~~~~~~~~~~~~~~~")
new_session = True
else:
#print("保持会话~~~~~~~~~~~~~~~~~~~~~~")
new_session = False
if '+' in data:
data = data.replace("+", "%2B")
logging.debug("data:" + repr(data))
self.data = data
#glo.set_value("post_data",data)
#verify = get_verify_str()
# if verify.upper()=='FALSE':
# # logging.debug("no need to vefify certification.")
# #verify = eval("False")
# verify = False
# cert = None
# else:
# verify=True
# cert = sys.path[0] + '/../' + get_certfile_path()
#cert = get_cert()
verify, cert = get_verify_cert()
if not '请求类型' in casedata.keys():
return "未指定请求类型"
if '数据传参' in casedata.keys() and casedata['数据传参'].lower() == 'json':
res = RunMethod().run_main(method=casedata['请求类型'],
url=url,
json=data,
headers=header,
verify=verify,
cert=cert,
new_session=new_session)
else:
res = RunMethod().run_main(method=casedata['请求类型'],
url=url,
data=data,
headers=header,
verify=verify,
cert=cert,
new_session=new_session)
return res
def write_json_byobj(self, data):
jsonUtil = JsonUtil()
retstr = jsonUtil.parseJsonObj(data)
self.finish(retstr)
def loadjson(self,jsonstr):
jsonUtil = JsonUtil()
return jsonUtil.parseJsonString(jsonstr)