def exportstate(ak, sk, id):
"""Get job state.
Args:
id: job id.
Returns:
json state of the job:
{"id":"5a8f83b058a9b60001ca3129",
"uid":1381102889,"name":"",
"desc":"","type":"dataflow","createTime":"2018-02-23T11:00:00.481+08:00",
"status":"running","message":"","startTime":"2018-02-23T11:00:09.242+08:00",
"jobType":6,"prefix":"","sourceFile":"","targetFile":"","logFile":"",
"params":{"bucket":"ava-test","cmd":"pulp","end_time":"2018-02-22T00:00:00+08:00",
"key":"pulp_20180223110000_a3386433-1845-11e8-85eb-f40f2431779c.json",
"prefix":"exportlogs/","query":"label[?type=='classification' \u0026\u0026 name=='pulp']",
"start_time":"2018-02-21T03:25:00+08:00"},
"targetFileStatistics":{"fileCount":0,"totalImageSize":0,"totalImageDimension":0},
"specVersion":"v1"}
"""
factory = AuthFactory(ak, sk)
url = REMOTE_API + "v1/dataflows/" + id
res = requests.get(url, auth=factory.get_qiniu_auth())
ret = json.loads(res.content)
return ret
def exportstate(ak, sk, id, remote=None):
"""Get job state.
Args:
id: job id.
Returns:
json state of the job:
{"id":"5a8f83b058a9b60001ca3129",
"uid":1381102889,"name":"",
"desc":"","type":"dataflow","createTime":"2018-02-23T11:00:00.481+08:00",
"status":"running","message":"","startTime":"2018-02-23T11:00:09.242+08:00",
"jobType":6,"prefix":"","sourceFile":"","targetFile":"","logFile":"",
"params":{"bucket":"ava-test","cmd":"pulp","end_time":"2018-02-22T00:00:00+08:00",
"key":"pulp_20180223110000_a3386433-1845-11e8-85eb-f40f2431779c.json",
"prefix":"exportlogs/","query":"label[?type=='classification' \u0026\u0026 name=='pulp']",
"start_time":"2018-02-21T03:25:00+08:00"},
"targetFileStatistics":{"fileCount":0,"totalImageSize":0,"totalImageDimension":0},
"specVersion":"v1"}
"""
factory = AuthFactory(ak, sk)
url = remote + "v1/dataflows/" + id
res = requests.get(url, auth=factory.get_qiniu_auth())
ret = json.loads(res.content)
return ret
def config():
factory = AuthFactory(ak,sk)
url = remoteapi+"v1/configuration"
content = {
"bucket":"ava-test",
}
res = requests.post(url,json=content,auth=factory.get_qiniu_auth())
print(res.content)
def exportlogs(ak,
sk,
cmd,
start_time,
end_time,
uids=[],
query="label[?type=='classification' && name=='pulp']",
key="",
bucket="ava-test",
prefix="exportlogs/",
remote=None):
"""Submit a job to export serving logs.
Args:
ak: Qiniu account access key.
sk: Qiniu account secret key.
cmd: cmd of the logs,could be pulp,facex-gender,detect...
start_time: start time of the logs,could be formatted as 2018-02-21T03:25:00
end_time: end time of the logs,for example, 2018-02-22T00:00:00
uids: uid array of the logs,for example,[1380588385, 1381278422]
query: query string of jmes path query,only the logs that match the jmes path will be exported
jmes path website:http://jmespath.org/
key: key of the exported file
bucket: bucket of the exported file
prefix: prefix of the exported file
Returns:
json of the job id: {"id":"5a8f83b058a9b60001ca3129"}
"""
factory = AuthFactory(ak, sk)
url = remote + "v1/dataflows/exportlogs"
if key == "":
id = str(uuid.uuid1())
now = datetime.now()
nowstr = now.strftime("%Y%m%d%H%M%S")
key = cmd + "_" + nowstr + "_" + id + ".json"
content = {
"cmd": cmd,
"start_time": start_time,
"end_time": end_time,
"bucket": bucket,
"prefix": prefix,
"key": key,
"query": query,
# "query":"label[?type=='classification' && name=='pulp'].data|[*][?(class=='pulp')&&score>`0.9`]",
"uid": uids
}
res = requests.post(url, json=content, auth=factory.get_qiniu_auth())
ret = json.loads(res.content)
return ret
def exportlogs(ak, sk, cmd, start_time, end_time, uids=[],
query="label[?type=='classification' && name=='pulp']",
key="", bucket="ava-test", prefix="exportlogs/"):
"""Submit a job to export serving logs.
Args:
ak: Qiniu account access key.
sk: Qiniu account secret key.
cmd: cmd of the logs,could be pulp,facex-gender,detect...
start_time: start time of the logs,could be formatted as 2018-02-21T03:25:00
end_time: end time of the logs,for example, 2018-02-22T00:00:00
uids: uid array of the logs,for example,[1380588385, 1381278422]
query: query string of jmes path query,only the logs that match the jmes path will be exported
jmes path website:http://jmespath.org/
key: key of the exported file
bucket: bucket of the exported file
prefix: prefix of the exported file
Returns:
json of the job id: {"id":"5a8f83b058a9b60001ca3129"}
"""
factory = AuthFactory(ak, sk)
url = REMOTE_API + "v1/dataflows/exportlogs"
if key == "":
id = str(uuid.uuid1())
now = datetime.now()
nowstr = now.strftime("%Y%m%d%H%M%S")
key = cmd + "_" + nowstr + "_" + id + ".json"
content = {
"cmd": cmd,
"start_time": start_time,
"end_time": end_time,
"bucket": bucket,
"prefix": prefix,
"key": key,
"query": query,
# "query":"label[?type=='classification' && name=='pulp'].data|[*][?(class=='pulp')&&score>`0.9`]",
"uid": uids
}
res = requests.post(url, json=content, auth=factory.get_qiniu_auth())
ret = json.loads(res.content)
return ret
def get_ava_auth(configs):
ava_auth_conf = configs["ava_auth_conf"]
upload_access_key = ava_auth_conf["your_access_key"]
upload_secret_key = ava_auth_conf["your_secret_key"]
process_access_key = ava_auth_conf["atlab_access_key"]
process_secret_key = ava_auth_conf["atlab_secret_key"]
upload_auth = qiniu.Auth(upload_access_key, upload_secret_key)
process_auth = AuthFactory(process_access_key, process_secret_key).get_qiniu_auth()
return upload_auth, process_auth
def get_auth_token(configs):
header = None
token = None
if 'ava_auth_conf' in configs:
conf = configs['ava_auth_conf']
factory = AuthFactory(conf["access_key"], conf["secret_key"])
if conf["auth"] == "qiniu/mac":
fauth = factory.get_qiniu_auth
else:
fauth = factory.get_qbox_auth
token = fauth()
elif 'Authorization' in configs:
header = {"Authorization": configs['Authorization']}
else:
header = {"Authorization": "QiniuStub uid=1&ut=2"}
# print 'token: ', token
return header, token
def token_gen(ak,sk):
factory = AuthFactory(ak,sk)
fauth = factory.get_qiniu_auth
token=fauth()
return token
def request_facex_api(api_names,
input_file,
output_dir=None,
config_file=None):
# print_help_info()
if not config_file:
config_file = default_config_file
if not osp.exists(config_file):
print("===> Error: Cannot find config_file: " + config_file)
return
print("===> Load configs from config_file: " + config_file)
configs = load_config_file(config_file)
print('===> configs: {}'.format(configs))
if 'bucket_url' not in configs:
print("===> Warning: 'bucket_url' not in config_file " + config_file)
configs['bucket_url'] = ''
return
header = None
token = None
if 'ava_auth_conf' in configs:
conf = configs['ava_auth_conf']
factory = AuthFactory(conf["access_key"], conf["secret_key"])
if conf["auth"] == "qiniu/mac":
fauth = factory.get_qiniu_auth
else:
fauth = factory.get_qbox_auth
token = fauth()
elif 'Authorization' in configs:
header = {"Authorization": configs['Authorization']}
else:
header = {"Authorization": "QiniuStub uid=1&ut=2"}
# print 'token: ', token
splits = api_names.split('+')
api_names = []
for it in splits:
it = it.strip()
if it == 'detection':
it = 'det'
if it == 'feat':
it = 'feature'
if it in _API_NAMES and it not in api_names:
api_names.append(it)
if len(api_names) < 0:
print("===> Error: Wrong api_names: " + api_names)
return
if 'det' not in api_names:
api_names.insert(0, 'det')
for name in api_names:
api_name = 'facex_%s_url' % name
if api_name not in configs:
print(
"===> Error: {} not in config_file: {}, continue to test next API"
.format(api_name, config_file))
return
if not osp.exists(input_file):
print("===> Error: Cannot find input_file: " + input_file)
return
url_list = []
fp = open(input_file)
for line in fp:
url = line.strip()
if len(url) < 1 or url.startswith('#'):
continue
if 'bucket_url' in configs and not url.startswith('http'):
url = configs['bucket_url'] + '/' + url
url_list.append(url)
fp.close()
if len(url_list) < 1:
print("===> Error: No valid image url")
if not output_dir:
time_str = get_time_string()
output_dir = output_base_dir + '_' + time_str
if not osp.exists(output_dir):
os.makedirs(output_dir)
req_fp_list = []
det_req_file = osp.join(output_dir, 'facex_det_req_data_list.json')
fp = open(det_req_file, 'w')
fp.write('[\n')
req_fp_list.append(fp)
attr_req_file = osp.join(output_dir, 'facex_attr_req_data_list.json')
fp = open(attr_req_file, 'w')
fp.write('[\n')
req_fp_list.append(fp)
resq_save_fps = {}
for api in api_names:
resp_save_file = 'facex_' + api + '_resp_list.json'
resp_save_file = osp.join(output_dir, resp_save_file)
fp = open(resp_save_file, 'w')
fp.write('[\n')
resq_save_fps[api] = fp
is_first = True
for url_idx, url in enumerate(url_list):
# facex-det request data
data_dict = {'data': {'uri': url}}
print("\n\n===>Process image: {}".format(url))
api_url_name = 'facex_det_url'
print("\n---> Test {}: {}".format(api_url_name, configs[api_url_name]))
js_str = json.dumps(data_dict, indent=4)
if not is_first:
req_fp_list[0].write(',\n')
req_fp_list[0].write(js_str)
req_fp_list[0].flush()
# call facex-det api
tm1 = time.clock()
resp = send_request_to_url(configs[api_url_name], data_dict, header,
token)
tm2 = time.clock()
print("---> Request takes {} seconds".format(tm2 - tm1))
# save facex-det response
facex_det_resp = json.loads(resp)
facex_det_resp['uri'] = url
# print facex_det_resp
js_str = json.dumps(facex_det_resp, indent=4)
if not is_first:
resq_save_fps['det'].write(',\n')
resq_save_fps['det'].write(js_str)
resq_save_fps['det'].flush()
print 'facex_Det_Resp', facex_det_resp
print 'facex_det_resp code', facex_det_resp['code']
print 'length of detection', len(
facex_det_resp['result']['detections'])
if (not facex_det_resp
# or facex_det_resp['code']
or 'result' not in facex_det_resp or
'detections' not in facex_det_resp['result'] or
len(facex_det_resp['result']['detections']) < 1):
print '--->facex-det failed or no face detected '
continue
# facex-det request data
faces = facex_det_resp['result']['detections']
print '\n--->%d faces detected' % len(faces)
for face_idx, face in enumerate(faces):
print '\n--->process face #%d' % face_idx
# facex-[age, gender, feature] request data
attr_data_dict = {
'data': {
'uri': url,
'attribute': {
'pts': face['pts']
}
}
}
js_str = json.dumps(attr_data_dict, indent=4)
if not is_first:
req_fp_list[1].write(',\n')
req_fp_list[1].write(js_str)
req_fp_list[1].flush()
for api in api_names:
if api == 'det':
continue
api_url_name = 'facex_%s_url' % api
print("\n---> Test {}: {}".format(api_url_name,
configs[api_url_name]))
tm1 = time.clock()
resp = send_request_to_url(configs[api_url_name],
attr_data_dict, header, token)
tm2 = time.clock()
print("---> Request takes {} seconds".format(tm2 - tm1))
if api is 'feature':
print('feature is in api')
facex_attr_resp = attr_data_dict
base_name = osp.basename(url)
base_name = osp.splitext(base_name)[0]
feat_fn_prefix = '%s_%d' % (base_name, face_idx)
fn_feat = feat_fn_prefix + '.dat'
print('fn_feat%s' % fn_feat)
fn_feat = osp.join(osp.abspath(output_dir), fn_feat)
fp_feat = open(fn_feat, 'wb')
fp_feat.write(resp)
feat_np = convert_feature_data_2_npy(resp)
fn_npy = feat_fn_prefix + '.npy'
fn_npy = osp.join(osp.abspath(output_dir), fn_npy)
np.save(fn_npy, feat_np)
facex_attr_resp['feat_data'] = fn_feat
facex_attr_resp['feat_npy'] = fn_npy
else:
facex_attr_resp = json.loads(resp)
facex_attr_resp['uri'] = url
facex_attr_resp['pts'] = face['pts']
js_str = json.dumps(facex_attr_resp, indent=4)
if not is_first:
resq_save_fps[api].write(',\n')
resq_save_fps[api].write(js_str)
resq_save_fps[api].flush()
is_first = False
for fp in req_fp_list:
fp.write('\n]\n')
fp.close()
for (k, fp) in resq_save_fps.items():
fp.write('\n]\n')
fp.close()
# if 'code' in facex_det_resp and facex_det_resp['code']!=0:
# print('===> Error: code!=0 in facex-det response')
# return
#
# if 'facex_det' in facex_det_resp and len(facex_det_resp['facex_det'])<1:
# print("===> Error: len(resp['facex_det'])<1")
# return
#
# data_dict = {
# 'image': data_dict['image'],
# 'facex_det': facex_det_resp['facex_det']
# }
#
return output_dir
while not stop_signal or not frame_data_queue.empty():
if not frame_data_queue.empty():
frame_data = frame_data_queue.get()
filepath = save_frame(frame_data, frames_dir)
upload_frame(filepath)
###############functions about process frames
process_access_key = "******************************"
process_secret_key = "******************************"
header = {"Content-Type": "application/json"}
detect_url = "http://serve.atlab.ai/v1/eval/facex-detect"
pose_url = "http://serve.atlab.ai/v1/eval/facex-pose"
process_auth = AuthFactory(process_access_key,
process_secret_key).get_qiniu_auth()
def detect_frame(fileurl):
data = {"data": {"uri": fileurl}}
r = requests.post(detect_url,
None,
data,
headers=header,
auth=process_auth)
contentObj = json.loads(r.content)
# print("detected-> " + fileurl)
return contentObj["result"]
def pose_frame(fileurl, det_rst):