def test_get_data_1():
file_id = "test001"
offset = 0
limit = 4
res = fm.get_data(file_id, offset, limit)
expected = {
'status':
'success',
'data_type':
'list',
'total':
4,
'list': [{
'name': 'A2.jpg',
'body':
'/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0a\nHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/wAALCAAcABwBAREA/8QAHwAAAQUBAQEB\nAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1Fh\nByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZ\nWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXG\nx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/9oACAEBAAA/APn+iiituw8LandzH7Rb\nT2VqiebLczwPtRMgA4AyxJKgAdSR0HNdBbeDvD+qyLa6fq99bXbDCLqFoUEjemRwOfcn0zXO654U\n1vw7qL2OpadPFKOVbyyVkH95TjkVmNa3CEboJRnnlDV+DxPrtqEEGsX0YT7m2dht+nNXP+E78V7S\nreIdRYHqHnLZ6+v1pU8eeKkTYuuXYXqV3cH8Pwqpc+KNcvJFe41S5dlXaDvxx17fWsiiiiv/2Q==\n',
'label': ' nameA\n'
}, {
'name': 'B.jpg',
'body':
'/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0a\nHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/wAALCAAcABwBAREA/8QAHwAAAQUBAQEB\nAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1Fh\nByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZ\nWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXG\nx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/9oACAEBAAA/APBrOyudQuVtrSF5pm6K\no/M+w96208E6yxx5UYP+/u/UZFRXPg/W7ZGf7GZVUZPlMGIH+71/SsLFeyaVounaTp1jLp9uxTUL\nRJvtcjczABQwx/ABJv474WtHHHtT4mIkHqpyD3U+o965HxXoegS6qjM7afIYyzpDENshaR239f8A\na244xtwOBTPAWtafBpd3Hq1zax4ZRFkv5zYBHGMjGCByOcDpiuse+0cqGj1OEKSATKyoFz6knGKz\nrrxNoenn97qdvIw5222ZSfoQNv5mvM/Eetvr2sy3uwxx4CRpnlVHTPv1P41k0UUV/9k=\n',
'label': ' nameB\n'
}, {
'name': 'C.jpg',
'body':
'/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0a\nHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/wAALCAAcABwBAREA/8QAHwAAAQUBAQEB\nAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1Fh\nByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZ\nWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXG\nx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/9oACAEBAAA/APEtF09L+eTzT8iKMLnG\n5icAfzrqtG0o+cSJvDdrGrMvl6xKAkrADOAMsMbhgkrnkDOCBjeJbe7g1eQalHZs84DwS2QXyGiH\nyqYinGwbcevHPOa5up7a8ubNibeZo8srEA8EqcjI74Neq6ZB4cvtSA1rUbO1lS3cNb3alEuFcsRI\nJAcBhnG0gcAEZ7cV4v1KCbUre2srq3urWztEto3hhZEHHO3eSx5/i79RXMUV1niNN94x9LO2Y49x\nn+tc/qMBhmTPeKJvzQVTorsdUfzprxmVRjT7bAGeyrWV4kAFxbYGP9Eg/wDQKw6//9k=\n',
'label': ' nameC\n'
}, {
'name': 'A.jpg',
'body':
'/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0a\nHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/wAALCAAcABwBAREA/8QAHwAAAQUBAQEB\nAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1Fh\nByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZ\nWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXG\nx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/9oACAEBAAA/APn+iiituw8LandzH7Rb\nT2VqiebLczwPtRMgA4AyxJKgAdSR0HNdBbeDvD+qyLa6fq99bXbDCLqFoUEjemRwOfcn0zXO654U\n1vw7qL2OpadPFKOVbyyVkH95TjkVmNa3CEboJRnnlDV+DxPrtqEEGsX0YT7m2dht+nNXP+E78V7S\nreIdRYHqHnLZ6+v1pU8eeKkTYuuXYXqV3cH8Pwqpc+KNcvJFe41S5dlXaDvxx17fWsiiiiv/2Q==\n',
'label': ' nameA\n'
}]
}
eq_(res, expected)
def test_get_data_4():
"""
error
offset > # of images
"""
file_id = "test001"
offset = 100000
limit = 100000
res = fm.get_data(file_id, offset, limit)
expected = {"status": "error"}
eq_(res, expected)
def test_get_data_2():
"""
error
set not-exist id
"""
file_id = "not_exist_id"
offset = 0
limit = 4
res = fm.get_data(file_id, offset, limit)
expected = {"status": "error"}
eq_(res, expected)
def handler(wsock, message):
if str(wsock) not in dictionary:
dictionary[str(wsock)] = {}
log_info("####################")
log_info("create new wsock dict.")
log_info("####################")
d = dictionary[str(wsock)]
log_info(dictionary.keys())
try:
obj = json.loads(message)
print(obj)
if obj["action"] == "startUploading":
d["uploading_size"] = 0
d["uploading_file"] = bytearray()
d["action"] = obj["action"]
d["file_size"] = obj.get("fileSize", 0)
d["name"] = obj.get("name", "")
d["description"] = obj.get("description", "")
res = {"action": obj["action"]}
send_message(wsock, res)
elif obj["action"] == "getDataList":
offset = obj.get("offset", 0)
limit = obj.get("limit")
res = fm.get_data_list()
res["action"] = obj["action"]
send_message(wsock, res)
elif obj["action"] == "getData":
offset = obj.get("offset", 0)
limit = obj.get("limit", 10)
data_id = obj.get("dataId")
res = fm.get_data(data_id, offset, limit)
res["action"] = obj["action"]
res["dataId"] = data_id
send_message(wsock, res)
elif obj["action"] == "getRecipeList":
offset = obj.get("offset", 0)
limit = obj.get("limit")
res = fm.get_recipe_list(offset, limit)
res["action"] = obj["action"]
send_message(wsock, res)
elif obj["action"] == "getModelList":
offset = obj.get("offset", 0)
limit = obj.get("limit")
res = fm.get_model_list()
res["action"] = obj["action"]
send_message(wsock, res)
elif obj["action"] == "startLearning":
recipe_id = obj["recipeId"]
data_id = obj["dataId"]
model_info = obj["info"]
config = obj["trainConfig"]
model = CNN(recipe_id)
res = model.train(config, data_id, wsock, model_info)
res["action"] = "finishLearning"
del model
send_message(wsock, res)
elif obj["action"] == "addRecipe":
recipe = obj["recipe"]
res = fm.save_recipe(recipe)
res["action"] = obj["action"]
send_message(wsock, res)
elif obj["action"] == "inferenceImages":
d["uploading_size"] = 0
d["uploading_file"] = bytearray()
d["model_id"] = obj["modelId"]
d["recipe_id"] = obj["recipeId"]
d["inference_type"] = obj["type"]
d["file_size"] = obj["fileSize"]
file_name = obj["fileName"]
d["file_name"] = file_name
log_info(d)
if "zip" == file_name.split(".")[1]:
action = "startUploadingInferenceZip"
d["action"] = "uploadingInferenceZip"
else:
action = "inferenceSingleImage"
d["action"] = action
res = {"action": action}
send_message(wsock, res)
elif obj["action"] == "deleteModel":
model_id = obj["modelId"]
r = fm.delete_model(model_id)
log_info(r)
res = {}
res["action"] = obj["action"]
res["modelId"] = model_id
send_message(wsock, res)
elif obj["action"] == "deleteRecipe":
recipe_id = obj["recipeId"]
r = fm.delete_recipe(recipe_id)
log_info(r)
res = {}
res["action"] = obj["action"]
res["recipeId"] = recipe_id
send_message(wsock, res)
elif obj["action"] == "deleteData":
data_id = obj["dataId"]
r = fm.delete_data(data_id)
log_info(r)
res = {}
res["action"] = obj["action"]
res["dataId"] = data_id
send_message(wsock, res)
elif obj["action"] == "updateData":
data = obj["dataInfo"]
file_id = obj["dataId"]
new_data = fm.update_data_info(data, file_id)
res = {"data": new_data, "action": obj["action"]}
log_info(res)
send_message(wsock, res)
elif obj["action"] == "updateModel":
model_id = obj["modelId"]
model = obj["model"]
put_res = fm.put_model_info(model, model_id)
res = {"model": put_res["detail"], "action": obj["action"]}
log_info(res)
send_message(wsock, res)
elif obj["action"] == "updateRecipe":
recipe_id = obj["recipeId"]
info = obj["info"]
res = fm.get_recipe(recipe_id)
if res and res.get("status") == "success":
body = res["detail"]["body"]
body["info"]["name"] = info["name"]
body["info"]["description"] = info["description"]
new_recipe = fm.update_recipe(recipe_id, body)
res = {"recipe": new_recipe["detail"], "action": obj["action"]}
else:
res = {"status": "error", "action": obj["action"]}
log_info(res)
send_message(wsock, res)
except (UnicodeDecodeError, json.decoder.JSONDecodeError):
print(d["action"])
if d["action"] == "startUploading":
def finished(d):
uploading_file = d["uploading_file"]
file_id = fm.generate_id()
result = fm.put_zip_file(uploading_file,
file_id,
is_expanding=True)
info = fm.get_data_statistics(file_id)
info["name"] = d["name"]
info["description"] = d["description"]
res = fm.put_data_info(info, file_id)
res["action"] = "uploaded"
return res
file_uploader(d, message, wsock, "uploadingLearningData", finished)
elif d["action"] == "uploadingInferenceZip":
def finished(d):
uploading_file = d["uploading_file"]
file_id = fm.generate_id()
result = fm.put_inference_zip(uploading_file,
file_id,
is_expanding=True)
log_info(result)
if result["status"] == "success":
image_path = result["image_path"]
model_id = d["model_id"]
recipe_id = d["recipe_id"]
model = CNN(recipe_id)
inference_type = d["inference_type"]
if inference_type == "classification":
inference_res = model.classify(model_id, image_path)
action = "finishClassfication"
elif inference_type == "vectorization":
inference_res = model.vectorize(model_id,
image_path,
is_similarity=True)
action = "finishVectorization"
res = fm.get_inferece_images(image_path)
image_list = res["list"]
res_list = []
for r, i in zip(inference_res, image_list):
r["body"] = i["body"]
res_list.append(r)
res = {"action": action, "id": file_id, "list": res_list}
return res
file_uploader(d, message, wsock, "uploadingInferenceZip", finished)
elif d["action"] == "inferenceSingleImage":
res = fm.save_inference(message)
file_id = res["detail"]["id"]
file_path = res["detail"]["file_path"]
recipe_id = d["recipe_id"]
model_id = d["model_id"]
file_name = d["file_name"]
model = CNN(recipe_id)
inference_type = d["inference_type"]
if inference_type == "classification":
inference_res = model.classify(model_id, file_path)
action = "finishClassfication"
if inference_type == "vectorization":
inference_res = model.vectorize(model_id,
file_path,
is_similarity=False)
action = "finishVectorization"
inference_res[0]["image_name"] = file_name
res = {"list": inference_res, "action": action}
fm.delete_inference(file_id)
del dictionary[str(wsock)]
send_message(wsock, res)
def handler(wsock, message):
d = dictionary[str(wsock)]
log_debug(dictionary.keys())
log_debug(d["size"])
try:
obj = json.loads(message)
print(obj)
if obj["action"] == "startUploading":
d["file_size"] = obj.get("fileSize", 0)
d["name"] = obj.get("name", "")
d["description"] = obj.get("description", "")
elif obj["action"] == "getDataList":
offset = obj.get("offset", 0)
limit = obj.get("limit")
res = fm.get_data_list()
res["action"] = obj["action"]
wsock.send(json.dumps(res))
elif obj["action"] == "getData":
offset = obj.get("offset", 0)
limit = obj.get("limit", 10)
data_id = obj.get("dataId")
res = fm.get_data(data_id, offset, limit)
res["action"] = obj["action"]
res["dataId"] = data_id
wsock.send(json.dumps(res))
elif obj["action"] == "getRecipeList":
offset = obj.get("offset", 0)
limit = obj.get("limit")
res = fm.get_recipe_list(offset, limit)
res["action"] = obj["action"]
wsock.send(json.dumps(res))
elif obj["action"] == "getModelList":
offset = obj.get("offset", 0)
limit = obj.get("limit")
res = fm.get_model_list()
res["action"] = obj["action"]
wsock.send(json.dumps(res))
elif obj["action"] == "startLearning":
recipe_id = obj["recipeId"]
data_id = obj["dataId"]
model_info = obj["info"]
config = obj["trainConfig"]
model = CNN(recipe_id)
res = model.train(config, data_id, wsock, model_info)
res["action"] = "finishLearning"
del model
wsock.send(json.dumps(res))
elif obj["action"] == "addRecipe":
recipe = obj["recipe"]
res = fm.save_recipe(recipe)
res["action"] = obj["action"]
wsock.send(json.dumps(res))
elif obj["action"] == "deleteModel":
model_id = obj["modelId"]
r = fm.delete_model(model_id)
log_debug(r)
res = {}
res["action"] = obj["action"]
res["modelId"] = model_id
wsock.send(json.dumps(res))
elif obj["action"] == "deleteRecipe":
recipe_id = obj["recipeId"]
r = fm.delete_recipe(recipe_id)
log_debug(r)
res = {}
res["action"] = obj["action"]
res["recipeId"] = recipe_id
wsock.send(json.dumps(res))
elif obj["action"] == "deleteData":
data_id = obj["dataId"]
r = fm.delete_data(data_id)
log_debug(r)
res = {}
res["action"] = obj["action"]
res["dataId"] = data_id
wsock.send(json.dumps(res))
elif obj["action"] == "updateData":
data = obj["dataInfo"]
file_id = obj["dataId"]
new_data = fm.put_data_info(data, file_id)
res = {
"data": new_data,
"action": obj["action"]
}
log_debug(res)
wsock.send(json.dumps(res))
elif obj["action"] == "updateModel":
model_id = obj["modelId"]
model = obj["model"]
put_res = fm.put_model_info(model, model_id)
res = {
"model": put_res["detail"],
"action": obj["action"]
}
log_debug(res)
wsock.send(json.dumps(res))
elif obj["action"] == "updateRecipe":
recipe_id = obj["recipeId"]
info = obj["info"]
res = fm.get_recipe(recipe_id)
if res and res.get("status") == "success":
body = res["detail"]["body"]
body["info"]["name"] = info["name"]
body["info"]["description"] = info["description"]
new_recipe = fm.update_recipe(recipe_id, body)
res = {
"recipe": new_recipe["detail"],
"action": obj["action"]
}
else:
res = {
"status": "error",
"action": obj["action"]
}
log_debug(res)
wsock.send(json.dumps(res))
except (UnicodeDecodeError, json.decoder.JSONDecodeError):
d["size"] += len(message)
d["uploading_file"] += message
response = {"status": "loading", "loadedSize": d["size"]}
print(response)
time.sleep(0.05) # for the progress bar.
wsock.send(json.dumps(response))
if d["size"] == int(d["file_size"]):
uploading_file = d["uploading_file"]
file_id = fm.generate_id()
result = fm.put_zip_file(uploading_file, file_id, is_expanding=True)
new_data = {
"name": d["name"],
"description": d["description"]
}
fm.put_data_info(new_data, file_id)
response = {"action": "uploaded", "fileId": file_id}
wsock.send(json.dumps(response))