def test_exists():
user = MockUser().mock_user()
store = Store(user)
store.rootpath = os.path.expanduser(tempdir + "/.autodl/")
store.path = posixpath.join(store.rootpath, store.user.get("username"))
project_path = store.create(project="test_project_dir")
project_path_exists = store.exist(project="test_project_dir")
assert project_path_exists == True
project_path_none = store.exist()
assert project_path_none == True
def download_code(request):
"""
Endpoint for dowloading generated code.
Inputs:
-------
request:
Requires: username & project_id in request data
token in request header
Returns:
--------
response: HttpResponse (Required File)
(Yes, not JsonResponse like other views)
"""
try:
username = request.data.get("username")
user = User(username=username, password=None)
user = user.find()
store_obj = Store(user)
project_id = request.data.get("project_id")
if not store_obj.exist(project_id):
raise Exception("No such project exists")
project_dir = store_obj.path + os.sep + project_id
f = open(project_dir + os.sep + "test.py", "r")
response = HttpResponse(f, content_type="application/x-python-code")
response["Content-Disposition"] = "attachment; filename=test.py"
except Exception as e:
response = HttpResponse("<h1>File not found</h1>")
return response
def get_hyperparams(request):
try:
username = request.data.get("username")
user = User(username=username, password=None)
user = user.find()
store_obj = Store(user)
project_id = request.data.get("project_id")
if not store_obj.exist(project_id):
raise Exception("No such project exists")
project_dir = store_obj.path + os.sep + project_id
with open(project_dir + os.sep + "hyperparams.json", "r") as f:
hyperparams = json.load(f)
status, success, message = 200, True, "hyperparams fetched"
except Exception as e:
status, success, message, hyperparams = (
500,
False,
"Could not fetch hyperparams ",
{},
)
return JsonResponse(
{"success": success, "message": message, "hyperparams": hyperparams},
status=status,
)
def save_layers(request):
try:
username = request.data.get("username")
user = User(username=username, password=None)
user = user.find()
store_obj = Store(user)
project_id = request.data.get("project_id")
layers = request.data.get("layer_json")
components = request.data.get("component_array")
if not store_obj.exist(project_id):
raise Exception("No such project exists")
project_dir = store_obj.path + os.sep + project_id
with open(project_dir + os.sep + "layers.json", "w") as f:
json.dump(layers, f)
with open(project_dir + os.sep + "components.json", "w") as f:
json.dump(components, f)
status, success, message = 200, True, "Layers saved successfully"
except Exception as e:
status, success, message = 500, False, "Could not save layers"
return JsonResponse({"success": success, "message": message}, status=status)
def get_project(request):
try:
username = request.data.get("username")
user = User(username=username, password=None)
user = user.find()
store_obj = Store(user)
project_id = request.data.get("project_id")
if not store_obj.exist(project_id):
raise Exception("No such project exists")
project_dir = store_obj.path + os.sep + project_id
with open(project_dir + os.sep + "layers.json", "r") as f:
layers = json.load(f)
b2f_json = layers
status, success, message = 200, True, "Layers Fetched"
except Exception as e:
status, success, message, b2f_json = (
500,
False,
"Layers could not be fetched",
{},
)
return JsonResponse(
{"success": success, "message": message, "b2f_json": b2f_json}, status=status
)
def get_preprocessing_params(request):
try:
username = request.data.get("username")
user = User(username=username, password=None)
user = user.find()
store_obj = Store(user)
project_id = request.data.get("project_id")
if not store_obj.exist(project_id):
raise Exception("No such project exists")
project_dir = store_obj.path + os.sep + project_id
with open(project_dir + os.sep + "preprocessing.json", "r") as f:
preprocessing = json.load(f)
status, success, message = 200, True, "preprocessing params fetched"
except Exception as e:
status, success, message, preprocessing = 500, False, str(e), {}
return JsonResponse(
{
"success": success,
"message": message,
"preprocessing": preprocessing
},
status=status,
)
def save_preprocessing_params(request):
try:
username = request.data.get("username")
user = User(username=username, password=None)
user = user.find()
store_obj = Store(user)
project_id = request.data.get("project_id")
preprocessing = request.data.get("preprocessing_params")
if not store_obj.exist(project_id):
raise Exception("No such project exists")
project_dir = store_obj.path + os.sep + project_id
with open(project_dir + os.sep + "preprocessing.json", "w") as f:
json.dump(preprocessing, f)
status, success, message = 200, True, "Preprocessing params saved successfully"
except Exception as e:
status, success, message = 500, False, str(e)
return JsonResponse({
"success": success,
"message": message
},
status=status)
def generate(request):
training_params = request.data.get("training_params")
username = request.data.get("username")
user = User(username=username, password=None)
user = user.find()
project_id = request.data.get("project_id")
store_obj = Store(user)
if not store_obj.exist(project_id):
raise Exception("No such project exists")
project_dir = store_obj.path + os.sep + project_id
with open(project_dir + os.sep + "meta.json", "r") as f:
metadata = json.load(f)
with open(project_dir + os.sep + "preprocessing.json", "r") as f:
preprocessing = json.load(f)
lib = metadata.get("lib", "keras").lower()
lang = metadata.get("lang", "python").lower()
meta_params = {}
meta_params["lib"] = lib
meta_params["lang"] = lang
meta_params["dataset-path"] = metadata.get("data_dir", ".")
meta_params["save_plots"] = training_params["plot"]
meta_params.update(preprocessing)
with open(project_dir + os.sep + "layers.json", "r") as f:
layers = json.load(f)
with open(project_dir + os.sep + "components.json", "r") as f:
components = json.load(f)
training_params.update(meta_params)
training_params.update(layers)
inputs = json_to_dict.MakeDict(training_params).parse()
parser_path = "dlmml.parser." + lang + "." + lib + ".main"
parser = importlib.import_module(parser_path)
# TODO: move to javascript in the next version
user_data = UserData(metadata, components, layers, inputs, preprocessing)
status, error = parser.generate_code(inputs)
if status:
print("Error", error)
msg = "Could not generate code"
path = ""
else:
print("File generated")
msg = "File Generated Successfully"
path = "file:///" + os.getcwd() + os.sep + "test.py"
format_code("test.py")
copy_file(project_dir)
return JsonResponse({"message": msg, "path": path})
def edit_project(request):
try:
username = request.data.get("username")
user = User(username=username, password=None)
user = user.find()
project_id = request.data.get("project_id")
project_name = request.data.get("project_name")
project_description = request.data.get("project_description")
data_dir = request.data.get("data_dir")
output_file_name = request.data.get("output_file_name")
owner = request.data.get("owner")
store_obj = Store(user)
if owner and not owner == username:
project_id = "shared_" + project_id
if not store_obj.exist(project_id):
raise Exception("No such project exists")
project_dir = store_obj.path + os.sep + project_id
with open(project_dir + os.sep + "meta.json", "r") as f:
metadata = json.load(f)
metadata["project_name"] = (
project_name if project_name is not None else metadata["project_name"]
)
metadata["project_description"] = (
project_description
if project_description is not None
else metadata["project_description"]
)
metadata["data_dir"] = (
data_dir if data_dir is not None else metadata["data_dir"]
)
metadata["output_file_name"] = (
output_file_name
if output_file_name is not None
else metadata["output_file_name"]
)
with open(project_dir + os.sep + "meta.json", "w") as f:
json.dump(metadata, f)
status, success, message = 200, True, "Project Updated Successfully"
except Exception as e:
status, success, message = 500, False, "Could not update the Project"
return JsonResponse({"success": success, "message": message}, status=status)
def create_project(request):
try:
username = request.data.get("username")
user = User(username=username, password=None)
user = user.find()
project_id = generate_uid()
project_name = request.data.get("project_name")
lang = request.data.get("language")
lib = request.data.get("library")
task = request.data.get("task")
data_dir = request.data.get("path")
output_file_name = request.data.get("output_file_name")
store_obj = Store(user)
if store_obj.exist(project_id):
raise Exception("Project Already Exists!")
project_dir = store_obj.create(project_id)
metadata = {
"project_id": project_id,
"project_name": project_name,
"lib": lib,
"lang": lang,
"task": task,
"data_dir": data_dir,
"output_file_name": output_file_name,
"username": username,
}
with open(project_dir + os.sep + "meta.json", "w") as f:
json.dump(metadata, f)
status, success, message = 200, True, "Project Created Successfully"
except Exception as e:
status, success, message = 500, False, str(e)
return JsonResponse({
"success": success,
"message": message
},
status=status)
def clone_project(request):
try:
username = request.data.get("username")
model_layers = request.data.get("model_layers")
preprocessing_parameters = request.data.get("preprocessing_parameters")
hyper_parameters = request.data.get("hyper_parameters")
user = User(username=username, password=None)
user = user.find()
project_id = request.data.get("project_id")
clone_id = generate_uid()
project_name = request.data.get("project_name")
project_description = request.data.get("project_description")
lang = request.data.get("language")
lib = request.data.get("library")
task = request.data.get("task")
data_dir = request.data.get("path")
output_file_name = request.data.get("output_file_name")
store_obj = Store(user)
if store_obj.exist(clone_id):
raise Exception("Project Already Exists!")
clone_dir = store_obj.create(clone_id)
metadata = {
"project_id": clone_id,
"project_name": project_name,
"project_description": project_description,
"lib": lib,
"lang": lang,
"task": task,
"data_dir": data_dir,
"output_file_name": output_file_name,
"username": username,
}
with open(clone_dir + os.sep + "meta.json", "w") as f:
json.dump(metadata, f)
project_dir = store_obj.path + os.sep + project_id
project_json = project_dir + os.sep + "layers.json"
if model_layers and os.path.isfile(project_json):
with open(model_layers) as f:
layers = json.load(f)
with open(clone_dir + os.sep + "layers.json", "w") as f:
json.dump(layers, f)
with open(project_dir + os.sep + "components.json", "r") as f:
components = json.load(f)
with open(clone_dir + os.sep + "components.json", "w") as f:
json.dump(components, f)
project_preprocessing = project_dir + os.sep + "preprocessing.json"
if preprocessing_parameters and os.path.isfile(project_preprocessing):
with open(project_preprocessing, "r") as f:
preprocessing = json.load(f)
with open(clone_dir + os.sep + "preprocessing.json", "w") as f:
json.dump(preprocessing, f)
project_hyperparameters = project_dir + os.sep + "hyperparams.json"
if hyper_parameters and os.path.isfile(project_hyperparameters):
with open(project_hyperparameters, "r") as f:
hyperparams = json.load(f)
with open(clone_dir + os.sep + "hyperparams.json", "w") as f:
json.dump(hyperparams, f)
status, success, message = 200, True, "Project Cloned Successfully"
except Exception as e:
status, success, message = 500, False, "Project could not be cloned"
return JsonResponse({"success": success, "message": message}, status=status)
def cloud_deploy(request):
"""
Endpoint for deploying user model to user-preferred cloud environment (AWS / GCP).
Inputs:
-------
request:
Requires: username, project_id, model_categories & pickle file details in request data
token in request header
Returns:
--------
response: JsonResponse for Cloud Deployment describing Success or Failure
"""
username = request.data.get("username")
user = User(username=username, password=None)
user = user.find()
store_obj = Store(user)
try:
project_id = request.data.get("project_id")
if not store_obj.exist(project_id):
raise Exception("No such project exists")
project_dir = store_obj.path + os.sep + project_id
# Accessing pickle uploaded in chunks
current_chunk = request.data.get("current_chunk")
total_chunks = request.data.get("total_chunks")
pkl_file_bytes = request.data.get("pkl_file_bytes")
pkl_file_content = bytearray(pkl_file_bytes.values())
pkl_dir = os.path.join(project_dir, "pickle")
pkl_path = os.path.join(pkl_dir, f"model.pkl")
# Appending pickle chunks
append_pkl_contents(pkl_dir, pkl_path, pkl_file_content)
if current_chunk == total_chunks:
logger.info("Pickle file uploaded successfully")
model_categories = request.data.get("model_categories")
deployment_dir = os.path.join(project_dir, "deployment")
remove_dir(deployment_dir)
deployment = Deployment(project_dir, deployment_dir,
model_categories)
deployment.clone_flask_app()
logger.debug("Cloned Flask App to be deployed")
deployment.copy_pkl_file(pkl_path)
remove_dir(pkl_dir)
deployment.edit_flask_app()
logger.debug("Modified Flask App to be deployed")
status, success, message = 200, True, "Cloud Deployment Successful"
else:
status, success, message = 204, True, "Cloud Deployment Underway"
except ProjectNotFound:
logger.error("Unable to find the Project")
raise
except PickleAppendFailed:
logger.error("Unable to append Pickle file contents")
remove_dir(pkl_dir)
raise
except CloneGenerationFailed:
logger.error("Unable to clone the Flask App")
remove_dir(pkl_dir)
raise
except PickleCopyFailed:
logger.error("Unable to copy pickle file into deployment directory")
remove_dir(deployment_dir)
remove_dir(pkl_dir)
raise
except AppUpsertionFailed:
logger.error("Unable to modify the Flask App")
remove_dir(deployment_dir)
remove_dir(pkl_dir)
raise
else:
logger.info("Cloud Deployment Successful")
return JsonResponse({
"success": success,
"message": message
},
status=status)
def local_deploy(request):
"""
Endpoint for locally deploying user model.
Inputs:
-------
request:
Requires: username, project_id, model_categories & model_download_type in request data
token in request header
Returns:
--------
response: HttpResponse (Required Download)
(Yes, not JsonResponse like other views)
"""
username = request.data.get("username")
user = User(username=username, password=None)
user = user.find()
store_obj = Store(user)
try:
project_id = request.data.get("project_id")
if not store_obj.exist(project_id):
raise ProjectNotFound(project_id)
project_dir = store_obj.path + os.sep + project_id
deployment_dir = os.path.join(project_dir, "deployment")
remove_dir(deployment_dir)
model_categories = request.data.get("model_categories")
model_download_type = request.data.get("model_download_type")
deployment = Deployment(project_dir, deployment_dir, model_categories,
model_download_type)
deployment.clone_flask_app()
logger.debug("Cloned Flask App to be deployed")
deployment.edit_flask_app()
logger.debug("Modified Flask App to be deployed")
response = deployment.download_app()
except ProjectNotFound:
logger.error("Unable to find the Project")
raise
except CloneGenerationFailed:
logger.error("Unable to clone the Flask App")
raise
except AppUpsertionFailed:
logger.error("Unable to modify the Flask App")
raise
except AppDownloadFailed:
logger.error("Unable to download the Flask App")
raise
else:
logger.info("Local Deployment successful")
return response
