def get_model(): meta = request.json if not valid_credentials(meta["api_key"]): return {"message": "invalid access code"}, 401 # todo: read and write access only to models dir model = f"{MODELS_DIR}/{meta['model_type']}/{meta['timestamp']}.model" if not os.path.isfile(model): return {"message": "invalid model - not found"}, 500 username = get_username(meta["api_key"]) log(f"get_model accessed by {username} - {meta['model_type']} - {meta['timestamp']}") return send_file(model, mimetype='application/octet-stream'), 200
def log_message(): """ Log a message from a user available data fields: api_key message """ meta = request.json if not valid_credentials(meta["api_key"]): return {"message": "invalid access code"}, 401 username = get_username(meta["api_key"]) message = meta["message"] log(f"Log message from {username} - {message}", True) return {'message': 'ok'}, 200
def evaluate_model(): meta = request.json if not valid_credentials(meta["api_key"]): return {"message": "invalid access code"}, 401 username = get_username(meta["api_key"]) log(f"evaluate_model accessed by {username} - {meta['model_type']} - {meta['timestamp']}") model = f"{MODELS_DIR}/{meta['model_type']}/{meta['timestamp']}.model" if not os.path.isfile(model): return {"message": "invalid model - not found"}, 500 eval_thread = Thread(target=evaluate_model_thread, args=(meta["model_type"], model), daemon=False) eval_thread.start() return {"message": "starting evaluation successful"}, 200
def upload_data(): """ Upload user data and save them to db/uploaded_data/<username>/<timestamp>.npz available data fields: api_key """ meta = request.form.to_dict() if not valid_credentials(meta["api_key"]): log(f"Upload request of {meta['model_type']} rejected because api key {meta['api_key']} is invalid") return {"message": "invalid access code"}, 401 username = get_username(meta["api_key"]) data_dir = f"{DB_DIR}/uploaded_data/{username}" if not os.path.exists(data_dir): os.makedirs(data_dir) request.files['data_binary'].save(f"{data_dir}/{int(time.time())}.npz") log(f"upload_data accessed by {username} - upload successful") return {"message": "upload successful"}, 200
def info_model(): meta = request.json if not valid_credentials(meta["api_key"]): return {"message": "invalid access code"}, 401 timestamps = get_models(meta["model_type"]) hashes = {} for stamp in timestamps: model_path = f"{MODELS_DIR}/{meta['model_type']}/{stamp}.model" model_hash = calculate_file_hash(model_path, True) hashes[stamp] = model_hash #print(timestamps) #print(hashes) out_dict = {"latest_timestamp": timestamps[-1], "hash": hashes[timestamps[-1]], "hashes": hashes, "timestamps": timestamps} json_file_path = f"{MODELS_DIR}/{meta['model_type']}/model.json" if os.path.exists(json_file_path): # add the content of the json file to the dictionary out_dict.update(json.load(open(json_file_path, "rb"))) return out_dict, 200
def upload_model(): """ available data fields: api_key model_type dice (optional) """ meta = request.form.to_dict() if not valid_credentials(meta["api_key"]): log(f"Upload request of {meta['model_type']} rejected because api key {meta['api_key']} is invalid") return {"message": "invalid access code"}, 401 username = get_username(meta["api_key"]) if meta["model_type"] not in get_model_types(): log(f"Upload request of {meta['model_type']} from {username} rejected because model type is invalid") return {"message": "invalid model type"}, 500 #model_binary = request.files['model_binary'].read() # read() is needed to get bytes from FileStorage object #model_delta = DynamicDLModel.Loads(model_binary) #model_orig_path = f"{MODELS_DIR}/{meta['model_type']}/{model_delta.timestamp_id}.model" #model_orig = DynamicDLModel.Load(open(model_orig_path, "rb")) # unused here - see merge_models # calc_delta(): new_model - orig_model = delta # apply_delta(): model_orig + delta = model_new # Model delta: only activate if rest of federated learning working properly # model = model_orig.apply_delta(model_delta) #model = model_delta # directly save received model to disk model_path = f"{MODELS_DIR}/{meta['model_type']}/uploads/{str(int(time.time()))}_{username}.model" request.files['model_binary'].save(model_path) dice = meta["dice"] if "dice" in meta else -1.0 log(f"upload_model accessed by {username} - {meta['model_type']} - {model_path} - client dice {dice}") try: original_hash = meta["hash"] except: original_hash = None local_hash = calculate_file_hash(model_path) if original_hash is not None and original_hash != local_hash: log("Error during model upload") return {"message": "Communication error during upload"}, 500 #model.dump(open(model_path, "wb")) print("Starting merge...") # Thread needed. With multiprocessing.Process this will block in docker+nginx # (daemon=True/False works both) # merge_thread = Thread(target=merge_model, args=(meta["model_type"], model_path), daemon=False) merge_thread = Thread(target=merge_model_thread, args=(meta["model_type"], model_path), daemon=False) merge_thread.start() merged_model = 1 if merged_model is not None: return {"message": "upload successful"}, 200 else: print("Info: merging of models failed, because validation Dice too low.") return {"message": "merging of models failed, because validation Dice too low."}, 500
def get_available_models(): meta = request.json if not valid_credentials(meta["api_key"]): return {"message": "invalid access code"}, 401 return {"models": get_model_types()}, 200