def load(uuid): if not isinstance(uuid, UUID): # TODO: not this uuid = UUID(uuid.replace("-", "")) path = os.path.join(current_app.config["UPLOAD_FOLDER"], f"{uuid.hex}.json") with open(path, "r") as f: data = json.load(f) return data
def prepareModelDownload(self, project, modelID, username, source='marketplace', modelName=None, modelDescription='', modelTags=[]): ''' Attempts to create a file from a model state, either from the database (if "modelID" is a UUID) or directly from one of the built-in configu- rations (if it is a str, corresponding to the built-in name). Constructs an AIDE Model Marketplace-compliant definition JSON file in the process, optionally wrapped together with a state dict binary file in a zip file, if needed. Saves the file to a temporary directory and returns the file path as a Celery result if successful. Various parameters like "modelName", "modelDescription", etc., are only needed if model state is pulled from the project's table and therefore does not automatically come with these metadata. ''' # try to parse modelID as UUID try: modelID = UUID(modelID) except: # need to load model from built-ins pass if isinstance(modelID, UUID): # load from database modelDefinition = { 'aide_model_version': MODEL_MARKETPLACE_VERSION, 'name': (modelName if modelName is not None else str(modelID)), 'description': modelDescription, 'tags': modelTags } if source.lower() == 'marketplace': # load from marketplace table queryStr = ''' SELECT name, description, author, timeCreated, tags, labelclasses, annotationType, predictionType, model_library, --TODO: more? stateDict FROM aide_admin.modelMarketplace WHERE id = %s; ''' elif source.lower() == 'project': # load from project table queryStr = sql.SQL(''' SELECT timeCreated, model_library, stateDict FROM {id_cnnstate} WHERE id = %s; ''').format(id_cnnstate=sql.Identifier(project, 'cnnstate')) result = self.dbConnector.execute(queryStr, (modelID, ), 1) if result is None or not len(result): raise Exception( f'Model state with id "{str(modelID)}" could not be found in database.' ) result = result[0] for key in result.keys(): if key == 'timecreated': modelDefinition['time_created'] = result[key].timestamp() elif key == 'labelclasses': modelDefinition[key] = json.loads(result[key]) elif key == 'tags': modelDefinition[key] = result[key].split(';;') elif key == 'model_library': modelDefinition['ai_model_library'] = result[key] elif key == 'annotationtype': modelDefinition['annotation_type'] = result[key] elif key == 'predictiontype': modelDefinition['prediction_type'] = result[key] elif key in modelDefinition: modelDefinition[key] = result[key] if 'author' not in modelDefinition: # append user name as author modelDefinition['author'] = username # get model implementation meta data modelImplementationID = modelDefinition['ai_model_library'] if modelImplementationID not in PREDICTION_MODELS: raise Exception( f'Model implementation with ID "{modelImplementationID}" is not installed in this instance of AIDE.' ) modelMeta = PREDICTION_MODELS[modelImplementationID] if 'annotation_type' not in modelDefinition: modelDefinition['annotation_type'] = modelMeta[ 'annotationType'] if 'prediction_type' not in modelDefinition: modelDefinition['prediction_type'] = modelMeta[ 'predictionType'] if 'labelclasses' not in modelDefinition: # query from current project and just append as a list queryStr = sql.SQL( 'SELECT name FROM {} ORDER BY name;').format( sql.Identifier(project, 'labelclass')) labelClasses = self.dbConnector.execute( queryStr, (project, ), 'all') labelClasses = [l['name'] for l in labelClasses] modelDefinition['labelclasses'] = labelClasses # model settings: grab from project if possible #TODO # state dict stateDict = result['statedict'] # prepare temporary output file destName = modelDefinition['name'] + '_' + current_time().strftime( '%Y-%m-%d_%H-%M-%S') for char in FILENAMES_PROHIBITED_CHARS: destName = destName.replace(char, '_') # write contents if stateDict is None: destName += '.json' json.dump(modelDefinition, open(os.path.join(self.tempDir, destName), 'w')) else: destName += '.zip' with zipfile.ZipFile(os.path.join(self.tempDir, destName), 'w', zipfile.ZIP_DEFLATED) as f: f.writestr('modelDefinition.json', json.dumps(modelDefinition)) bio = io.BytesIO(stateDict) f.writestr('modelState.bin', bio.getvalue()) return destName else: # built-in model; copy to temp dir and return path directly sourcePath = modelID.replace('aide://', '').strip('/') if not os.path.exists(sourcePath): raise Exception( f'Model file "{sourcePath}" could not be found.') _, fileName = os.path.split(sourcePath) destPath = os.path.join(self.tempDir, fileName) if not os.path.exists(destPath): shutil.copyfile(sourcePath, destPath) return destPath