def __init__(self, modelpackagepath): modelInfo = ModelInfo(json.loads(open(absPath(os.path.join(modelpackagepath, "ModelInfo.json"))).read())) loggerInstance.logInsights(f"Model type is {modelInfo.modelType}") if modelInfo.modelType == "TfIdfSearchModel": self.model = TfIdfSearchModel(modelpackagepath) elif modelInfo.modelType == "WmdSearchModel": self.model = WmdSearchModel(modelpackagepath)
def moveModels(productid, path=""): try: copyFolder(os.path.join(modelsPath, productid), absPath(os.path.join(path, productid))) loggerInstance.logInsights( "copyModelToFolder: Copied Folder for product {0} to {1}".format( productid, absPath(os.path.join(path, productid)))) except Exception as e: raise ModelDownloadFailed("Model can't be downloaded " + str(e))
def verifyFile(filename, absolute=False, prelogMessage=""): fName = absPath(filename) if not absolute else filename try: with open(fName, "rb") as fp: fp.close() loggerInstance.logInsights("{0}Verified File {1}".format( prelogMessage, fName)) return True except FileNotFoundError: loggerInstance.logInsights("{0}Failed to Verify File {1}".format( prelogMessage, fName)) return False
def activate_job(): if app.config['MODEL_SYNC_ENABLED']: productIds = getAllProductIds(resourceConfig) sah = StorageAccountHelper(loggerInstance) loggerInstance.logInsights("Starting model sync for {0}".format( ','.join(productIds))) thread = threading.Thread(target=sah.watchModels, args=(productIds, )) thread.start() while True: modelDownloadPending = [ sah.firstTime[productId] if productId in sah.firstTime else True for productId in productIds ] if any(modelDownloadPending): time.sleep(2) else: break loggerInstance.logInsights("Search service startup succeeded")
def refreshModel(productId): prelogMessage = refreshModelMessage.format(productId) modelpackagepath = os.path.join(modelsPath, productId) try: # Try to load the given model temp = TextSearchModel(modelpackagepath) del temp if os.path.isdir(os.path.join("SearchModule", productId)): shutil.rmtree(os.path.join("SearchModule", productId)) copyFolder(modelpackagepath, os.path.join("SearchModule", productId)) #shutil.rmtree(absPath(path)) loggerInstance.logInsights(f"{prelogMessage}Verified the new model by loading it. Triggering the switch.") loadModel(productId, model=TextSearchModel(os.path.join("SearchModule", productId))) loggerInstance.logInsights(f"{prelogMessage}Successfully refreshed model.") return "Model Refreshed Successfully" except Exception as e: #shutil.rmtree(absPath(path)) loggerInstance.logHandledException("modelRefreshTask", ModelRefreshException(f"{prelogMessage}{str(e)}")) return "Failed to refresh Model Exception:" + str(e)
def loadModel(productId, model=None): prelogMessage = loadModelMessage.format(productId) if model: loggerInstance.logInsights(f"{prelogMessage}From provided pre-loaded model.") loaded_models[productId] = model return if productId in loaded_models: loggerInstance.logInsights(f"{prelogMessage}Model is already loaded in app") return modelpackagepath = os.path.join("SearchModule", productId) loggerInstance.logInsights(f"{prelogMessage}Loading from folder {modelpackagepath}") if not os.path.isdir(absPath(modelpackagepath)): loggerInstance.logInsights(f"{prelogMessage}Could not find model folder. Getting model from download location.") moveModels(productId, "SearchModule") loaded_models[productId] = TextSearchModel(modelpackagepath)