def __init__(self, modelpackagepath):
modelInfo = ModelInfo(json.loads(open(absPath(os.path.join(modelpackagepath, "ModelInfo.json"))).read()))
loggerInstance.logInsights(f"Model type is {modelInfo.modelType}")
if modelInfo.modelType == "TfIdfSearchModel":
self.model = TfIdfSearchModel(modelpackagepath)
elif modelInfo.modelType == "WmdSearchModel":
self.model = WmdSearchModel(modelpackagepath)
def moveModels(productid, path=""):
try:
copyFolder(os.path.join(modelsPath, productid),
absPath(os.path.join(path, productid)))
loggerInstance.logInsights(
"copyModelToFolder: Copied Folder for product {0} to {1}".format(
productid, absPath(os.path.join(path, productid))))
except Exception as e:
raise ModelDownloadFailed("Model can't be downloaded " + str(e))
def verifyFile(filename, absolute=False, prelogMessage=""):
fName = absPath(filename) if not absolute else filename
try:
with open(fName, "rb") as fp:
fp.close()
loggerInstance.logInsights("{0}Verified File {1}".format(
prelogMessage, fName))
return True
except FileNotFoundError:
loggerInstance.logInsights("{0}Failed to Verify File {1}".format(
prelogMessage, fName))
return False
def activate_job():
if app.config['MODEL_SYNC_ENABLED']:
productIds = getAllProductIds(resourceConfig)
sah = StorageAccountHelper(loggerInstance)
loggerInstance.logInsights("Starting model sync for {0}".format(
','.join(productIds)))
thread = threading.Thread(target=sah.watchModels, args=(productIds, ))
thread.start()
while True:
modelDownloadPending = [
sah.firstTime[productId]
if productId in sah.firstTime else True
for productId in productIds
]
if any(modelDownloadPending):
time.sleep(2)
else:
break
loggerInstance.logInsights("Search service startup succeeded")
def refreshModel(productId):
prelogMessage = refreshModelMessage.format(productId)
modelpackagepath = os.path.join(modelsPath, productId)
try:
# Try to load the given model
temp = TextSearchModel(modelpackagepath)
del temp
if os.path.isdir(os.path.join("SearchModule", productId)):
shutil.rmtree(os.path.join("SearchModule", productId))
copyFolder(modelpackagepath, os.path.join("SearchModule", productId))
#shutil.rmtree(absPath(path))
loggerInstance.logInsights(f"{prelogMessage}Verified the new model by loading it. Triggering the switch.")
loadModel(productId, model=TextSearchModel(os.path.join("SearchModule", productId)))
loggerInstance.logInsights(f"{prelogMessage}Successfully refreshed model.")
return "Model Refreshed Successfully"
except Exception as e:
#shutil.rmtree(absPath(path))
loggerInstance.logHandledException("modelRefreshTask", ModelRefreshException(f"{prelogMessage}{str(e)}"))
return "Failed to refresh Model Exception:" + str(e)
def loadModel(productId, model=None):
prelogMessage = loadModelMessage.format(productId)
if model:
loggerInstance.logInsights(f"{prelogMessage}From provided pre-loaded model.")
loaded_models[productId] = model
return
if productId in loaded_models:
loggerInstance.logInsights(f"{prelogMessage}Model is already loaded in app")
return
modelpackagepath = os.path.join("SearchModule", productId)
loggerInstance.logInsights(f"{prelogMessage}Loading from folder {modelpackagepath}")
if not os.path.isdir(absPath(modelpackagepath)):
loggerInstance.logInsights(f"{prelogMessage}Could not find model folder. Getting model from download location.")
moveModels(productId, "SearchModule")
loaded_models[productId] = TextSearchModel(modelpackagepath)