def init_models():
# In Kube mode create models when scheduler is launched which is always the first container.
local_models_path = "../configs/custom_defaults/trained_models.json"
if 'INIT_MODELS' in os.environ and os.environ['INIT_MODELS'].strip():
remote_models_path = os.environ['INIT_MODELS']
if not remote_models_path.startswith(
'/root/DVA/configs/custom_defaults/'):
local_models_path = 'custom_models.json'
get_path_to_file(remote_models_path, local_models_path)
else:
local_models_path = remote_models_path
default_models = json.loads(file(local_models_path).read())
if settings.KUBE_MODE and 'LAUNCH_SCHEDULER' in os.environ:
init_event = TEvent.objects.create(operation="perform_init",
duration=0,
started=True,
completed=True,
start_ts=timezone.now())
for m in default_models:
create_model(m, init_event)
elif not settings.KUBE_MODE:
init_event = TEvent.objects.create(operation="perform_init",
duration=0,
started=True,
completed=True,
start_ts=timezone.now())
for m in default_models:
create_model(m, init_event)
def init_process():
if 'INIT_PROCESS' in os.environ:
path = os.environ.get('INIT_PROCESS', None)
if path and path.strip():
if not path.startswith('/root/DVA/configs/custom_defaults/'):
get_path_to_file(path, "temp.json")
path = 'temp.json'
try:
jspec = json.load(file(path))
except:
logging.exception("could not load : {}".format(path))
else:
p = DVAPQLProcess()
if DVAPQL.objects.count() == 0:
p.create_from_json(jspec)
p.launch()
detector_type=m.get("detector_type", ""),
arguments=m.get("arguments", {}),
model_type=TrainedModel.DETECTOR,
)
if created:
dm.download()
else:
dm, created = TrainedModel.objects.get_or_create(
name=m['name'],
mode=m.get('mode', TrainedModel.TENSORFLOW),
files=m.get('files', []),
algorithm=m.get('algorithm', ""),
arguments=m.get("arguments", {}),
shasum=m.get('shasum', None),
model_type=m['model_type'])
if created:
dm.download()
if 'INIT_PROCESS' in os.environ and DVAPQL.objects.count() == 0:
path = os.environ.get('INIT_PROCESS')
p = DVAPQLProcess()
if not path.startswith('/root/DVA/configs/custom_defaults/'):
get_path_to_file(path, "temp.json")
path = 'temp.json'
try:
jspec = json.load(file(path))
except:
logging.exception("could not load : {}".format(path))
else:
p.create_from_json(jspec)
p.launch()
os.mkdir("{}/{}".format(settings.MEDIA_ROOT, create_dirname))
except:
pass
if ExternalServer.objects.count() == 0:
for e in json.loads(
file("../configs/custom_defaults/external.json").read()):
de, _ = ExternalServer.objects.get_or_create(name=e['name'],
url=e['url'])
de.pull()
local_models_path = "../configs/custom_defaults/trained_models.json"
if 'INIT_MODELS' in os.environ and os.environ['INIT_MODELS'].strip():
remote_models_path = os.environ['INIT_MODELS']
if not remote_models_path.startswith(
'/root/DVA/configs/custom_defaults/'):
local_models_path = 'custom_models.json'
get_path_to_file(remote_models_path, local_models_path)
else:
local_models_path = remote_models_path
default_models = json.loads(file(local_models_path).read())
for m in default_models:
if m['model_type'] == TrainedModel.DETECTOR:
dm, created = TrainedModel.objects.get_or_create(
name=m['name'],
algorithm=m['algorithm'],
mode=m['mode'],
files=m.get('files', []),
model_filename=m.get("filename", ""),
detector_type=m.get("detector_type", ""),
arguments=m.get("arguments", {}),
model_type=TrainedModel.DETECTOR,
)
