def save(self, filename):
logger.info('saving')
res = []
for each in self.tasks:
res.append(each.toDict())
with open(filename, 'w') as f:
json.dump(res, f)
def get_model(self, model_id):
always_refresh = True
model_id = cherrypy.request.params.get('model_id')
print(model_id)
request_folder_name = model_id
train_path = os.path.join(self.BASE_FILE_PATH, request_folder_name)
if not os.path.exists(train_path):
raise cherrypy.HTTPError(404)
taf = ToArchiveFolder(train_path)
compressedFile = os.path.join(STATIC_FILE_PATH, model_id)
if not os.path.exists(compressedFile):
os.makedirs(compressedFile)
compressedFile = os.path.join(compressedFile, model_id + '.zip')
if os.path.exists(compressedFile) and not always_refresh:
logger.info('model exists,skipping')
result = {
'code':
'200',
'url':
'http://134.175.1.246:80/static/' + model_id + '/' + model_id +
'.zip'
}
return json.dumps(result)
else:
taf.zip(compressedFile)
result = {
'code':
'200',
'url':
'http://134.175.1.246:80/static/' + model_id + '/' + model_id +
'.zip'
}
return json.dumps(result)
def __init__(self, filepath):
if zipfile.is_zipfile(filepath):
self._af = zipfile.ZipFile(filepath, 'r')
self.filepath = filepath
logger.info("Successfully load file" + filepath)
else:
logger.error("Cannot load file" + filepath)
def _init_model_(self, model_path):
logger.info('initiating model')
detection_graph = tf.Graph()
with detection_graph.as_default():
od_graph_def = tf.GraphDef()
with tf.gfile.GFile(model_path, 'rb') as fid:
serialized_graph = fid.read()
od_graph_def.ParseFromString(serialized_graph)
tf.import_graph_def(od_graph_def, name='')
self.graph = detection_graph
logger.info('model initialized')
def zip(self, target):
if self.folderPath is None:
logger.error("Folder Path Not Specified")
return False
else:
abs_src = os.path.abspath(self.folderPath)
contents = os.walk(self.folderPath)
self._af = zipfile.ZipFile(target, 'w', zipfile.ZIP_DEFLATED)
for root, dirs, files in contents:
for file in files:
abs_name = os.path.abspath(os.path.join(root, file))
arcname = abs_name[len(abs_src) + 1:]
logger.info('adding file' + abs_name)
self._af.write(abs_name, arcname)
self._af.close()
def unzip(self, extractTo, deleteOrigin=False):
uncompress_size = sum((file.file_size for file in self._af.infolist()))
extracted_size = 0
pbar = tqdm(total=uncompress_size,
initial=extracted_size,
unit='B',
unit_scale=True,
desc="uncompressing " + self.filepath)
for file in self._af.infolist():
self._af.extract(file, extractTo)
extracted_size += file.file_size
pbar.update(extracted_size)
self._af.extractall(extractTo)
logger.info("Successfully unzip file")
if deleteOrigin:
os.remove(self.filepath)
logger.info("Successfully delete original file")
def start(self, notify_func=None, args=None):
if self.config is None:
logger.error('No Config Found')
return
train_pipeline_file = self.config['pipeline_config_file']
configs = self._get_configs_from_pipeline_file(train_pipeline_file)
model_config = configs['model']
train_config = configs['train_config']
input_config = configs['train_input_config']
logger.info('Building Model')
model_fn = functools.partial(model_builder.build,
model_config=model_config,
is_training=True)
logger.info('creating input dict')
create_input_dict_fn = functools.partial(self.get_next, input_config)
ps_tasks = 0
worker_replicas = 1
worker_job_name = 'obj_detection_trainer'
task = 0
is_chief = True
master = ''
num_clones = 1
clone_on_cpu = False
try:
logger.info('Training Started')
trainer_m.train(create_input_dict_fn, model_fn, train_config,
master, task, num_clones, worker_replicas,
clone_on_cpu, ps_tasks, worker_job_name, is_chief,
self.config, notify_func, args)
except:
logger.error('Cannot Start Training')
traceback.print_exc(file=sys.stdout)
def init(self, model_path):
if not self.hasInitialized:
self._init_model_(model_path)
else:
logger.info('model has initialized, skipping')
def init(self, model_name, model_path):
if not self.hasInitialized:
self._init_model_(model_name, model_path)
else:
logger.info('model has been initialized, skip')
