def create_output_model(model,
saved_path,
framework,
task,
singlefile=False,
model_name=None):
if task is None:
return saved_path
try:
WeightsFileHandler._model_store_lookup_lock.acquire()
# check if object already has InputModel
trains_out_model, ref_model = WeightsFileHandler._model_out_store_lookup.get(
id(model), (None, None))
# notice ref_model() is not an error/typo this is a weakref object call
if ref_model is not None and model != ref_model():
# old id pop it - it was probably reused because the object is dead
WeightsFileHandler._model_out_store_lookup.pop(id(model))
trains_out_model, ref_model = None, None
if not saved_path:
get_logger(TrainsFrameworkAdapter).warning(
"Could retrieve model location, skipping auto model logging"
)
return saved_path
# check if we have output storage, and generate list of files to upload
if Path(saved_path).is_dir():
files = [
str(f) for f in Path(saved_path).rglob('*') if f.is_file()
]
elif singlefile:
files = [str(Path(saved_path).absolute())]
else:
files = [
str(f) for f in Path(saved_path).parent.glob(
str(Path(saved_path).name) + '.*')
]
target_filename = None
if len(files) > 1:
# noinspection PyBroadException
try:
target_filename = Path(saved_path).stem
except Exception:
pass
else:
target_filename = files[0]
# check if object already has InputModel
if trains_out_model is None:
# if we are overwriting a local file, try to load registered model
# if there is an output_uri, then by definition we will not overwrite previously stored models.
if not task.output_uri:
try:
in_model_id = InputModel.load_model(
weights_url=saved_path)
if in_model_id:
in_model_id = in_model_id.id
get_logger(TrainsFrameworkAdapter).info(
"Found existing registered model id={} [{}] reusing it."
.format(in_model_id, saved_path))
except:
in_model_id = None
else:
in_model_id = None
trains_out_model = OutputModel(
task=task,
# config_dict=config,
name=(task.name + ' - ' +
model_name) if model_name else None,
label_enumeration=task.get_labels_enumeration(),
framework=framework,
base_model_id=in_model_id)
# noinspection PyBroadException
try:
ref_model = weakref.ref(model)
except Exception:
ref_model = None
WeightsFileHandler._model_out_store_lookup[id(model)] = (
trains_out_model, ref_model)
# upload files if we found them, or just register the original path
if trains_out_model.upload_storage_uri:
if len(files) > 1:
trains_out_model.update_weights_package(
weights_filenames=files,
auto_delete_file=False,
target_filename=target_filename)
else:
# create a copy of the stored file,
# protect against someone deletes/renames the file before async upload finish is done
target_filename = Path(files[0]).name
# HACK: if pytorch-lightning is used, remove the temp '.part' file extension
if sys.modules.get('pytorch_lightning'
) and target_filename.lower().endswith(
'.part'):
target_filename = target_filename[:-len('.part')]
fd, temp_file = mkstemp(prefix='.trains.upload_model_',
suffix='.tmp')
os.close(fd)
shutil.copy(files[0], temp_file)
trains_out_model.update_weights(
weights_filename=temp_file,
auto_delete_file=True,
target_filename=target_filename)
else:
trains_out_model.update_weights(weights_filename=None,
register_uri=saved_path)
except Exception as ex:
get_logger(TrainsFrameworkAdapter).debug(str(ex))
finally:
WeightsFileHandler._model_store_lookup_lock.release()
return saved_path
def create_output_model(model,
saved_path,
framework,
task,
singlefile=False,
model_name=None):
# type: (Optional[Any], Optional[str], Optional[str], Optional[Task], bool, Optional[str]) -> str
if task is None:
return saved_path
try:
WeightsFileHandler._model_store_lookup_lock.acquire()
# # disable model reuse, let Model module try to find it for use
trains_out_model, ref_model = None, None # noqa: F841
# check if object already has InputModel
# trains_out_model, ref_model = WeightsFileHandler._model_out_store_lookup.get(
# id(model) if model is not None else None, (None, None))
# # notice ref_model() is not an error/typo this is a weakref object call
# # noinspection PyCallingNonCallable
# if ref_model is not None and model != ref_model():
# # old id pop it - it was probably reused because the object is dead
# WeightsFileHandler._model_out_store_lookup.pop(id(model))
# trains_out_model, ref_model = None, None
local_model_path = os.path.abspath(
saved_path) if saved_path else saved_path
model_info = WeightsFileHandler.ModelInfo(
model=trains_out_model,
upload_filename=None,
local_model_path=local_model_path,
local_model_id=saved_path,
framework=framework,
task=task)
if not model_info.local_model_path:
# get_logger(TrainsFrameworkAdapter).debug(
# "Could not retrieve model location, skipping auto model logging")
return saved_path
# check if we have output storage, and generate list of files to upload
if Path(model_info.local_model_path).is_dir():
files = [
str(f)
for f in Path(model_info.local_model_path).rglob('*')
if f.is_file()
]
elif singlefile:
files = [str(Path(model_info.local_model_path).absolute())]
else:
files = [
str(f)
for f in Path(model_info.local_model_path).parent.glob(
str(Path(model_info.local_model_path).name) + '.*')
]
target_filename = None
if len(files) > 1:
# noinspection PyBroadException
try:
target_filename = Path(model_info.local_model_path).stem
except Exception:
pass
else:
target_filename = Path(files[0]).name
# call pre model callback functions
model_info.upload_filename = target_filename
for cb in WeightsFileHandler._model_pre_callbacks.values():
# noinspection PyBroadException
try:
model_info = cb(WeightsFileHandler.CallbackType.save,
model_info)
except Exception:
pass
# if callbacks force us to leave they return None
if model_info is None:
# callback forced quit
return saved_path
# update the trains_out_model after the pre callbacks
trains_out_model = model_info.model
# check if object already has InputModel
if trains_out_model is None:
# noinspection PyProtectedMember
in_model_id, model_uri = Model._local_model_to_id_uri.get(
model_info.local_model_id or model_info.local_model_path,
(None, None))
if not in_model_id:
# if we are overwriting a local file, try to load registered model
# if there is an output_uri, then by definition we will not overwrite previously stored models.
if not task.output_uri:
# noinspection PyBroadException
try:
in_model_id = InputModel.load_model(
weights_url=model_info.local_model_path)
if in_model_id:
in_model_id = in_model_id.id
get_logger(TrainsFrameworkAdapter).info(
"Found existing registered model id={} [{}] reusing it."
.format(in_model_id,
model_info.local_model_path))
except Exception:
in_model_id = None
else:
in_model_id = None
trains_out_model = OutputModel(
task=task,
# config_dict=config,
name=(task.name + ' - ' +
model_name) if model_name else None,
label_enumeration=task.get_labels_enumeration(),
framework=framework,
base_model_id=in_model_id)
# # disable model reuse, let Model module try to find it for use
# if model is not None:
# # noinspection PyBroadException
# try:
# ref_model = weakref.ref(model)
# except Exception:
# ref_model = None
# WeightsFileHandler._model_out_store_lookup[id(model)] = (trains_out_model, ref_model)
model_info.model = trains_out_model
# call post model callback functions
for cb in WeightsFileHandler._model_post_callbacks.values():
# noinspection PyBroadException
try:
model_info = cb(WeightsFileHandler.CallbackType.save,
model_info)
except Exception:
pass
trains_out_model = model_info.model
target_filename = model_info.upload_filename
# upload files if we found them, or just register the original path
if trains_out_model.upload_storage_uri:
if len(files) > 1:
trains_out_model.update_weights_package(
weights_filenames=files,
auto_delete_file=False,
target_filename=target_filename)
else:
# create a copy of the stored file,
# protect against someone deletes/renames the file before async upload finish is done
# HACK: if pytorch-lightning is used, remove the temp '.part' file extension
if sys.modules.get('pytorch_lightning'
) and target_filename.lower().endswith(
'.part'):
target_filename = target_filename[:-len('.part')]
fd, temp_file = mkstemp(prefix='.trains.upload_model_',
suffix='.tmp')
os.close(fd)
shutil.copy(files[0], temp_file)
trains_out_model.update_weights(
weights_filename=temp_file,
auto_delete_file=True,
target_filename=target_filename,
update_comment=False)
else:
trains_out_model.update_weights(
weights_filename=None,
register_uri=model_info.local_model_path)
# update back the internal Model lookup, and replace the local file with our file
# noinspection PyProtectedMember
Model._local_model_to_id_uri[model_info.local_model_id] = (
trains_out_model.id, trains_out_model.url)
except Exception as ex:
get_logger(TrainsFrameworkAdapter).debug(str(ex))
finally:
WeightsFileHandler._model_store_lookup_lock.release()
return saved_path
elapsed = time.time() - start
draw = ImageDraw.Draw(image)
font = ImageFont.truetype('/Library/Fonts/Arial.ttf', 55)
print('elapsed time:', elapsed, ' s')
print(path, tf_colors[color])
# output path either base dir or subdir for color
# output_path = output_base_path + filename
output_path = output_base_path + tf_colors[color] + '/' + filename
# draw text and write file in main output dir
draw.text(
(10, 10),
tf_colors[color],
font=font,
fill=(255, 255, 0),
)
copyfile(root + '/' + filename, output_path)
image.save(output_path)
if path.lower().find(tf_colors[color]) == -1:
cnt_error += 1
else:
cnt_ok += 1
# tmp
# if cnt_error + cnt_ok > 5:
# break
idx += 1
# if cnt_error + cnt_ok > 5:
# break
print('Succes rate:', 100 * cnt_ok / (cnt_ok + cnt_error), ' %')
