def create_output_model(model, saved_path, framework, task, singlefile=False, model_name=None): if task is None: return saved_path try: WeightsFileHandler._model_store_lookup_lock.acquire() # check if object already has InputModel trains_out_model, ref_model = WeightsFileHandler._model_out_store_lookup.get( id(model), (None, None)) # notice ref_model() is not an error/typo this is a weakref object call if ref_model is not None and model != ref_model(): # old id pop it - it was probably reused because the object is dead WeightsFileHandler._model_out_store_lookup.pop(id(model)) trains_out_model, ref_model = None, None if not saved_path: get_logger(TrainsFrameworkAdapter).warning( "Could retrieve model location, skipping auto model logging" ) return saved_path # check if we have output storage, and generate list of files to upload if Path(saved_path).is_dir(): files = [ str(f) for f in Path(saved_path).rglob('*') if f.is_file() ] elif singlefile: files = [str(Path(saved_path).absolute())] else: files = [ str(f) for f in Path(saved_path).parent.glob( str(Path(saved_path).name) + '.*') ] target_filename = None if len(files) > 1: # noinspection PyBroadException try: target_filename = Path(saved_path).stem except Exception: pass else: target_filename = files[0] # check if object already has InputModel if trains_out_model is None: # if we are overwriting a local file, try to load registered model # if there is an output_uri, then by definition we will not overwrite previously stored models. if not task.output_uri: try: in_model_id = InputModel.load_model( weights_url=saved_path) if in_model_id: in_model_id = in_model_id.id get_logger(TrainsFrameworkAdapter).info( "Found existing registered model id={} [{}] reusing it." .format(in_model_id, saved_path)) except: in_model_id = None else: in_model_id = None trains_out_model = OutputModel( task=task, # config_dict=config, name=(task.name + ' - ' + model_name) if model_name else None, label_enumeration=task.get_labels_enumeration(), framework=framework, base_model_id=in_model_id) # noinspection PyBroadException try: ref_model = weakref.ref(model) except Exception: ref_model = None WeightsFileHandler._model_out_store_lookup[id(model)] = ( trains_out_model, ref_model) # upload files if we found them, or just register the original path if trains_out_model.upload_storage_uri: if len(files) > 1: trains_out_model.update_weights_package( weights_filenames=files, auto_delete_file=False, target_filename=target_filename) else: # create a copy of the stored file, # protect against someone deletes/renames the file before async upload finish is done target_filename = Path(files[0]).name # HACK: if pytorch-lightning is used, remove the temp '.part' file extension if sys.modules.get('pytorch_lightning' ) and target_filename.lower().endswith( '.part'): target_filename = target_filename[:-len('.part')] fd, temp_file = mkstemp(prefix='.trains.upload_model_', suffix='.tmp') os.close(fd) shutil.copy(files[0], temp_file) trains_out_model.update_weights( weights_filename=temp_file, auto_delete_file=True, target_filename=target_filename) else: trains_out_model.update_weights(weights_filename=None, register_uri=saved_path) except Exception as ex: get_logger(TrainsFrameworkAdapter).debug(str(ex)) finally: WeightsFileHandler._model_store_lookup_lock.release() return saved_path
def create_output_model(model, saved_path, framework, task, singlefile=False, model_name=None): # type: (Optional[Any], Optional[str], Optional[str], Optional[Task], bool, Optional[str]) -> str if task is None: return saved_path try: WeightsFileHandler._model_store_lookup_lock.acquire() # # disable model reuse, let Model module try to find it for use trains_out_model, ref_model = None, None # noqa: F841 # check if object already has InputModel # trains_out_model, ref_model = WeightsFileHandler._model_out_store_lookup.get( # id(model) if model is not None else None, (None, None)) # # notice ref_model() is not an error/typo this is a weakref object call # # noinspection PyCallingNonCallable # if ref_model is not None and model != ref_model(): # # old id pop it - it was probably reused because the object is dead # WeightsFileHandler._model_out_store_lookup.pop(id(model)) # trains_out_model, ref_model = None, None local_model_path = os.path.abspath( saved_path) if saved_path else saved_path model_info = WeightsFileHandler.ModelInfo( model=trains_out_model, upload_filename=None, local_model_path=local_model_path, local_model_id=saved_path, framework=framework, task=task) if not model_info.local_model_path: # get_logger(TrainsFrameworkAdapter).debug( # "Could not retrieve model location, skipping auto model logging") return saved_path # check if we have output storage, and generate list of files to upload if Path(model_info.local_model_path).is_dir(): files = [ str(f) for f in Path(model_info.local_model_path).rglob('*') if f.is_file() ] elif singlefile: files = [str(Path(model_info.local_model_path).absolute())] else: files = [ str(f) for f in Path(model_info.local_model_path).parent.glob( str(Path(model_info.local_model_path).name) + '.*') ] target_filename = None if len(files) > 1: # noinspection PyBroadException try: target_filename = Path(model_info.local_model_path).stem except Exception: pass else: target_filename = Path(files[0]).name # call pre model callback functions model_info.upload_filename = target_filename for cb in WeightsFileHandler._model_pre_callbacks.values(): # noinspection PyBroadException try: model_info = cb(WeightsFileHandler.CallbackType.save, model_info) except Exception: pass # if callbacks force us to leave they return None if model_info is None: # callback forced quit return saved_path # update the trains_out_model after the pre callbacks trains_out_model = model_info.model # check if object already has InputModel if trains_out_model is None: # noinspection PyProtectedMember in_model_id, model_uri = Model._local_model_to_id_uri.get( model_info.local_model_id or model_info.local_model_path, (None, None)) if not in_model_id: # if we are overwriting a local file, try to load registered model # if there is an output_uri, then by definition we will not overwrite previously stored models. if not task.output_uri: # noinspection PyBroadException try: in_model_id = InputModel.load_model( weights_url=model_info.local_model_path) if in_model_id: in_model_id = in_model_id.id get_logger(TrainsFrameworkAdapter).info( "Found existing registered model id={} [{}] reusing it." .format(in_model_id, model_info.local_model_path)) except Exception: in_model_id = None else: in_model_id = None trains_out_model = OutputModel( task=task, # config_dict=config, name=(task.name + ' - ' + model_name) if model_name else None, label_enumeration=task.get_labels_enumeration(), framework=framework, base_model_id=in_model_id) # # disable model reuse, let Model module try to find it for use # if model is not None: # # noinspection PyBroadException # try: # ref_model = weakref.ref(model) # except Exception: # ref_model = None # WeightsFileHandler._model_out_store_lookup[id(model)] = (trains_out_model, ref_model) model_info.model = trains_out_model # call post model callback functions for cb in WeightsFileHandler._model_post_callbacks.values(): # noinspection PyBroadException try: model_info = cb(WeightsFileHandler.CallbackType.save, model_info) except Exception: pass trains_out_model = model_info.model target_filename = model_info.upload_filename # upload files if we found them, or just register the original path if trains_out_model.upload_storage_uri: if len(files) > 1: trains_out_model.update_weights_package( weights_filenames=files, auto_delete_file=False, target_filename=target_filename) else: # create a copy of the stored file, # protect against someone deletes/renames the file before async upload finish is done # HACK: if pytorch-lightning is used, remove the temp '.part' file extension if sys.modules.get('pytorch_lightning' ) and target_filename.lower().endswith( '.part'): target_filename = target_filename[:-len('.part')] fd, temp_file = mkstemp(prefix='.trains.upload_model_', suffix='.tmp') os.close(fd) shutil.copy(files[0], temp_file) trains_out_model.update_weights( weights_filename=temp_file, auto_delete_file=True, target_filename=target_filename, update_comment=False) else: trains_out_model.update_weights( weights_filename=None, register_uri=model_info.local_model_path) # update back the internal Model lookup, and replace the local file with our file # noinspection PyProtectedMember Model._local_model_to_id_uri[model_info.local_model_id] = ( trains_out_model.id, trains_out_model.url) except Exception as ex: get_logger(TrainsFrameworkAdapter).debug(str(ex)) finally: WeightsFileHandler._model_store_lookup_lock.release() return saved_path
elapsed = time.time() - start draw = ImageDraw.Draw(image) font = ImageFont.truetype('/Library/Fonts/Arial.ttf', 55) print('elapsed time:', elapsed, ' s') print(path, tf_colors[color]) # output path either base dir or subdir for color # output_path = output_base_path + filename output_path = output_base_path + tf_colors[color] + '/' + filename # draw text and write file in main output dir draw.text( (10, 10), tf_colors[color], font=font, fill=(255, 255, 0), ) copyfile(root + '/' + filename, output_path) image.save(output_path) if path.lower().find(tf_colors[color]) == -1: cnt_error += 1 else: cnt_ok += 1 # tmp # if cnt_error + cnt_ok > 5: # break idx += 1 # if cnt_error + cnt_ok > 5: # break print('Succes rate:', 100 * cnt_ok / (cnt_ok + cnt_error), ' %')