def testBadSavedModelFileFormat(self): export_dir = self._get_export_dir("test_bad_saved_model_file_format") # Attempt to load a SavedModel from an export directory that does not exist. with self.test_session(graph=ops.Graph()) as sess: with self.assertRaisesRegexp(IOError, "SavedModel file does not exist at: %s" % export_dir): loader.load(sess, ["foo"], export_dir) os.makedirs(export_dir) # Write an invalid binary proto to saved_model.pb. path_to_pb = os.path.join(export_dir, constants.SAVED_MODEL_FILENAME_PB) with open(path_to_pb, "w") as f: f.write("invalid content") with self.test_session(graph=ops.Graph()) as sess: with self.assertRaisesRegexp(IOError, "Cannot parse file.*%s" % constants.SAVED_MODEL_FILENAME_PB): loader.load(sess, ["foo"], export_dir) # Cleanup the directory and start again. file_io.delete_recursively(export_dir) os.makedirs(export_dir) # Write an invalid text proto to saved_model.pbtxt path_to_pbtxt = os.path.join(export_dir, constants.SAVED_MODEL_FILENAME_PBTXT) with open(path_to_pbtxt, "w") as f: f.write("invalid content") with self.test_session(graph=ops.Graph()) as sess: with self.assertRaisesRegexp(IOError, "Cannot parse file.*%s" % constants.SAVED_MODEL_FILENAME_PBTXT): loader.load(sess, ["foo"], export_dir)
def _delete_file_or_dir(pathname): if file_io.is_directory(pathname): file_io.delete_recursively(pathname) print(f'Deleted backup dir {pathname}') elif file_io.file_exists(pathname): file_io.delete_file(pathname) print(f'Deleted backup file {pathname}')
def create_object_test(): """Verifies file_io's object manipulation methods .""" starttime = int(round(time.time() * 1000)) dir_name = "%s/tf_gcs_test_%s" % (FLAGS.gcs_bucket_url, starttime) print("Creating dir %s." % dir_name) file_io.create_dir(dir_name) # Create a file in this directory. file_name = "%s/test_file.txt" % dir_name print("Creating file %s." % file_name) file_io.write_string_to_file(file_name, "test file creation.") list_files_pattern = "%s/test_file*.txt" % dir_name print("Getting files matching pattern %s." % list_files_pattern) files_list = file_io.get_matching_files(list_files_pattern) print(files_list) assert len(files_list) == 1 assert files_list[0] == file_name # Cleanup test files. print("Deleting file %s." % file_name) file_io.delete_file(file_name) # Delete directory. print("Deleting directory %s." % dir_name) file_io.delete_recursively(dir_name)
def create_dir_test(): """Verifies file_io directory handling methods .""" starttime = int(round(time.time() * 1000)) dir_name = "%s/tf_gcs_test_%s" % (FLAGS.gcs_bucket_url, starttime) print("Creating dir %s" % dir_name) file_io.create_dir(dir_name) elapsed = int(round(time.time() * 1000)) - starttime print("Created directory in: %d milliseconds" % elapsed) # Check that the directory exists. dir_exists = file_io.is_directory(dir_name) print("%s directory exists: %s" % (dir_name, dir_exists)) # List contents of just created directory. print("Listing directory %s." % dir_name) starttime = int(round(time.time() * 1000)) print(file_io.list_directory(dir_name)) elapsed = int(round(time.time() * 1000)) - starttime print("Listed directory %s in %s milliseconds" % (dir_name, elapsed)) # Delete directory. print("Deleting directory %s." % dir_name) starttime = int(round(time.time() * 1000)) file_io.delete_recursively(dir_name) elapsed = int(round(time.time() * 1000)) - starttime print("Deleted directory %s in %s milliseconds" % (dir_name, elapsed))
def save_inference_model(export_dir, inference_model, session=None, replace=True): if session is None: session = tf.get_default_session() assert session is not None if replace and is_directory(export_dir): get_logger().info('replacing %s', export_dir) delete_recursively(export_dir) prediction_signature = predict_signature_def( inputs={INPUTS_KEY: inference_model.inputs_tensor}, outputs={ k: v for k, v in { OUTPUTS_KEY: inference_model.outputs_tensor, LABELS_KEY: inference_model.labels_tensor, COLORS_KEY: inference_model.colors_tensor }.items() if v is not None }) signature_def_map = { DEFAULT_SERVING_SIGNATURE_DEF_KEY: prediction_signature } legacy_init_op = tf.group(tf.tables_initializer(), name='legacy_init_op') builder = SavedModelBuilder(export_dir) builder.add_meta_graph_and_variables(session, [SERVING], signature_def_map=signature_def_map, legacy_init_op=legacy_init_op) builder.save()
def _save_oncloud(model, export_path): tmpPath = './tmp_folder' ### Allow overwriting of export_path if it already exists by removing it first.. if file_io.file_exists(tmpPath): # print("Need to overwrite preexisting path. Recursively deleting... ", tmpPath) file_io.delete_recursively(tmpPath) builder = saved_model_builder.SavedModelBuilder(tmpPath) signature = predict_signature_def(inputs={'input': model.inputs[0]}, outputs={'income': model.outputs[0]}) with K.get_session() as sess: builder.add_meta_graph_and_variables( sess=sess, tags=[tag_constants.SERVING], signature_def_map={ signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: signature }) # Relevant to here: http://liufuyang.github.io/2017/04/02/just-another-tensorflow-beginner-guide-4.html # Also, similar hack done in task.py modelSavePath = builder.save() # Save model on to google storage with file_io.FileIO(modelSavePath, mode='rb') as input_f: with file_io.FileIO(os.path.join(export_path, basename(modelSavePath)), mode='w+') as output_f: output_f.write(input_f.read())
def __init__(self, export_dir, overwrite=False): """Construct a SavedModelBuilder. Args: export_dir: The directory to export to overwrite: If `True` deletes export_dir if it already exists """ self._saved_model = saved_model_pb2.SavedModel() self._saved_model.saved_model_schema_version = ( constants.SAVED_MODEL_SCHEMA_VERSION) self._export_dir = export_dir if file_io.file_exists(export_dir): if overwrite: file_io.delete_recursively(export_dir) else: raise AssertionError( "Export directory already exists. Please specify a different export " "directory: %s or change overwrite flag to `True`" % export_dir) file_io.recursive_create_dir(self._export_dir) # Boolean to track whether variables and assets corresponding to the # SavedModel have been saved. Specifically, the first meta graph to be added # MUST use the add_meta_graph_and_variables() API. Subsequent add operations # on the SavedModel MUST use the add_meta_graph() API which does not save # weights. self._has_saved_variables = False
def testCreateRecursiveDir(self): dir_path = os.path.join(self._base_dir, "temp_dir/temp_dir1/temp_dir2") file_io.recursive_create_dir(dir_path) file_path = os.path.join(dir_path, "temp_file") file_io.FileIO(file_path, mode="w").write("testing") self.assertTrue(file_io.file_exists(file_path)) file_io.delete_recursively(os.path.join(self._base_dir, "temp_dir")) self.assertFalse(file_io.file_exists(file_path))
def _analyze(args, cell): # For now, always run python2. If needed we can run python3 when the current kernel # is py3. Since now our transform cannot work on py3 anyway, I would rather run # everything with python2. cmd_args = ['python', 'analyze.py', '--output', _abs_path(args['output'])] if args['cloud']: cmd_args.append('--cloud') training_data = args['training_data'] if args['cloud']: tmpdir = os.path.join(args['output'], 'tmp') else: tmpdir = tempfile.mkdtemp() try: if isinstance(training_data, dict): if 'csv' in training_data and 'schema' in training_data: schema = training_data['schema'] schema_file = _create_json_file(tmpdir, schema, 'schema.json') cmd_args.append('--csv=' + _abs_path(training_data['csv'])) cmd_args.extend(['--schema', schema_file]) elif 'bigquery_table' in training_data: cmd_args.extend(['--bigquery', training_data['bigquery_table']]) elif 'bigquery_sql' in training_data: # see https://cloud.google.com/bigquery/querying-data#temporary_and_permanent_tables print('Creating temporary table that will be deleted in 24 hours') r = bq.Query(training_data['bigquery_sql']).execute().result() cmd_args.extend(['--bigquery', r.full_name]) else: raise ValueError('Invalid training_data dict. ' 'Requires either "csv_file_pattern" and "csv_schema", or "bigquery".') elif isinstance(training_data, google.datalab.ml.CsvDataSet): schema_file = _create_json_file(tmpdir, training_data.schema, 'schema.json') for file_name in training_data.input_files: cmd_args.append('--csv=' + _abs_path(file_name)) cmd_args.extend(['--schema', schema_file]) elif isinstance(training_data, google.datalab.ml.BigQueryDataSet): # TODO: Support query too once command line supports query. cmd_args.extend(['--bigquery', training_data.table]) else: raise ValueError('Invalid training data. Requires either a dict, ' 'a google.datalab.ml.CsvDataSet, or a google.datalab.ml.BigQueryDataSet.') features = args['features'] features_file = _create_json_file(tmpdir, features, 'features.json') cmd_args.extend(['--features', features_file]) if args['package']: code_path = os.path.join(tmpdir, 'package') _archive.extract_archive(args['package'], code_path) else: code_path = MLTOOLBOX_CODE_PATH _shell_process.run_and_monitor(cmd_args, os.getpid(), cwd=code_path) finally: file_io.delete_recursively(tmpdir)
def _maybe_remove_file(self): # Remove the checkpoint directory in multi-worker training where this worker # should not checkpoint. It is a dummy directory previously saved for sync # distributed training. if (self.model._in_multi_worker_mode() and # pylint: disable=protected-access not multi_worker_util.should_save_checkpoint()): file_io.delete_recursively(self._temp_file_dir) del self._temp_file_dir
def testSavedModel(self): sp = SentencepieceTokenizer(self.model) test_module = TestSavedModelModule(sp) inputs = constant_op.constant(['hello world']) expected_result = test_module.tokenize(inputs) temp_dir = tempfile.mkdtemp(dir=test.get_temp_dir()) save.save(test_module, temp_dir) restored_model = load.load(temp_dir) self.assertAllEqual(restored_model.tokenize(inputs), expected_result) file_io.delete_recursively(temp_dir)
def testGetMatchingFiles(self): dir_path = os.path.join(self._base_dir, "temp_dir") file_io.create_dir(dir_path) files = ["file1.txt", "file2.txt", "file3.txt"] for name in files: file_path = os.path.join(dir_path, name) file_io.FileIO(file_path, mode="w").write("testing") expected_match = [os.path.join(dir_path, name) for name in files] self.assertItemsEqual(file_io.get_matching_files(os.path.join(dir_path, "file*.txt")), expected_match) file_io.delete_recursively(dir_path) self.assertFalse(file_io.file_exists(os.path.join(dir_path, "file3.txt")))
def testGetMatchingFiles(self): dir_path = os.path.join(self._base_dir, "temp_dir") file_io.create_dir(dir_path) files = ["file1.txt", "file2.txt", "file3.txt"] for name in files: file_path = os.path.join(dir_path, name) file_io.FileIO(file_path, mode="w").write("testing") expected_match = [os.path.join(dir_path, name) for name in files] self.assertItemsEqual( file_io.get_matching_files(os.path.join(dir_path, "file*.txt")), expected_match) file_io.delete_recursively(dir_path) self.assertFalse(file_io.file_exists(os.path.join(dir_path, "file3.txt")))
def delete_backup(self): """Delete the backup directories. Delete the backup directories which should not exist after `fit()` successfully finishes. """ # pylint: disable=protected-access for pathname in file_io.get_matching_files( self.write_checkpoint_manager._prefix + '*'): file_io.delete_recursively(pathname) for pathname in file_io.get_matching_files( os.path.join(self.write_checkpoint_manager.directory, 'checkpoint')): file_io.delete_recursively(pathname)
def testGradientsValuesFromDumpWorks(self): y = math_ops.add(self.w, -1.0, name="y") z = math_ops.square(y, name="z") grad_debugger = debug_gradients.GradientsDebugger() with grad_debugger.watch_gradients_by_tensors(self.sess.graph, [self.w, self.u, y]): train_op = gradient_descent.GradientDescentOptimizer(0.1).minimize( z) self.sess.run(variables.global_variables_initializer()) run_options = config_pb2.RunOptions(output_partition_graphs=True) dump_dir = tempfile.mkdtemp() debug_url = "file://" + dump_dir debug_utils.watch_graph(run_options, self.sess.graph, debug_urls=debug_url) run_metadata = config_pb2.RunMetadata() self.assertAllClose(2.0, self.sess.run(self.u)) self.sess.run(train_op, options=run_options, run_metadata=run_metadata) self.assertAllClose(-1.0, self.sess.run(self.u)) dump = debug_data.DebugDumpDir( dump_dir, partition_graphs=run_metadata.partition_graphs) dump.set_python_graph(self.sess.graph) y_grad_values = debug_gradients.gradient_values_from_dump( grad_debugger, y, dump) self.assertEqual(1, len(y_grad_values)) self.assertAllClose(10.0, y_grad_values[0]) w_grad_values = debug_gradients.gradient_values_from_dump( grad_debugger, self.w, dump) self.assertEqual(1, len(w_grad_values)) self.assertAllClose(10.0, w_grad_values[0]) u_grad_values = debug_gradients.gradient_values_from_dump( grad_debugger, self.u, dump) self.assertEqual(1, len(u_grad_values)) self.assertAllClose(30.0, u_grad_values[0]) with self.assertRaisesRegexp( LookupError, r"This GradientsDebugger has not received any gradient tensor for " r"x-tensor v:0"): debug_gradients.gradient_values_from_dump(grad_debugger, self.v, dump) # Cleanup. file_io.delete_recursively(dump_dir)
def export(sess, inputs, outputs, output_dir): if file_io.file_exists(output_dir): file_io.delete_recursively(output_dir) signature_def = build_signature(inputs, outputs) signature_def_map = { signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: signature_def } builder = saved_model_builder.SavedModelBuilder(output_dir) builder.add_meta_graph_and_variables( sess, tags=[tag_constants.SERVING], signature_def_map=signature_def_map) builder.save()
def remove_temp_dirpath(dirpath, strategy=None): """Removes the temp path after writing is finished.""" if strategy is None: # Infer strategy from `distribution_strategy_context` if not given. strategy = distribution_strategy_context.get_strategy() if strategy is None: # If strategy is still not available, this is not in distributed training. # Fallback to no-op. return # TODO(anjalisridhar): Consider removing the check for multi worker mode since # it is redundant when used with the should_checkpoint property. if (strategy.extended._in_multi_worker_mode() and # pylint: disable=protected-access not strategy.extended.should_checkpoint): # If this worker is not chief and hence should not save file, remove # the temporary directory. file_io.delete_recursively(_get_temp_dir(dirpath, strategy))
def _save_model(self): signature = tf.saved_model.signature_def_utils.predict_signature_def( inputs={"example": self._input_example}, outputs={"predict": self._predict}) if file_io.file_exists(self._save_model_dir): file_io.delete_recursively(self._save_model_dir) builder = tf.saved_model.builder.SavedModelBuilder( self._save_model_dir) import tensorflow.saved_model.signature_constants as s_const builder.add_meta_graph_and_variables( sess=self._sess, tags=[tf.saved_model.tag_constants.SERVING], signature_def_map={ s_const.DEFAULT_SERVING_SIGNATURE_DEF_KEY: signature }, ) builder.save()
def to_savedmodel(model, export_path): """Convert the Keras HDF5 model into TensorFlow SavedModel.""" if export_path.startswith('gs://'): _save_oncloud(model, export_path) else: ### Allow overwriting of export_path if it already exists by removing it first.. if file_io.file_exists(export_path): file_io.delete_recursively(export_path) builder = saved_model_builder.SavedModelBuilder(export_path) signature = predict_signature_def(inputs={'input': model.inputs[0]}, outputs={'income': model.outputs[0]}) with K.get_session() as sess: builder.add_meta_graph_and_variables( sess=sess, tags=[tag_constants.SERVING], signature_def_map={ signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: signature }) builder.save()
def tearDown(self): file_io.delete_recursively(self._base_dir)
def testDeleteRecursivelyFail(self): fake_dir_path = os.path.join(self._base_dir, "temp_dir") with self.assertRaises(errors.NotFoundError): file_io.delete_recursively(fake_dir_path)
def tearDownModule(): file_io.delete_recursively(test.get_temp_dir())
def tearDown(self): ops.reset_default_graph() if os.path.isdir(self._dump_root): file_io.delete_recursively(self._dump_root) super(SessionDebugGrapplerInteractionTest, self).tearDown()
def delete_recursively(cls, dirname): file_io.delete_recursively(dirname)
def tearDown(self): # Tear down temporary dump directory. file_io.delete_recursively(self._dump_root)
def create_dir_test(): """Verifies file_io directory handling methods.""" # Test directory creation. starttime_ms = int(round(time.time() * 1000)) dir_name = "%s/tf_gcs_test_%s" % (FLAGS.gcs_bucket_url, starttime_ms) print("Creating dir %s" % dir_name) file_io.create_dir(dir_name) elapsed_ms = int(round(time.time() * 1000)) - starttime_ms print("Created directory in: %d milliseconds" % elapsed_ms) # Check that the directory exists. dir_exists = file_io.is_directory(dir_name) assert dir_exists print("%s directory exists: %s" % (dir_name, dir_exists)) # Test recursive directory creation. starttime_ms = int(round(time.time() * 1000)) recursive_dir_name = "%s/%s/%s" % (dir_name, "nested_dir1", "nested_dir2") print("Creating recursive dir %s" % recursive_dir_name) file_io.recursive_create_dir(recursive_dir_name) elapsed_ms = int(round(time.time() * 1000)) - starttime_ms print("Created directory recursively in: %d milliseconds" % elapsed_ms) # Check that the directory exists. recursive_dir_exists = file_io.is_directory(recursive_dir_name) assert recursive_dir_exists print("%s directory exists: %s" % (recursive_dir_name, recursive_dir_exists)) # Create some contents in the just created directory and list the contents. num_files = 10 files_to_create = ["file_%d.txt" % n for n in range(num_files)] for file_num in files_to_create: file_name = "%s/%s" % (dir_name, file_num) print("Creating file %s." % file_name) file_io.write_string_to_file(file_name, "test file.") print("Listing directory %s." % dir_name) starttime_ms = int(round(time.time() * 1000)) directory_contents = file_io.list_directory(dir_name) print(directory_contents) elapsed_ms = int(round(time.time() * 1000)) - starttime_ms print("Listed directory %s in %s milliseconds" % (dir_name, elapsed_ms)) assert set(directory_contents) == set(files_to_create + ["nested_dir1/"]) # Test directory renaming. dir_to_rename = "%s/old_dir" % dir_name new_dir_name = "%s/new_dir" % dir_name file_io.create_dir(dir_to_rename) assert file_io.is_directory(dir_to_rename) assert not file_io.is_directory(new_dir_name) starttime_ms = int(round(time.time() * 1000)) print("Will try renaming directory %s to %s" % (dir_to_rename, new_dir_name)) file_io.rename(dir_to_rename, new_dir_name) elapsed_ms = int(round(time.time() * 1000)) - starttime_ms print("Renamed directory %s to %s in %s milliseconds" % ( dir_to_rename, new_dir_name, elapsed_ms)) assert not file_io.is_directory(dir_to_rename) assert file_io.is_directory(new_dir_name) # Test Delete directory recursively. print("Deleting directory recursively %s." % dir_name) starttime_ms = int(round(time.time() * 1000)) file_io.delete_recursively(dir_name) elapsed_ms = int(round(time.time() * 1000)) - starttime_ms dir_exists = file_io.is_directory(dir_name) assert not dir_exists print("Deleted directory recursively %s in %s milliseconds" % ( dir_name, elapsed_ms))
def create_object_test(): """Verifies file_io's object manipulation methods .""" starttime_ms = int(round(time.time() * 1000)) dir_name = "%s/tf_gcs_test_%s" % (FLAGS.gcs_bucket_url, starttime_ms) print("Creating dir %s." % dir_name) file_io.create_dir(dir_name) num_files = 5 # Create files of 2 different patterns in this directory. files_pattern_1 = ["%s/test_file_%d.txt" % (dir_name, n) for n in range(num_files)] files_pattern_2 = ["%s/testfile%d.txt" % (dir_name, n) for n in range(num_files)] starttime_ms = int(round(time.time() * 1000)) files_to_create = files_pattern_1 + files_pattern_2 for file_name in files_to_create: print("Creating file %s." % file_name) file_io.write_string_to_file(file_name, "test file creation.") elapsed_ms = int(round(time.time() * 1000)) - starttime_ms print("Created %d files in %s milliseconds" % (len(files_to_create), elapsed_ms)) # Listing files of pattern1. list_files_pattern = "%s/test_file*.txt" % dir_name print("Getting files matching pattern %s." % list_files_pattern) starttime_ms = int(round(time.time() * 1000)) files_list = file_io.get_matching_files(list_files_pattern) elapsed_ms = int(round(time.time() * 1000)) - starttime_ms print("Listed files in %s milliseconds" % elapsed_ms) print(files_list) assert set(files_list) == set(files_pattern_1) # Listing files of pattern2. list_files_pattern = "%s/testfile*.txt" % dir_name print("Getting files matching pattern %s." % list_files_pattern) starttime_ms = int(round(time.time() * 1000)) files_list = file_io.get_matching_files(list_files_pattern) elapsed_ms = int(round(time.time() * 1000)) - starttime_ms print("Listed files in %s milliseconds" % elapsed_ms) print(files_list) assert set(files_list) == set(files_pattern_2) # Test renaming file. file_to_rename = "%s/oldname.txt" % dir_name file_new_name = "%s/newname.txt" % dir_name file_io.write_string_to_file(file_to_rename, "test file.") assert file_io.file_exists(file_to_rename) assert not file_io.file_exists(file_new_name) print("Will try renaming file %s to %s" % (file_to_rename, file_new_name)) starttime_ms = int(round(time.time() * 1000)) file_io.rename(file_to_rename, file_new_name) elapsed_ms = int(round(time.time() * 1000)) - starttime_ms print("File %s renamed to %s in %s milliseconds" % ( file_to_rename, file_new_name, elapsed_ms)) assert not file_io.file_exists(file_to_rename) assert file_io.file_exists(file_new_name) # Delete directory. print("Deleting directory %s." % dir_name) file_io.delete_recursively(dir_name)
def _remove_dir(dir_to_remove): file_io.delete_recursively(dir_to_remove)
def export_fn(estimator, export_dir_base, checkpoint_path=None, eval_result=None): with ops.Graph().as_default() as g: contrib_variables.create_global_step(g) input_ops = feature_transforms.build_csv_serving_tensors_for_training_step( args.analysis, features, schema, stats, keep_target) model_fn_ops = estimator._call_model_fn(input_ops.features, None, model_fn_lib.ModeKeys.INFER) output_fetch_tensors = make_prediction_output_tensors( args=args, features=features, input_ops=input_ops, model_fn_ops=model_fn_ops, keep_target=keep_target) # Don't use signature_def_utils.predict_signature_def as that renames # tensor names if there is only 1 input/output tensor! signature_inputs = {key: tf.saved_model.utils.build_tensor_info(tensor) for key, tensor in six.iteritems(input_ops.default_inputs)} signature_outputs = {key: tf.saved_model.utils.build_tensor_info(tensor) for key, tensor in six.iteritems(output_fetch_tensors)} signature_def_map = { 'serving_default': signature_def_utils.build_signature_def( signature_inputs, signature_outputs, tf.saved_model.signature_constants.PREDICT_METHOD_NAME)} if not checkpoint_path: # Locate the latest checkpoint checkpoint_path = saver.latest_checkpoint(estimator._model_dir) if not checkpoint_path: raise ValueError("Couldn't find trained model at %s." % estimator._model_dir) export_dir = saved_model_export_utils.get_timestamped_export_dir( export_dir_base) if (model_fn_ops.scaffold is not None and model_fn_ops.scaffold.saver is not None): saver_for_restore = model_fn_ops.scaffold.saver else: saver_for_restore = saver.Saver(sharded=True) with tf_session.Session('') as session: saver_for_restore.restore(session, checkpoint_path) init_op = control_flow_ops.group( variables.local_variables_initializer(), resources.initialize_resources(resources.shared_resources()), tf.tables_initializer()) # Perform the export builder = saved_model_builder.SavedModelBuilder(export_dir) builder.add_meta_graph_and_variables( session, [tag_constants.SERVING], signature_def_map=signature_def_map, assets_collection=ops.get_collection( ops.GraphKeys.ASSET_FILEPATHS), legacy_init_op=init_op) builder.save(False) # Add the extra assets if assets_extra: assets_extra_path = os.path.join(compat.as_bytes(export_dir), compat.as_bytes('assets.extra')) for dest_relative, source in assets_extra.items(): dest_absolute = os.path.join(compat.as_bytes(assets_extra_path), compat.as_bytes(dest_relative)) dest_path = os.path.dirname(dest_absolute) file_io.recursive_create_dir(dest_path) file_io.copy(source, dest_absolute) # only keep the last 3 models saved_model_export_utils.garbage_collect_exports( export_dir_base, exports_to_keep=3) # save the last model to the model folder. # export_dir_base = A/B/intermediate_models/ if keep_target: final_dir = os.path.join(args.job_dir, 'evaluation_model') else: final_dir = os.path.join(args.job_dir, 'model') if file_io.is_directory(final_dir): file_io.delete_recursively(final_dir) file_io.recursive_create_dir(final_dir) recursive_copy(export_dir, final_dir) return export_dir
def tearDown(self): file_io.delete_recursively(test.get_temp_dir())
def delete_temp_dir(dirname=temp_dir): try: file_io.delete_recursively(dirname) except errors.OpError as e: logging.error('Error removing %s: %s', dirname, e)
def export_fn(estimator, export_dir_base, checkpoint_path=None, eval_result=None): with ops.Graph().as_default() as g: contrib_variables.create_global_step(g) input_ops = serving_from_csv_input(train_config, args, keep_target) model_fn_ops = estimator._call_model_fn(input_ops.features, None, model_fn_lib.ModeKeys.INFER) output_fetch_tensors = make_output_tensors( train_config=train_config, args=args, input_ops=input_ops, model_fn_ops=model_fn_ops, keep_target=keep_target) signature_def_map = { 'serving_default': signature_def_utils.predict_signature_def(input_ops.default_inputs, output_fetch_tensors) } if not checkpoint_path: # Locate the latest checkpoint checkpoint_path = saver.latest_checkpoint(estimator._model_dir) if not checkpoint_path: raise NotFittedError("Couldn't find trained model at %s." % estimator._model_dir) export_dir = saved_model_export_utils.get_timestamped_export_dir( export_dir_base) with tf_session.Session('') as session: # variables.initialize_local_variables() variables.local_variables_initializer() data_flow_ops.tables_initializer() saver_for_restore = saver.Saver( variables.global_variables(), sharded=True) saver_for_restore.restore(session, checkpoint_path) init_op = control_flow_ops.group( variables.local_variables_initializer(), data_flow_ops.tables_initializer()) # Perform the export builder = saved_model_builder.SavedModelBuilder(export_dir) builder.add_meta_graph_and_variables( session, [tag_constants.SERVING], signature_def_map=signature_def_map, assets_collection=ops.get_collection( ops.GraphKeys.ASSET_FILEPATHS), legacy_init_op=init_op) builder.save(False) # Add the extra assets if assets_extra: assets_extra_path = os.path.join(compat.as_bytes(export_dir), compat.as_bytes('assets.extra')) for dest_relative, source in assets_extra.items(): dest_absolute = os.path.join(compat.as_bytes(assets_extra_path), compat.as_bytes(dest_relative)) dest_path = os.path.dirname(dest_absolute) gfile.MakeDirs(dest_path) gfile.Copy(source, dest_absolute) # only keep the last 3 models saved_model_export_utils.garbage_collect_exports( python_portable_string(export_dir_base), exports_to_keep=3) # save the last model to the model folder. # export_dir_base = A/B/intermediate_models/ if keep_target: final_dir = os.path.join(args.job_dir, 'evaluation_model') else: final_dir = os.path.join(args.job_dir, 'model') if file_io.is_directory(final_dir): file_io.delete_recursively(final_dir) file_io.recursive_create_dir(final_dir) _recursive_copy(export_dir, final_dir) return export_dir
def tearDown(self): if os.path.isdir(self.dump_root): file_io.delete_recursively(self.dump_root) ops.reset_default_graph()
from __future__ import print_function import tensorflow as tf from tensorflow.python.lib.io import file_io import numpy as np from model import inference FLAGS = tf.app.flags.FLAGS tf.app.flags.DEFINE_integer('input_height', '72', '') tf.app.flags.DEFINE_integer('input_width', '24', '') tf.app.flags.DEFINE_integer('input_ch', '1', '') tf.app.flags.DEFINE_string('restore', None, '') tf.app.flags.DEFINE_bool('as_text', False, '') if file_io.file_exists('./model'): file_io.delete_recursively('./model') builder = tf.saved_model.builder.SavedModelBuilder('./model') with tf.Session(graph=tf.Graph()) as sess: ph_images = tf.placeholder( tf.float32, shape=[None, FLAGS.input_height, FLAGS.input_width, FLAGS.input_ch]) ph_lengths = tf.placeholder(tf.float32) ph_widths = tf.placeholder(tf.float32) ph_areas = tf.placeholder(tf.float32) logits = inference(ph_images, ph_lengths, ph_widths, ph_areas, 1.0, False) accuracies = tf.nn.softmax(logits, name='accuracies') _, predicts = tf.nn.top_k(accuracies, k=2, name='predicts') saver = tf.train.Saver()