def get_distribution_strategy(params):
"""Returns the distribution strategy to use."""
if params["turn_off_distribution_strategy"]:
return None
if params["use_tpu"]:
# Some of the networking libraries are quite chatty.
for name in ["googleapiclient.discovery", "googleapiclient.discovery_cache",
"oauth2client.transport"]:
logging.getLogger(name).setLevel(logging.ERROR)
tpu_cluster_resolver = tf.distribute.cluster_resolver.TPUClusterResolver(
tpu=params["tpu"],
zone=params["tpu_zone"],
project=params["tpu_gcp_project"],
coordinator_name="coordinator"
)
logging.info("Issuing reset command to TPU to ensure a clean state.")
tf.Session.reset(tpu_cluster_resolver.get_master())
# Estimator looks at the master it connects to for MonitoredTrainingSession
# by reading the `TF_CONFIG` environment variable, and the coordinator
# is used by StreamingFilesDataset.
tf_config_env = {
"session_master": tpu_cluster_resolver.get_master(),
"eval_session_master": tpu_cluster_resolver.get_master(),
"coordinator": tpu_cluster_resolver.cluster_spec()
.as_dict()["coordinator"]
}
os.environ['TF_CONFIG'] = json.dumps(tf_config_env)
distribution = tf.distribute.experimental.TPUStrategy(
tpu_cluster_resolver, steps_per_run=100)
else:
distribution = distribution_utils.get_distribution_strategy(
num_gpus=params["num_gpus"])
return distribution
from .tokenization_ctrl import CTRLTokenizer
from .tokenization_distilbert import DistilBertTokenizer, DistilBertTokenizerFast
from .tokenization_flaubert import FlaubertTokenizer
from .tokenization_gpt2 import GPT2Tokenizer, GPT2TokenizerFast
from .tokenization_openai import OpenAIGPTTokenizer, OpenAIGPTTokenizerFast
from .tokenization_roberta import RobertaTokenizer, RobertaTokenizerFast
from .tokenization_t5 import T5Tokenizer
from .tokenization_transfo_xl import TransfoXLCorpus, TransfoXLTokenizer, TransfoXLTokenizerFast
# Tokenizers
from .tokenization_utils import PreTrainedTokenizer
from .tokenization_xlm import XLMTokenizer
from .tokenization_xlm_roberta import XLMRobertaTokenizer
from .tokenization_xlnet import SPIECE_UNDERLINE, XLNetTokenizer
logger = logging.getLogger(__name__) # pylint: disable=invalid-name
if is_sklearn_available():
from .data import glue_compute_metrics, xnli_compute_metrics
# Modeling
if is_torch_available():
from .modeling_utils import PreTrainedModel, prune_layer, Conv1D, top_k_top_p_filtering
from .modeling_auto import (
AutoModel,
AutoModelForPreTraining,
AutoModelForSequenceClassification,
AutoModelForQuestionAnswering,
AutoModelWithLMHead,
AutoModelForTokenClassification,
ALL_PRETRAINED_MODEL_ARCHIVE_MAP,
# Logging options ##########################################################'
logging.root.removeHandler(absl.logging._absl_handler)
absl.logging._warn_preinit_stderr = False
date = pd.datetime.now().date()
hour = pd.datetime.now().hour
minute = pd.datetime.now().minute
logging.basicConfig(
level=logging.INFO,
format='%(asctime)-15s %(name)-5s %(levelname)-8s %(message)s',
filename="image2seq/logs/train_log_{}_{}{}.txt".format(date, hour, minute))
console = logging.StreamHandler()
console.setLevel(logging.INFO)
formatter = logging.Formatter(
'%(asctime)-15s %(name)-5s %(levelname)-8s %(message)s')
console.setFormatter(formatter)
logging.getLogger("").addHandler(console)
#############################################################################
# Model Setup #
#############################################################################
logging.info("MODEL SETUP - Tensorflow version".format(tf.__version__))
logging.info("MODEL SETUP - Training Script - train_nested.py")
from tensorflow.python.client import device_lib
logging.info("MODEL SETUP - CUDA VISIBLE DEVICES {}".format(
device_lib.list_local_devices()))
tf.compat.v1.debugging.assert_equal(True, tf.test.is_gpu_available())
tf.compat.v1.debugging.assert_equal(True, tf.test.is_built_with_cuda())
image2seq = DRAKENESTEDSINGLELSTM()
logging.info("MODEL SETUP - image2seq model {} instantiated".format(
image2seq.get_model_name()))
import hanlp.losses
import hanlp.metrics
import hanlp.optimizers
import hanlp.pretrained
import hanlp.utils
from hanlp.version import __version__
import os
if not os.environ.get('HANLP_SHOW_TF_LOG', None):
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
os.environ['TF_CPP_MIN_VLOG_LEVEL'] = '3'
import absl.logging, logging
logging.getLogger('tensorflow').setLevel(logging.ERROR)
logging.root.removeHandler(absl.logging._absl_handler)
exec('absl.logging._warn_preinit_stderr = False'
) # prevent exporting _warn_preinit_stderr
if not os.environ.get('HANLP_GREEDY_GPU', None):
exec('from hanlp.utils.tf_util import nice_gpu')
exec('nice_gpu()')
exec('''
from hanlp.utils.util import ls_resource_in_module
ls_resource_in_module(hanlp.pretrained)
''')
def load(save_dir,
from absl import flags
from absl import logging
import numpy as np
import json
import logging
# http 接口
from flask import Flask, jsonify, request
import bert_example
import predict_utils
import tagging_converter
import utils
import tensorflow as tf
app = Flask(__name__)
logger = logging.getLogger('log')
logger.setLevel(logging.DEBUG)
while logger.hasHandlers():
for i in logger.handlers:
logger.removeHandler(i)
user_name = "" # wzk/
version = "1.0.0.0"
block_list = os.path.realpath(__file__).split("/")
path = "/".join(block_list[:-2])
sys.path.append(path)
# FLAGS = flags.FLAGS
FLAGS = tf.app.flags.FLAGS
