def check_is_install_module(mod_name):
try:
__import__(mod_name)
return True
except Exception as err:
pylog.error(err)
return False
def start_progress(progress_name):
pid = _get_progress_pid(progress_name)
if not pid:
ret = os.system("nohup python %s > /dev/null 2>&1 &" % progress_name)
if ret == 0:
return True
else:
pylog.error("progressname:%s status:%s", (progress_name, ret))
return False
def open_file_with_full_name(full_path, open_type):
try:
file_object = open(full_path, open_type)
return file_object
except Exception as e:
if e.args[0] == 2:
open(full_path, 'w')
else:
pylog.error(e)
def read(name, path=_get_save_path()):
try:
md5_str = _get_md5_name(name)
file_obj = _get_read_save_file(path, md5_str)
obj = pickle.load(file_obj)
return obj
except Exception as err:
pylog.error(err)
return False
def get_key_value_str(key, val, db_type):
key_value_item = ""
if db_type == DB_TYPE_MYSQL:
key_value_item = '`%s` %s' % (key, val)
elif db_type == DB_TYPE_SQLITE:
key_value_item = '%s %s' % (key, val)
else:
pylog.error("upkown db_type")
return key_value_item
def save(content, name, path=_get_save_path()):
try:
md5_str = _get_md5_name(name)
file_obj = _get_write_save_file(path, md5_str)
pickle.dump(content, file_obj, 0)
except Exception as err:
pylog.error(err)
return False
return True
def get_table_create_str(key_value_list_str, tb_name, db_type):
if db_type == DB_TYPE_MYSQL:
return "CREATE TABLE IF NOT EXISTS `%s` (\n%s\n ) ENGINE=MyISAM DEFAULT CHARSET=utf8 ;" % (
tb_name, key_value_list_str)
elif db_type == DB_TYPE_SQLITE:
return "CREATE TABLE IF NOT EXISTS %s(\n%s\n)" % (tb_name,
key_value_list_str)
else:
pylog.error("upkown db_type")
return ""
def get_item_value(key_str, db_type):
dic = {}
if db_type == DB_TYPE_MYSQL:
dic = get_mysql_key_type_dic()
elif db_type == DB_TYPE_SQLITE:
dic = get_sqlite_key_type_dic()
else:
pylog.error("upkown db_type")
return dic[key_str]
def http_get(url):
method = "GET"
client = tornado.httpclient.HTTPClient()
try:
request = tornado.httpclient.HTTPRequest(url=url, method=method)
response = client.fetch(request)
return (True, response.body)
except tornado.httpclient.HTTPError as err:
pylog.error("err:%s url:%s" % (err, url))
return (False, err.message)
def http_post(url, data):
method = "POST"
client = tornado.httpclient.HTTPClient()
try:
request = tornado.httpclient.HTTPRequest(url=url,
method=method,
body=data)
response = client.fetch(request)
return (True, response.body)
except tornado.httpclient.HTTPError as err:
#pylog.error(err)
pylog.error("err:%s url:%s, data:%s" % (err, url, data))
return (False, err.message)
def _load_spacy_tokenizer(self, special_tokens_lt, config):
try:
import ftfy
import spacy
en_spacy_path = get_spacy_file_path(config)
self.nlp = spacy.load(
en_spacy_path, disable=['parser', 'tagger', 'ner', 'textcat'])
self.fix_text = ftfy.fix_text
self.nlp_type = "spacy"
except Exception as err:
pylog.error(
"Error:%s \nftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."
% err)
self._load_bert_toenizer(special_tokens_lt)
def delete_dir(src):
if os.path.isfile(src):
try:
os.remove(src)
except Exception as e:
pylog.error(e)
return False
elif os.path.isdir(src):
for item in os.listdir(src):
itemsrc = os.path.join(src, item)
delete_dir(itemsrc)
try:
os.rmdir(src)
except Exception as e:
pylog.error(e)
return False
return True
def __init__(self, config, *inputs, **kwargs):
super(OpenAIGPTPreTrainedModel, self).__init__()
if not isinstance(config, OpenAIGPTConfig):
pylog.error("config should be an istance of class OpenAIGPTConfig")
self.config = config
def handler_to_signal(sig_num, frame):
sig_name = SIG_DIC[sig_num]
pylog.error("receive signal:%s, then exit" % sig_name)
exit()