def main(argv):
config = load_config(argv[1])
ops = argv[2].split(",")
model = None
if ("train" in ops):
for i in xrange(5):
model = train_model(config, model_seq=i + 3)
def __init__(self, config_path=DEFAULT_CONFIG_PATH):
self.config = load_config(config_path)
corenlp_config = self.config["data"]["stanford_corenlp"]
self.tagger = CoreNLPPOSTagger(
url="http://%s:%d" %
(corenlp_config["host"], corenlp_config["port"]))
self.pos_map = self.config["model"]["STANFORD_POS_MAP"]
def main():
"""
Setup consumer
"""
config = loader.load_config()
logging.info("Connecting to Kafka broker at %s:%s", config.kafka_host, config.kafka_port)
if config.benchmark:
create_sample_messages(config)
start_consumer(config)
def main():
"""
Setup consumer
"""
config = loader.load_config()
logging.info("Connecting to Kafka broker at %s:%s", config.kafka_host,
config.kafka_port)
if config.benchmark:
create_sample_messages(config)
start_consumer(config)
def tag_claims(doc, config_path, model=None, model_path=None, cache=True):
global model_cache
config = load_config(config_path)
classes = sorted(config["model"]["CLASSES"])
annotation_types = config["model"]["ANNOTATIONS"]
if (model is None):
if (not model_path):
if (cache and model_cache is not None):
model = model_cache
else:
model = DeepClaimTagger(config)
model.load(config["data"]["model_path"] % (config["id"], 0))
model_cache = model
else:
model = DeepClaimTagger(config)
model.load(model_path)
model_cache = model
sent_encs = encode_document(doc, config)[0]
input_morph = []
input_attr = []
for (enc_morph, enc_attrs) in sent_encs:
input_morph.append(enc_morph)
input_attr.append(enc_attrs)
enc_sent_segments = model.predict([
np.array(input_morph, dtype=np.uint8),
np.array(input_attr, dtype=np.float32)
])
for (sentence, enc_sent_seg) in izip(doc.sentences, enc_sent_segments):
dec_segments = decode_sentence(enc_sent_seg, classes)
token_confidence = confidence(enc_sent_seg)
avg_confidence = [
float(np.mean(token_conf)) for token_conf in token_confidence
]
for i in xrange(len(sentence.tokens)):
segcls = dec_segments[i]
sentence.tokens[i].annotations["PATCLAIM_SEG"] = segcls
sentence.annotations["PATCLAIM_SEG_INFO"] = {
"token_confidence": token_confidence,
"confidence": avg_confidence
}
logger.info("Segmentation: " + " ".join([
token.surface + ":" + seg for (token, seg) in izip(
sentence.tokens, dec_segments[0:len(sentence.tokens)])
]))
return doc
def pos_tag(doc, config_path, model_path=None, cache=True):
global model_cache
config = load_config(config_path)
pos_classes = sorted(config["model"]["POS_CLASSES"])
inv_pos_map = dict()
for item in config["model"]["POS_MAP"].items():
vals = item[1].split("|")
for val in vals:
inv_pos_map[val] = item[0]
if (not model_path):
if (cache and model_cache is not None):
model = model_cache
else:
model = DeepTDVPOSTagger(config)
model.load(config["data"]["model_path"] % (config["id"], 0))
model_cache = model
else:
model = DeepTDVPOSTagger(config)
model.load(model_path)
model_cache = model
sent_encs = encode_document(doc, config)[0]
input_morph = []
input_attr = []
for (enc_morph, enc_attrs) in sent_encs:
input_morph.append(enc_morph)
input_attr.append(enc_attrs)
for (sentence, enc_sent_pos) in izip(doc.sentences, model.predict([np.array(input_morph, dtype=np.uint8), np.array(input_attr, dtype=np.float16)])):
dec_pos = decode_sentence(enc_sent_pos, pos_classes)
token_confidence = confidence(enc_sent_pos)
avg_confidence = float(np.mean(token_confidence))
for i in xrange(len(sentence.tokens)):
if (i < config["model"]["MAX_SENT_LEN"]):
if (token_confidence[i] > config["hyperparam"]["confidence_thresh"]):
pos = dec_pos[i]
else:
pos = "noun"
else:
pos = "x"
sentence.tokens[i].annotations["UPOS"] = inv_pos_map[pos]
sentence.tokens[i].annotations["POS"] = pos
sentence.annotations["UPOS_TAGGER_INFO"] = {"token_confidence": token_confidence, "confidence": avg_confidence}
return doc
def model_paths(config_paths=[]):
model_num = 0
last_config_path = ""
paths = []
for config_path in config_paths:
if (config_path == last_config_path):
model_num += 1
else:
model_num = 0
config = load_config(config_path)
paths.append(config["data"]["model_path"] % (config["id"], model_num))
last_config_path = config_path
return paths
config.influxdb_timeout, config.influxdb_use_udp,
config.influxdb_retention_policy, config.influxdb_time_precision)
def show_version():
"""
Output current version and exit
"""
print("{} {}".format(__title__, __version__))
sys.exit(0)
if __name__ == '__main__':
FORMAT = '%(asctime)-15s %(name)s:%(levelname)s:%(message)s'
logging.basicConfig(format=FORMAT)
config = loader.load_config()
if config.version:
show_version()
# Check for a pidfile to see if the program already runs
try:
pf = file(config.pidfile, 'r')
pid = int(pf.read().strip())
pf.close()
except IOError:
pid = None
if pid:
message = "pidfile %s already exist. Daemon already running?\n"
sys.stderr.write(message % config.pidfile)
sys.exit(1)
pid = str(os.getpid())
reuse=True)
opt = tf.train.AdamOptimizer(lr_ph)
update_op = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_op):
train_op = opt.minimize(loss)
saver = tf.train.Saver()
sess = tf.Session()
sess.run(tf.global_variables_initializer())
if args.pretrained:
loc = os.path.join(args.model_dir, args.model_name,
args.dataset_name + '.ckpt')
saver.restore(sess, loc)
config = load_config(args.config)
train_gen = EpisodeGenerator(args.dataset_dir, 'train', config)
test_gen = EpisodeGenerator(args.dataset_dir, 'test', config)
if args.train:
max_iter = train_gen.dataset_size[args.dataset_name] * args.max_epoch \
// (nway * qsize)
show_step = args.show_epoch * max_iter // args.max_epoch
save_step = args.save_epoch * max_iter // args.max_epoch
avger = np.zeros([4])
for i in range(1, max_iter + 1):
stt = time.time()
cur_epoch = i * (
nway * qsize) // train_gen.dataset_size[args.dataset_name]
lr = args.lr if i < 0.7 * max_iter else args.lr * .1
sx, sy, qx, qy = train_gen.get_episode(nway, kshot, qsize)
fd = {\
def __init__(self, config_path=DEFAULT_CONFIG_PATH):
self.config = load_config(config_path)
self.conn = MongoClient(self.config["data"]["semantic_db"]["host"],
self.config["data"]["semantic_db"]["port"])
self.db = self.conn.semdb
self.pos_map = self.config["model"]["POS_MAP"]
from config.loader import load_config
SSL_VERIFY = True if load_config().get("mattermost").get(
"ssl_verify") is "yes" else False
BOT_URL = load_config().get("mattermost").get("url")
BOT_LOGIN = load_config().get("mattermost").get("login")
BOT_PASSWORD = load_config().get("mattermost").get("password")
BOT_TEAM = load_config().get("mattermost").get("team")
IGNORE_NOTIFIES = ['@channel', '@all', '@here']
DEFAULT_REPLY = "I didn't understand your message"
