def main():
# Add missing directories if needed
for directory in ("tmp", "bin",):
if not os.path.exists(directory):
os.makedirs(directory)
# Parse console arguments
if len(sys.argv) == 2:
os.chdir("tmp/")
if sys.argv[1] == "build":
build()
elif sys.argv[1] == "run":
run()
elif sys.argv[1] == "test":
build()
run()
elif sys.argv[1] == "clean":
os.chdir("..")
shutil.rmtree("tmp")
print("Cleaned ./tmp directory.")
elif sys.argv[1] == "help":
print("Usage:")
print(" build\t- Build project")
print(" run\t- Run project")
print(" test\t- Build and run project")
print(" clean\t- Clean temporary directory\n")
else:
print("Invalid argument: " + sys.argv[1])
sys.exit(1)
else:
print("Missing arguments. Use 'help' to display usage.")
sys.exit(1)
def main():
try:
# Add missing directories if needed
for directory in ("tmp", "bin",):
if not os.path.exists(directory):
os.makedirs(directory)
# Parse console arguments
if len(sys.argv) == 2:
os.chdir("tmp/")
if sys.argv[1] == "build":
build()
elif sys.argv[1] == "run":
run()
elif sys.argv[1] == "test":
build()
run()
elif sys.argv[1] == "clean":
os.chdir("..")
shutil.rmtree("tmp")
print "Cleaned ./tmp directory."
else:
raise ValueError("Invalid argument: " + sys.argv[1])
else:
raise ValueError("Missing arguments")
except ValueError as exc:
print exc
print "Usage:"
print " build\t- Build project"
print " run\t- Run project"
print " test\t- Build and run project"
print " clean\t- Clean temporary directory\n"
except Exception as exc:
print exc
def do_add(self, context, *args):
'''
Add the given node(s) to the cluster.
Installs packages, sets up corosync and pacemaker, etc.
Must be executed from a node in the existing cluster.
'''
params = self._args_implicit(context, args, 'node')
paramdict = utils.nvpairs2dict(params)
node = paramdict.get('node')
if node:
node = node.replace(',', ' ').split()
else:
node = []
nodes = paramdict.get('nodes')
if not nodes:
nodes = utils.list_cluster_nodes()
nodes += node
params += ['nodes=%s' % (','.join(nodes))]
return scripts.run('add', params)
def do_add(self, context, *args):
'''
Add the given node(s) to the cluster.
Installs packages, sets up corosync and pacemaker, etc.
Must be executed from a node in the existing cluster.
'''
params = self._args_implicit(context, args, 'node')
paramdict = utils.nvpairs2dict(params)
node = paramdict.get('node')
if node:
node = node.replace(',', ' ').split()
else:
node = []
nodes = paramdict.get('nodes')
if not nodes:
nodes = utils.list_cluster_nodes()
nodes += node
params += ['nodes=%s' % (','.join(nodes))]
return scripts.run('add', params)
def do_health(self, context, *args):
'''
Extensive health check.
'''
params = self._args_implicit(context, args, 'nodes')
return scripts.run('health', params)
def do_remove(self, context, *args):
'''
Remove the given node(s) from the cluster.
'''
params = self._args_implicit(context, args, 'node')
return scripts.run('remove', params)
def do_init(self, context, *args):
'''
Initialize a cluster with the given hosts as nodes.
'''
return scripts.run('init', self._args_implicit(context, args, 'nodes'))
self.writer.add_summary(summary)
return loss, accuracy
def test_sentences(self, session, nxt_method, is_valid=True):
model_accuracy = self.model.valid_accuracy if is_valid else self.model.test_accuracy
batch_data, batch_labels, finished = nxt_method()
feed_dict = {self.model.dropout_keep_prob: 1.0, self.model.train_labels: batch_labels}
for fid in self.data_generator.fids:
feed_dict[self.model.train_dataset[fid]] = batch_data[fid]
summary, accuracy = session.run([self.model.merge_all, model_accuracy], feed_dict=feed_dict)
self.writer.add_summary(summary)
return accuracy
def predict(self, text):
resp = self.data_generator.text2vec(text, self.mes.lang)
batches = resp['batches']
resp.pop('batches')
feed_dict = {self.model.dropout_keep_prob: 1.0}
for fid in self.data_generator.fids:
feed_dict[self.model.train_dataset[fid]] = batches[0][fid]
logits = self.session.run([self.model.logits], feed_dict=feed_dict)[0]
resp['tag'] = utils.get_tag_from_logits(logits[0])
return resp
if __name__ == '__main__':
# predictor = PredictorNOLSTM('ctrip', 'web', trainable=False)
# predictor.predict(u'我有点小生气')
scripts.run()
def cmdRunScript(args):
try:
scripts.run(args[1])
except OSError:
print('Script run failed')
def do_health(self, context, *args):
'''
Extensive health check.
'''
params = self._args_implicit(context, args, 'nodes')
return scripts.run('health', params)
def do_remove(self, context, *args):
'''
Remove the given node(s) from the cluster.
'''
params = self._args_implicit(context, args, 'node')
return scripts.run('remove', params)
def do_init(self, context, *args):
'''
Initialize a cluster with the given hosts as nodes.
'''
return scripts.run('init', self._args_implicit(context, args, 'nodes'))
from scripts import run
if __name__ == '__main__':
run()
def do_run(self, context, name, *args):
'''
Run the given script.
'''
return scripts.run(name, args)
import os
from utils import Config
import argparse
from scripts import run
parser = argparse.ArgumentParser()
parser.add_argument("-m", "--model", help="Model to use")
parser.add_argument("-a", "--action", help="Action to use")
parser.add_argument("--nthreads",
'-t',
type=int,
default=2,
help="Number of threads to use")
parser.add_argument("--embedding_type", '-e', help="Type of embedding to use")
args = parser.parse_args()
config = Config('./config', args=args)
config.set(
'embedding_path',
os.path.abspath(os.path.join(os.path.curdir, './wordembeddings.word2vec')))
run(config)
def run_user_script(script):
res = scripts.run(script)
return res
