def test_target_language(self):
"""Test the target language (-t) flag."""
args = ["arg0", "-ttl", "-wTest.foo", "file.xml"]
sys.argv = args
c = util.get_config()
self.assertTrue(c.command_line.xml_input_file == "file.xml")
self.assertTrue(c.command_line.target_language == "tl")
args[1] = "-tandroid"
c = util.get_config()
self.assertTrue(c.command_line.target_language == "android")
args[1] = "-tddcpp"
c = util.get_config()
self.assertTrue(c.command_line.target_language == "ddcpp")
args[1] = "-tFoo"
try:
c.parse()
self.assertTrue(False,
"Invalid target language should result in a SystemExit.")
except SystemExit:
self.assertTrue(sys.exc_info() is not None)
return
def get_config(self,module_id,option,default,boolean=False): """ Pass-through function for convenience - module_id - - option - - default - - boolean - """ util.get_config(self.cfg,module_id,option,default,boolean)
def test_object_path(self):
"""Test the object path (-b) flag."""
args = ["arg0", "-b/TestFoo", "-ttl", "-wTest.Foo", "file.xml"]
sys.argv = args
c = util.get_config()
self.assertTrue(c.command_line.xml_input_file == "file.xml")
self.assertTrue(c.command_line.target_language == "tl")
self.assertTrue(c.command_line.object_path == "/TestFoo")
args = ["arg0", "-b", "/TestFoo", "-wTest.Foo", "-ttl", "file.xml"]
sys.argv = args
c = util.get_config()
self.assertTrue(c.command_line.object_path == "/TestFoo")
return
def test_well_known_name(self):
"""Test the well known name flag."""
args = ["arg0", "-ttl", "-wTest.My.Foo", "file.xml"]
sys.argv = args
c = util.get_config()
self.assertTrue(c.command_line.well_known_name == "Test.My.Foo")
return
def main():
hooks = [CheckTabs(), CheckIndentation()]
should_check_pep8 = get_config("check-pep8", as_bool=True, default=True)
if should_check_pep8:
hooks += [CheckPep8()]
# create temp directory for getting copies of files from staging.
# TODO: Make all the hooks operate on strings instead of files, and get rid of this.
temp_dir = tempfile.mkdtemp()
temp_dir_with_slash = temp_dir + os.sep
failure_encountered = False
try:
for filename in changed_files():
if not is_python_file(filename):
continue
relevant_hooks = [hook for hook in hooks if hook.should_process_file(filename)]
if relevant_hooks:
temp_filename = make_temp_copy(temp_dir_with_slash, filename)
for relevant_hook in relevant_hooks:
passes, error_message = relevant_hook.file_passes(temp_filename, original_filename=filename)
if not passes:
failure_encountered = True
print (error_message)
break
finally:
shutil.rmtree(temp_dir, True)
return int(failure_encountered)
def setup_training(model, batcher):
"""Does setup before starting training (run_training)"""
train_dir = os.path.join(FLAGS.log_root, "train")
if not os.path.exists(train_dir): os.makedirs(train_dir)
model.build_graph() # build the graph
if FLAGS.convert_to_coverage_model:
assert FLAGS.coverage, "To convert your non-coverage model to a coverage model, run with convert_to_coverage_model=True and coverage=True"
convert_to_coverage_model()
if FLAGS.restore_best_model:
restore_best_model()
saver = tf.train.Saver(max_to_keep=3) # keep 3 checkpoints at a time
sv = tf.train.Supervisor(logdir=train_dir,
is_chief=True,
saver=saver,
summary_op=None,
save_summaries_secs=60, # save summaries for tensorboard every 60 secs
save_model_secs=60, # checkpoint every 60 secs
global_step=model.global_step)
summary_writer = sv.summary_writer
tf.logging.info("Preparing or waiting for session...")
sess_context_manager = sv.prepare_or_wait_for_session(config=util.get_config())
tf.logging.info("Created session.")
try:
run_training(model, batcher, sess_context_manager, sv, summary_writer) # this is an infinite loop until interrupted
except KeyboardInterrupt:
tf.logging.info("Caught keyboard interrupt on worker. Stopping supervisor...")
sv.stop()
def initUI(self):
self.setGeometry(200, 200, 600, 200)
self.setWindowTitle(__name_app__ + " Config")
self.config = get_config()
self.src_path = self.config.get('main', 'source_dir')
self.dst_path = self.config.get('main', 'dest_dir')
QtGui.QLabel("Parameter", self).setGeometry(QtCore.QRect(10, 10, 140, 20))
QtGui.QLabel("Source folder: ", self).setGeometry(QtCore.QRect(10, 30, 140, 20))
QtGui.QLabel("Destination folder: ", self).setGeometry(QtCore.QRect(10, 50, 140, 20))
QtGui.QLabel("Value", self).setGeometry(QtCore.QRect(150, 10, 300, 20))
self.src = QtGui.QLabel(self.src_path, self)
self.src.setGeometry(QtCore.QRect(150, 30, 300, 20))
self.src.setFrameStyle(QtGui.QFrame.Panel | QtGui.QFrame.Sunken)
self.dst = QtGui.QLabel(self.dst_path, self)
self.dst.setGeometry(QtCore.QRect(150, 50, 300, 20))
self.dst.setFrameStyle(QtGui.QFrame.Panel | QtGui.QFrame.Sunken)
self.src_btn = QtGui.QPushButton("...", self)
self.src_btn.setGeometry(QtCore.QRect(450, 30, 30, 20))
self.src_btn.clicked.connect(lambda: self.ChgDirBtnClk(self.src))
self.dst_btn = QtGui.QPushButton("...", self)
self.dst_btn.setGeometry(QtCore.QRect(450, 50, 30, 20))
self.dst_btn.clicked.connect(lambda: self.ChgDirBtnClk(self.dst))
self.save_btn = QtGui.QPushButton("Save", self)
self.save_btn.setGeometry(QtCore.QRect((450 - 80), 80, 80, 30))
self.save_btn.clicked.connect(self.SaveBtnClk)
self.cancel_btn = QtGui.QPushButton("Cancel", self)
self.cancel_btn.setGeometry(QtCore.QRect(450, 80, 80, 30))
self.cancel_btn.clicked.connect(self.close)
def __init__(self, model, batcher, vocab, dqn = None):
"""Initialize decoder.
Args:
model: a Seq2SeqAttentionModel object.
batcher: a Batcher object.
vocab: Vocabulary object
"""
self._model = model
self._model.build_graph()
self._batcher = batcher
self._vocab = vocab
self._saver = tf.train.Saver() # we use this to load checkpoints for decoding
self._sess = tf.Session(config=util.get_config())
if FLAGS.ac_training:
self._dqn = dqn
self._dqn_graph = tf.Graph()
with self._dqn_graph.as_default():
self._dqn.build_graph()
self._dqn_saver = tf.train.Saver() # we use this to load checkpoints for decoding
self._dqn_sess = tf.Session(config=util.get_config())
_ = util.load_dqn_ckpt(self._dqn_saver, self._dqn_sess)
# Load an initial checkpoint to use for decoding
ckpt_path = util.load_ckpt(self._saver, self._sess, FLAGS.decode_from)
if FLAGS.single_pass:
# Make a descriptive decode directory name
ckpt_name = "{}-ckpt-".format(FLAGS.decode_from) + ckpt_path.split('-')[
-1] # this is something of the form "ckpt-123456"
self._decode_dir = os.path.join(FLAGS.log_root, get_decode_dir_name(ckpt_name))
else: # Generic decode dir name
self._decode_dir = os.path.join(FLAGS.log_root, "decode")
# Make the decode dir if necessary
if not os.path.exists(self._decode_dir): os.mkdir(self._decode_dir)
if FLAGS.single_pass:
# Make the dirs to contain output written in the correct format for pyrouge
self._rouge_ref_dir = os.path.join(self._decode_dir, "reference")
if not os.path.exists(self._rouge_ref_dir): os.mkdir(self._rouge_ref_dir)
self._rouge_dec_dir = os.path.join(self._decode_dir, "decoded")
if not os.path.exists(self._rouge_dec_dir): os.mkdir(self._rouge_dec_dir)
def test_no_args(self):
"""Test what happens when no arguments are given."""
sys.argv = ["DummyArg0"]
with self.assertRaises(SystemExit) as cm:
c = util.get_config()
self.assertTrue(sys.exc_info() is not None)
return
def test_well_known_name_optional(self):
"""Test the well known name flag."""
args = ["arg0", "-ttl", "file.xml"]
sys.argv = args
# non-optional for 'tl'
self.assertRaises(config.ConfigException, util.get_config)
# optional for 'ddcpp'
args[1] = "-tddcpp"
c = util.get_config()
self.assertTrue(c.command_line.well_known_name == None)
return
def post(self):
try:
title = self.request.get('title').strip()
if title:
config = util.get_config()
del config.config[title]
config.put()
memcache.delete(title, namespace='config')
# else, just redirect to this dir
self.redirect('.')
except:
self.redirect('.')
def search_track():
lines = open('data/SongswithChords.csv').read().split('\r')
api_key = util.get_config([musicovery_api])[musicovery_api]
f = open('data/songswithchords_musicoveryid.csv','w')
http = urllib3.PoolManager()
for line in lines[1::
if line.strip() == '':
continue
data = re.compile(',').split(line.strip())
artist = data[1].strip().replace(' ','%20').replace('&', '%27')
title = data[2].strip().replace(' ', '%20').replace('&', '%27')
url = musicovery_search.format(**{'title':title}, 'artist':artist, 'api_key':api_key})
def main(debug=False):
try:
util.load_logging_system()
logger = logging.getLogger(__name__)
config = util.get_config()
pidfile = config.get("PATH","pid")
pid = os.getpid()
open(pidfile,'w').write("%d"%pid)
manager.Manager(debug).run()
except Exception, ex:
logger.exception("Houston, we have a problem!")
def get_echonest_music_qualities(file_name = 'data/echo_music_songids.txt'):
'''
Function used to retrieve the information in EchoNest for all the songs
in MSD subset
'''
api_key = util.get_config([echonest_api])[echonest_api]
outfile = open('data/echo_musicovery_qualities.csv','w')
track_count, echo_count = 0, 0
with open(file_name) as f:
http = urllib3.PoolManager()
for line in f:
print line
track_count += 1
song_id = line.strip()
params = {}
params['key'] = api_key
params['id'] = song_id
url = song_profile_url.format(**params)
print url
try_again = True
count = 0
while try_again:
count += 1
r = http.request('GET', url)
jsondata = json.loads(r.data)
if jsondata['response']['status']['code'] == 3:
time.sleep(3)
else:
try_again = False
if count > 15:
print song_id + ' information not found' + r.data
jsondata['response']['status']['code'] = 5
try_again = False
if jsondata['response']['status']['code'] == 5:
print song_id + ' information not found' + r.data
if len(jsondata['response']['songs']) != 0:
echo_count += 1
interested_keys = ['energy', 'liveness', 'tempo', 'speechiness',
'acousticness', 'danceability','instrumentalness',
'loudness', 'valence','key','mode', 'time_signature']
outline = song_id
for ikey in interested_keys:
outline = outline + ',' + \
str(jsondata['response']['songs'][0]['audio_summary'][ikey])
outfile.write(outline+ '\n')
outfile.flush()
outfile.close()
print 'Outof : ' + str(track_count) + ' EchoNest had information for:' + str(echo_count)
def get(self):
try:
title = self.request.get('title').strip()
if title:
config = util.get_config()
vals = {
'item' : {
'title' : title,
'value' : config.config[title],
}
}
self.template( 'config-del.html', vals, 'admin' );
else:
self.redirect('.')
except:
self.redirect('.')
def get(self):
config = util.get_config()
title = None
value = None
if self.request.get('title'):
title = self.request.get('title')
if title in config.config:
value = config.config[title]
vals = {
'item' : {
'title' : title,
'value' : value,
}
}
self.template( 'config-form.html', vals, 'admin' );
def setup_training_discriminator(model):
"""Does setup before starting training (run_training)"""
train_dir = os.path.join(FLAGS.log_root, "train-discriminator")
if not os.path.exists(train_dir): os.makedirs(train_dir)
model.build_graph() # build the graph
saver = tf.train.Saver(max_to_keep=10) # we use this to load checkpoints for decoding
sess = tf.Session(config=util.get_config())
init = tf.global_variables_initializer()
sess.run(init)
#util.load_ckpt(saver, sess, ckpt_dir="train-discriminator")
return sess, saver,train_dir
def run_eval(model, batcher, vocab):
"""Repeatedly runs eval iterations, logging to screen and writing summaries. Saves the model with the best loss seen so far."""
model.build_graph() # build the graph
saver = tf.train.Saver(max_to_keep=3) # we will keep 3 best checkpoints at a time
sess = tf.Session(config=util.get_config())
eval_dir = os.path.join(FLAGS.log_root, "eval") # make a subdir of the root dir for eval data
bestmodel_save_path = os.path.join(eval_dir, 'bestmodel') # this is where checkpoints of best models are saved
summary_writer = tf.summary.FileWriter(eval_dir)
running_avg_loss = 0 # the eval job keeps a smoother, running average loss to tell it when to implement early stopping
best_loss = None # will hold the best loss achieved so far
while True:
_ = util.load_ckpt(saver, sess) # load a new checkpoint
batch = batcher.next_batch() # get the next batch
# run eval on the batch
t0=time.time()
results = model.run_eval_step(sess, batch)
t1=time.time()
tf.logging.info('seconds for batch: %.2f', t1-t0)
# print the loss and coverage loss to screen
loss = results['loss']
tf.logging.info('loss: %f', loss)
if FLAGS.coverage:
coverage_loss = results['coverage_loss']
tf.logging.info("coverage_loss: %f", coverage_loss)
# add summaries
summaries = results['summaries']
train_step = results['global_step']
summary_writer.add_summary(summaries, train_step)
# calculate running avg loss
running_avg_loss = calc_running_avg_loss(np.asscalar(loss), running_avg_loss, summary_writer, train_step)
# If running_avg_loss is best so far, save this checkpoint (early stopping).
# These checkpoints will appear as bestmodel-<iteration_number> in the eval dir
if best_loss is None or running_avg_loss < best_loss:
tf.logging.info('Found new best model with %.3f running_avg_loss. Saving to %s', running_avg_loss, bestmodel_save_path)
saver.save(sess, bestmodel_save_path, global_step=train_step, latest_filename='checkpoint_best')
best_loss = running_avg_loss
# flush the summary writer every so often
if train_step % 100 == 0:
summary_writer.flush()
def post(self):
title = None
value = None
config = util.get_config()
try:
# get all the incoming values
title = self.request.get('title').strip()
value = self.request.get('value').strip()
config.config[title] = value
config.put()
memcache.delete(title, namespace='config')
self.redirect('.')
except Exception, err:
vals['item'] = self.request.POST
vals['err'] = err
self.template( 'config-form.html', vals, 'admin' );
def get_echonest_id(file_name = 'data/musicovery_unique.csv'):
api_key = util.get_config([echonest_api])[echonest_api]
lines = open(file_name).read().split('\r') # weird mac issue
http = urllib3.PoolManager()
outfile = open('data/EchoMusic_musicovery.csv','w')
for line in lines:
sparams = {}
song_id, title, artist, mbid, valence, arousal = re.compile(',').split(line.strip())
print line
generic = title + ' ' + artist
generic = generic.replace('\'','%27')
generic = generic.replace('&', ' ')
splitAtSpace = re.compile('\s+').split(generic)
generic = '%20'.join(splitAtSpace)
sparams['key'] = api_key
sparams['combined'] = generic
url = song_search_url.format(**sparams)
try_again = True
count = 0
while try_again:
count += 1
r = http.request('GET', url)
jsondata = json.loads(r.data)
if jsondata['response']['status']['code'] == 3:
time.sleep(3)
else:
try_again = False
if count > 10:
print song_id + ' information not found' + r.data
jsondata['response']['status']['code'] = 5
try_again = False
if jsondata['response']['status']['code'] == 5:
print song_id + ' information not found' + r.data
if len(jsondata['response']['songs']) != 0:
dataline = unicode(line,errors='replace') + ','
dataline = dataline + jsondata['response']['songs'][0]['artist_name'].replace(',',' ') + \
',' + jsondata['response']['songs'][0]['artist_id'] + \
',' + jsondata['response']['songs'][0]['title'].replace(',',' ') + \
',' + jsondata['response']['songs'][0]['id'] + \
','+ song_id
outfile.write(dataline + '\n')
outfile.flush()
outfile.close()
def file_passes(self, temp_filename, original_filename=None):
if original_filename is None:
original_filename = temp_filename
pep8_path = os.path.join(os.path.dirname(__file__), "pep8", "pep8.py")
pep8_ignore = get_config("pep8-ignore")
pep8_command = "{pep8_path} --ignore={ignore} -r {filename}".format(pep8_path=pep8_path,
ignore=pep8_ignore,
filename=temp_filename)
pep8_out, pep8_err, pep8_rc = run_command(pep8_command)
if len(pep8_err) > 0:
return False, "# Internal error checking pep8:\n{pep8_err}\n".format(pep8_err=pep8_err)
if len(pep8_out) > 0:
assert temp_filename.endswith(original_filename)
temp_dir = temp_filename[:-len(original_filename)]
error_message = "# pep8 problems with {f}:".format(f=original_filename)
pep8_formatted = "\n".join(["# " + line.replace(temp_dir, "") for line in pep8_out.splitlines()])
return False, error_message + "\n" + pep8_formatted
return True, None
def convert_to_coverage_model():
"""Load non-coverage checkpoint, add initialized extra variables for coverage, and save as new checkpoint"""
tf.logging.info("converting non-coverage model to coverage model..")
# initialize an entire coverage model from scratch
sess = tf.Session(config=util.get_config())
print "initializing everything..."
sess.run(tf.global_variables_initializer())
# load all non-coverage weights from checkpoint
saver = tf.train.Saver([v for v in tf.global_variables() if "coverage" not in v.name and "Adagrad" not in v.name])
print "restoring non-coverage variables..."
curr_ckpt = util.load_ckpt(saver, sess)
print "restored."
# save this model and quit
new_fname = curr_ckpt + '_cov_init'
print "saving model to %s..." % (new_fname)
new_saver = tf.train.Saver() # this one will save all variables that now exist
new_saver.save(sess, new_fname)
print "saved."
exit()
def setup_training(model, batcher):
"""Does setup before starting training (run_training)"""
train_dir = os.path.join(FLAGS.log_root, "train")
if FLAGS.finetune:
if not os.path.exists(train_dir):
print (util.bcolors.OKGREEN + 'Copying See et al. pre-trained model (%s) to (%s) to be fine-tuned' % (os.path.join(FLAGS.pretrained_path, 'train'), train_dir) + util.bcolors.ENDC)
os.makedirs(train_dir)
files = glob.glob(os.path.join(os.path.join(FLAGS.pretrained_path, 'train'), "*model*"))
files.extend(glob.glob(os.path.join(os.path.join(FLAGS.pretrained_path, 'train'), "*checkpoint*")))
for file in files:
if os.path.isfile(file):
shutil.copy2(file, train_dir)
if not os.path.exists(train_dir): os.makedirs(train_dir)
model.build_graph() # build the graph
if FLAGS.convert_to_coverage_model:
assert FLAGS.coverage, "To convert your non-coverage model to a coverage model, run with convert_to_coverage_model=True and coverage=True"
convert_to_coverage_model()
if FLAGS.restore_best_model:
restore_best_model()
saver = tf.train.Saver(max_to_keep=3) # keep 3 checkpoints at a time
sv = tf.train.Supervisor(logdir=train_dir,
is_chief=True,
saver=saver,
summary_op=None,
save_summaries_secs=60, # save summaries for tensorboard every 60 secs
save_model_secs=60, # checkpoint every 60 secs
global_step=model.global_step)
summary_writer = sv.summary_writer
logging.info("Preparing or waiting for session...")
sess_context_manager = sv.prepare_or_wait_for_session(config=util.get_config())
logging.info("Created session.")
try:
run_training(model, batcher, sess_context_manager, sv, summary_writer) # this is an infinite loop until interrupted
except KeyboardInterrupt:
logging.info("Caught keyboard interrupt on worker. Stopping supervisor...")
sv.stop()
def setup_training(model, batcher, config):
train_directory = config['train_folder']
if not os.path.exists(train_directory):
os.makedirs(train_directory)
saver = tf.train.Saver(max_to_keep=3) # keep 3 checkpoints at a time
sv = tf.train.Supervisor(
logdir=train_directory,
is_chief=True,
saver=saver,
summary_op=None,
save_summaries_secs=60, # save summaries for tensorboard every 60 secs
save_model_secs=60, # checkpoint every 60 secs
#global_step=model.global_step
)
#sess_context_manager = sv.prepare_or_wait_for_session(config=util.get_config())
sess_context_manager = sv.prepare_or_wait_for_session(
config=util.get_config()) ### managed_session
with sess_context_manager as sess:
loss_lst = []
loss_long, loss_all = 0, 0
for it in range(config['train_iter']):
t1 = time()
batch = batcher.next_batch()
to_return = model.model_train(batch, sess)
loss = to_return['loss']
loss_all += loss
loss_long += loss
t2 = time()
#print(t2 - t1)
if it > 0 and it % 100 == 0:
print('iter: {}, loss: {}'.format(it, str(loss_all)[:6]))
loss_all = 0
if it % 1000 == 0 and it > 0:
loss_lst.append(loss_long)
print(loss_lst)
loss_long = 0
def setup_training(mode, generator, discriminator, generator_batcher,
discriminator_batcher):
train_dir = os.path.join(FLAGS.log_root, "train")
if not os.path.exists(train_dir):
os.makedirs(train_dir)
if FLAGS.restore_best_model:
restore_best_model()
return
saver = tf.train.Saver(max_to_keep=3) # keep 3 checkpoints at a time
supervisor = tf.train.Supervisor(
logdir=train_dir,
is_chief=True,
saver=saver,
summary_op=None,
save_summaries_secs=60, # save summaries for tensorboard every 60 secs
save_model_secs=60, # checkpoint every 60 secs
global_step=generator.global_step)
summary_writer = supervisor.summary_writer
sess_context_manager = supervisor.prepare_or_wait_for_session(
config=util.get_config())
try:
if mode == "pretrain":
trainer.pretrain_generator(generator, generator_batcher,
summary_writer, sess_context_manager)
elif mode == "train":
trainer.pretrain_discriminator(discriminator, sess_context_manager)
trainer.adversarial_train(generator, discriminator,
generator_batcher, discriminator_batcher,
summary_writer, sess_context_manager)
else:
raise ValueError("Caught invalid value of mode!")
except KeyboardInterrupt:
tf.logging.info(
"Caught keyboard interrupt on worker. Stopping supervisor...")
supervisor.stop()
def __init__(self, model, batcher, vocab):
"""Initialize decoder.
Args:
model: a Seq2SeqAttentionModel object.
batcher: a Batcher object.
vocab: Vocabulary object
"""
self._model = model
self._model.build_graph()
self._batcher = batcher
self._vocab = vocab
self._saver = tf.train.Saver() # we use this to load checkpoints for decoding
self._sess = tf.Session(config=util.get_config())
# Load an initial checkpoint to use for decoding
ckpt_path = util.load_ckpt(self._saver, self._sess)
if FLAGS.single_pass:
# Make a descriptive decode directory name
ckpt_name = "ckpt-" + ckpt_path.split('-')[-1] # this is something of the form "ckpt-123456"
self._decode_dir = os.path.join(FLAGS.log_root, get_decode_dir_name(ckpt_name))
if os.path.exists(self._decode_dir):
raise Exception("single_pass decode directory %s should not already exist" % self._decode_dir)
else: # Generic decode dir name
self._decode_dir = os.path.join(FLAGS.log_root, "decode")
# Make the decode dir if necessary
if not os.path.exists(self._decode_dir): os.mkdir(self._decode_dir)
#if FLAGS.single_pass:
if FLAGS.single_pass:
# Make the dirs to contain output written in the correct format for pyrouge
self._rouge_ref_dir = os.path.join(self._decode_dir, "reference")
if not os.path.exists(self._rouge_ref_dir): os.mkdir(self._rouge_ref_dir)
self._rouge_dec_dir = os.path.join(self._decode_dir, "decoded")
if not os.path.exists(self._rouge_dec_dir): os.mkdir(self._rouge_dec_dir)
def setup_training(model, batcher):
"""Does setup before starting training (run_training)"""
train_dir = os.path.join(FLAGS.log_root, "train")
if not os.path.exists(train_dir):
os.makedirs(train_dir)
model.build_graph() # build the graph
if FLAGS.convert_to_coverage_model:
if not FLAGS.coverage:
raise ValueError(
"To convert your non-coverage model to a coverage model, run with convert_to_coverage_model=True"
"and coverage=True")
convert_to_coverage_model()
if FLAGS.restore_best_model:
restore_best_model()
saver = tf.train.Saver(max_to_keep=3) # keep 3 checkpoints at a time
sv = tf.train.Supervisor(
logdir=train_dir,
is_chief=True,
saver=saver,
summary_op=None,
save_summaries_secs=60, # save summaries for tensorboard every 60 secs
save_model_secs=60, # checkpoint every 60 secs
global_step=model.global_step)
summary_writer = sv.summary_writer
tf.logging.info("Preparing or waiting for session...")
sess_context_manager = sv.prepare_or_wait_for_session(
config=util.get_config())
tf.logging.info("Created session.")
try:
run_training(
model, batcher, sess_context_manager, sv,
summary_writer) # this is an infinite loop until interrupted
except KeyboardInterrupt:
tf.logging.info(
"Caught keyboard interrupt on worker. Stopping supervisor...")
sv.stop()
def restore_best_model():
"""Load bestmodel file from eval directory, add variables for adagrad, and save to train directory"""
tf.logging.info("Restoring bestmodel for training...")
# Initialize all vars in the model
sess = tf.Session(config=util.get_config())
print "Initializing all variables..."
sess.run(tf.initialize_all_variables())
# Restore the best model from eval dir
saver = tf.train.Saver([v for v in tf.all_variables() if "Adagrad" not in v.name])
print "Restoring all non-adagrad variables from best model in eval dir..."
curr_ckpt = util.load_ckpt(saver, sess, "eval")
print "Restored %s." % curr_ckpt
# Save this model to train dir and quit
new_model_name = curr_ckpt.split("/")[-1].replace("bestmodel", "model")
new_fname = os.path.join(FLAGS.log_root, "train", new_model_name)
print "Saving model to %s..." % (new_fname)
new_saver = tf.train.Saver() # this saver saves all variables that now exist, including Adagrad variables
new_saver.save(sess, new_fname)
print "Saved."
exit()
def restore_best_model():
"""Load bestmodel file from eval directory, add variables for adagrad, and save to train directory"""
tf.logging.info("Restoring bestmodel for training...")
# Initialize all vars in the model
sess = tf.Session(config=util.get_config())
print("Initializing all variables...")
sess.run(tf.initialize_all_variables())
# Restore the best model from eval dir
saver = tf.train.Saver([v for v in tf.all_variables() if "Adagrad" not in v.name])
print("Restoring all non-adagrad variables from best model in eval dir...")
curr_ckpt = util.load_ckpt(saver, sess, "eval")
print ("Restored %s." % curr_ckpt)
# Save this model to train dir and quit
new_model_name = curr_ckpt.split("/")[-1].replace("bestmodel", "model")
new_fname = os.path.join(FLAGS.log_root, "train", new_model_name)
print ("Saving model to %s..." % (new_fname))
new_saver = tf.train.Saver() # this saver saves all variables that now exist, including Adagrad variables
new_saver.save(sess, new_fname)
print ("Saved.")
exit()
def restore_best_model():
print("Restoring bestmodel for training...")
# Initialize all vars in the model
sess = tf.Session(config=util.get_config())
print("Initializing all variables...")
sess.run(tf.initialize_all_variables())
# Restore the best model from eval dir
saver = tf.train.Saver(
[v for v in tf.all_variables() if "Adagrad" not in v.name])
print("Restoring all non-adagrad variables from best model in eval dir...")
curr_ckpt = util.load_ckpt(saver, sess, "val")
print("Restored %s." % curr_ckpt)
new_model_name = curr_ckpt.split("/")[-1].replace("bestmodel", "model")
new_fname = os.path.join(FLAGS.exp_name, "train", new_model_name)
print("Saving model to %s..." % (new_fname))
new_saver = tf.train.Saver()
new_saver.save(sess, new_fname)
print("Saved.")
exit()
def convert_to_coverage_model():
"""Load non-coverage checkpoint, add initialized extra variables for coverage, and save as new checkpoint"""
tf.logging.info("converting non-coverage model to coverage model..")
# initialize an entire coverage model from scratch
sess = tf.Session(config=util.get_config())
print("initializing everything...")
sess.run(tf.global_variables_initializer())
# load all non-coverage weights from checkpoint
saver = tf.train.Saver([v for v in tf.global_variables(
) if "coverage" not in v.name and "Adagrad" not in v.name])
print("restoring non-coverage variables...")
curr_ckpt = util.load_ckpt(saver, sess)
print("restored.")
# save this model and quit
new_fname = curr_ckpt + '_cov_init'
print("saving model to %s..." % (new_fname))
new_saver = tf.train.Saver() # this one will save all variables that now exist
new_saver.save(sess, new_fname)
print("saved.")
exit()
def __init__(self, model, batcher, vocab, ckpt_path):
self._model = model
self._model.build_graph()
self._batcher = batcher
self._vocab = vocab
self.ckpt_path = ckpt_path
self._saver = tf.train.Saver()
self._sess = tf.Session(config=util.get_config())
self._saver.restore(self._sess, self.ckpt_path)
print("load mode from %s" % self.ckpt_path)
self.model_num = self.ckpt_path.split('-')[-1]
ckpt_name = "ckpt-" + self.model_num # this is something of the form "ckpt-123456"
self._decode_dir = os.path.join(FLAGS.log_root,
get_infer_dir(ckpt_name))
# Make the decode dir if necessary
if not os.path.exists(self._decode_dir):
os.mkdir(self._decode_dir)
else:
raise Exception("infer directory %s should not already exist")
def infer(self):
def build_infer():
embedding = Nmt._build_embedding(self.src_vocab_size,
self.opts.embedding_size,
name="source_embedding")
input_, encoder_outputs, encoder_state = self._build_encoder(
embedding)
embedding = Nmt._build_embedding(self.tgt_vocab_size,
self.opts.embedding_size,
name="tgt_embedding")
samples, logits = self._build_decoder(encoder_outputs,
encoder_state,
embedding,
train=False)
return samples, logits
with ipu_scope('/device:IPU:0'):
data, vocab = self._build_inputs()
batch = ipu_compiler.compile(build_infer, [])
# Create a restoring object
saver = tf.train.Saver()
ipu_options = util.get_config(report_n=0)
utils.configure_ipu_system(ipu_options)
session = tf.Session()
checkpoint = CHECKPOINT_FILE + 'ckpt'
saver.restore(session, checkpoint)
# Run a dummy value to force the graph compilation
session.run(batch, feed_dict=next(data))
while True:
feed_dict = next(data)
predictions, _ = session.run(batch, feed_dict=feed_dict)
print_data(feed_dict[self.placeholders['source']], vocab[0],
predictions, vocab[1])
if not self.opts.interact:
break
def setup_training(model, batcher):
"""Does setup before starting training (run_training)"""
train_dir = os.path.join(FLAGS.log_root, "train")
if not os.path.exists(train_dir): os.makedirs(train_dir)
model.build_graph() # build the graph
if FLAGS.convert_to_coverage_model:
assert FLAGS.coverage, "To convert your non-coverage model to a coverage model, run with convert_to_coverage_model=True and coverage=True"
convert_to_coverage_model()
if FLAGS.restore_best_model:
restore_best_model()
saver = tf.train.Saver(max_to_keep=30) # keep 3 checkpoints at a time
saver_hook = tf.train.CheckpointSaverHook(
train_dir, save_steps=5000,
saver=saver) #Saves checkpoints every N steps or seconds.
summary_writer = tf.summary.FileWriter(train_dir) #将汇总的写入文件
#Saves summaries every N steps.
summary_hook = tf.train.SummarySaverHook(save_steps=20,
summary_op=model._summaries,
summary_writer=summary_writer)
tf.logging.info("Created session.")
session = tf.train.MonitoredTrainingSession(
hooks=[saver_hook, summary_hook],
config=util.get_config(),
checkpoint_dir=train_dir)
summary_writer.add_graph(session.graph)
try:
run_training(
model, batcher, session,
summary_writer) # this is an infinite loop until interrupted
except KeyboardInterrupt:
tf.logging.info(
"Caught keyboard interrupt on worker. Stopping supervisor...")
session.close()
def get_songs():
api_key = util.get_config([musicovery_api])[musicovery_api]
valences = range(50000, 1000000, 100000)
arousals = range(50000, 1000000, 100000)
f = open('data/musicovery_60.csv','w')
http = urllib3.PoolManager()
for valence in valences:
for arousal in arousals:
print 'getting songs for valence ' + str(valence) + ' arousal: ' + str(arousal)
url = musicovery_url.format(**{'valence':valence, 'arousal':arousal, 'api_key':api_key})
r = http.request('GET', url)
jsondata = json.loads(r.data)
print url
if jsondata['root']['response']['code'] != '100':
print 'ERROR out ' + jsondata['root']['response']['code']
time.sleep(2)
r = http.request('GET', url)
jsondata = json.loads(r.data)
for track in jsondata['root']['tracks']['track']:
try:
title = defaultMe(track['title'])
id = defaultMe(track['id'])
artist = defaultMe(track['artist']['name'])
artist_mbid = defaultMe(track['artist']['mbid'])
music_valence = defaultMe(track['valence'])
music_arousal = defaultMe(track['arousal'])
releasedate = defaultMe(track['releasedate'])
genre = defaultMe(track['genre'])
line = str(id) + ',' + title + ',' + artist + ',' + \
artist_mbid + ',' + str(music_valence) + ',' + \
str(music_arousal)
f.write(line + '\n')
f.flush()
except:
print 'some error for one track!!'
continue
f.close()
def setup_training(generator, discriminator, generator_batcher,
discriminator_batcher):
"""Does setup before starting training (run_training)"""
train_dir = os.path.join(FLAGS.log_root, "train")
if not os.path.exists(train_dir):
os.makedirs(train_dir)
if FLAGS.restore_best_model:
restore_best_model()
saver = tf.train.Saver(max_to_keep=5) # keep 3 checkpoints at a time
sv = tf.train.Supervisor(
logdir=train_dir,
is_chief=True,
saver=saver,
summary_op=None,
save_summaries_secs=60, # save summaries for tensorboard every 60 secs
save_model_secs=60, # checkpoint every 60 secs
global_step=generator.global_step)
summary_writer = sv.summary_writer
tf.logging.info("Preparing or waiting for session...")
sess_context_manager = sv.prepare_or_wait_for_session(
config=util.get_config())
tf.logging.info("Created session.")
try:
if FLAGS.pretrain_discriminator:
pre_train_discriminator(discriminator, sess_context_manager)
run_training(generator, discriminator, generator_batcher,
discriminator_batcher, summary_writer,
sess_context_manager)
#model, batcher, sess_context_manager, sv, summary_writer) # this is an infinite loop until interrupted
except KeyboardInterrupt:
tf.logging.info(
"Caught keyboard interrupt on worker. Stopping supervisor...")
sv.stop()
def get_echonest_analysis(file_name = 'data/echo_music_songids.txt'):
api_key = util.get_config([echonest_api])[echonest_api]
with open(file_name) as f:
for line in f:
http = urllib3.PoolManager()
song_id = line.strip()
params = {}
params['key'] = api_key
params['id'] = song_id
url = song_profile_url.format(**params)
print url
try_again = True
count = 0
while try_again:
count += 1
print count
r = http.request('GET', url)
jsondata = json.loads(r.data)
if jsondata['response']['status']['code'] == 3:
time.sleep(3)
else:
try_again = False
if count > 15:
print song_id + ' information not found' + r.data
jsondata['response']['status']['code'] = 5
try_again = False
if jsondata['response']['status']['code'] == 5:
print song_id + ' information not found' + r.data
if len(jsondata['response']['songs']) != 0:
amazon_url = jsondata['response']['songs'][0]['audio_summary']['analysis_url']
r_amazon = http.request('GET', amazon_url)
try:
json_amazon = json.loads(r_amazon.data)
json.dump(json_amazon,open('data/echo_analysis/'+song_id+'.json','w'))
except:
print 'song_id: ' + song_id + ' has no analysis data'
def test_encoder_layer_build(self, batch_size, prefix):
os.chdir(os.path.dirname(os.path.realpath(__file__)))
os.chdir('../')
from model import AMConfig
from model import ConformerAM
from util import get_config
cfg = get_config(1)
cfg.configure_ipu_system()
config_file = 'configs/train_fp32_kl_loss.yaml'
config = AMConfig.from_yaml(config_file)
config['use_ipu_dropout'] = False
model = ConformerAM(config)
x = tf.random_normal(shape=(batch_size, model.config['maxlen_in'],
model.config['adim']))
mask_adder = tf.random_normal(shape=(batch_size, 1, 1,
model.config['maxlen_in']))
pos_emb = tf.random_normal(shape=(1, model.config['maxlen_in'],
model.config['adim']))
layer = model._build_encoder_layer(x, mask_adder, pos_emb, prefix)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
output = sess.run(layer)
assert output.shape == (batch_size, model.config['maxlen_in'],
model.config['adim'])
def helper_infection_rate_param_sweep(filename,
expected_avg_list,
infection_rate_list=None):
'''
Purpose: helper function for task 6
Inputs:
filename: (str) json file to open
expected_avg_list: (list) expected value
infection_rate_list: (list) optional input parameter
'''
starting_state, random_seed, max_num_days, _, num_trials = \
util.get_config(filename)
if infection_rate_list is None:
infection_rate_list = [0, 0.25, 0.5, 0.75, 1.0]
# helper function for testing
actual_avg_list = sir.infection_rate_param_sweep(starting_state,
random_seed, max_num_days,
infection_rate_list,
num_trials)
if actual_avg_list != pytest.approx(expected_avg_list):
s = "Actual ({:}) and expected ({:}) final states do not match"
pytest.fail(s.format(actual_avg_list, expected_avg_list))
def run_decoding(model, batcher):
model.build_graph() # build the graph
saver = tf.train.Saver(
max_to_keep=3) # we will keep 3 best checkpoints at a time
sess = tf.Session(config=util.get_config())
decode_dir = os.path.join(
FLAGS.exp_name,
"decode") # make a subdir of the root dir for eval data
batches = batcher.getBatches()
_ = util.load_ckpt(saver, sess) # load a new checkpoint
epoch_avg_loss = 0.
epoch_decode_steps = 0
for batch in batches:
results = model.run_decode_step(sess, batch)
loss = results['loss']
epoch_decode_steps += 1
epoch_avg_loss = (epoch_avg_loss * (epoch_decode_steps - 1.) +
loss) / epoch_decode_steps
print("Average loss %f" % (epoch_avg_loss))
def __init__(self, model, vocab,single_pass,hps,pointer_gen,log_root):
"""Initialize decoder.
Args:
model: a Seq2SeqAttentionModel object.
batcher: a Batcher object.
vocab: Vocabulary object
"""
self._model = model
self._model.build_graph()
# self._batcher = batcher
self._vocab = vocab
self._saver = tf.train.Saver() # we use this to load checkpoints for decoding
self._sess = tf.Session(config=util.get_config())
self.single_pass=single_pass
self. max_dec_steps=hps.max_dec_steps
self.min_dec_steps=hps.min_dec_steps
self.max_dec_steps=hps.max_dec_steps
self.beam_size=hps.beam_size
self.pointer_gen=pointer_gen
# Load an initial checkpoint to use for decoding
ckpt_path = util.load_ckpt(self._saver, self._sess,log_root)
print ckpt_path
def get_cluster() -> Cluster:
"""
Get the cluster to connect to. Because we only ever want to connect to a
single cassandra cluster, the output of this function is cached.
Effectively, the cluster is a singleton.
"""
config = get_config()
ips = config['db_addresses']
port = int(config['db_port'])
keyspace = config['db_keyspace']
username = config['db_username']
password = config['db_password']
auth: PlainTextAuthProvider = PlainTextAuthProvider(username, password)
cluster: Cluster = Cluster(ips, port=port, auth_provider=auth)
cluster.register_user_type(keyspace, 'class_session', ClassSession)
cluster.register_user_type(keyspace, 'class_stream', ClassStream)
cluster.register_user_type(keyspace, 'class', Class)
return cluster
def main():
"""
Main function, it will start the timer, instantiate de logger
and inititate de build/get process
:return:
"""
try:
start = time.time()
config = util.get_config()
set_globals(config)
util.change_cwd(BASE_PATH)
except Exception as err:
print(err)
try:
create_logger()
except Exception as err:
print(err)
return
logger.info('Process starded....')
print('####################')
is_proc_ok = start_process(config)
minutes = (time.time() - start) / 60
if is_proc_ok:
util.print_and_log(logger.info,
'Complete process took {} minutes'.format(minutes))
else:
util.print_and_log(
logger.info,
'Complete process took {} minutes, with ERRORS. Check log.'.format(
minutes))
def config_collection(shutit): """Collect core config from config files for all seen modules. """ cfg = shutit.cfg shutit_map = shutit.shutit_map for mid in module_ids(shutit): # Default to None so we can interpret as ifneeded util.get_config(cfg,mid,'build',None,boolean=True) util.get_config(cfg,mid,'remove',False,boolean=True) util.get_config(cfg,mid,'do_repository_work',False,boolean=True) # ifneeded will (by default) only take effect if 'build' is not specified # It can, however, be forced to a value, but this should be unusual if cfg[mid]['build'] is None: shutit.get_config(mid,'build_ifneeded',True,boolean=True) cfg[mid]['build'] = False else: shutit.get_config(mid,'build_ifneeded',False,boolean=True)
def copy_checkpoint(source, target):
for ext in (".index", ".data-00000-of-00001"):
shutil.copyfile(source + ext, target + ext)
if __name__ == "__main__":
# util.set_gpus()
if len(sys.argv) > 1:
name = sys.argv[1]
print("Running experiment: {} (from command-line argument).".format(name))
else:
name = os.environ["EXP"]
print("Running experiment: {} (from environment variable).".format(name))
config = util.get_config("experiments.conf")[name]
config["log_dir"] = util.mkdirs(os.path.join(config["log_root"], name))
# Dynamic batch size.
config["batch_size"] = -1
config["max_tokens_per_batch"] = -1
# Use dev lm, if provided.
if config["lm_path"] and "lm_path_dev" in config and config["lm_path_dev"]:
config["lm_path"] = config["lm_path_dev"]
util.print_config(config)
data = LSGNData(config)
model = SRLModel(data, config)
evaluator = LSGNEvaluator(config)
# -*- coding: utf-8 -*-
import wrap
# 用于输出运行日志
import logging
logging.basicConfig(
level=logging.INFO,
format=
'127.0.0.1 - - [%(asctime)s - %(name)s - %(levelname)s - %(message)s]')
logger = logging.getLogger(__name__)
# 数据库连接信息
import mysql.connector
import util
config = util.get_config()
USER = config["user"]
PASSWORD = config["password"]
DATABASE = config["database"]
# 连接mysql数据库
def get_connect():
conn = mysql.connector.connect(user=USER,
password=PASSWORD,
database=DATABASE)
return conn
# ==============TODO留作参考注册函数=======================
# def save_user(username, password):
#!/bin/env python
# -*- encoding: utf-8 -*-
# 导入web目录__init__ 创建flask实例
from web import app
import os, sys, logging, logging.config
import db, util
work_dir = os.path.dirname(os.path.realpath(__file__))
app.secret_key = 'A0Zr98j/3yX R~XHH!jmN]LWX/,?RT'
service_conf = os.path.join(work_dir, 'conf/service.conf')
config = util.get_config(service_conf, 'web')
#print config
#将参数追加到app.config字典中，就可以随意调用了
app.config.update(config)
if __name__ == '__main__':
app.run(host=config.get('bind', '0.0.0.0'),
port=int(config.get('port')),
debug=True)
pred_answers.append(best_ans)
f1s.append(f1)
hits.append(hit)
print('evaluation.......')
print('how many eval samples......', len(f1s))
print('avg_f1', np.mean(f1s))
print('avg_hits', np.mean(hits))
model.train()
return np.mean(f1s), np.mean(hits)
if __name__ == "__main__":
# config_file = sys.argv[2]
cfg = get_config()
random.seed(cfg['seed'])
np.random.seed(cfg['seed'])
torch.manual_seed(cfg['seed'])
torch.cuda.manual_seed_all(cfg['seed'])
if cfg['mode'] == 'train':
train(cfg)
elif cfg['mode'] == 'test':
documents = load_documents(cfg['data_folder'] +
cfg['{}_documents'.format(cfg['mode'])])
test_data = DataLoader(cfg, documents, mode='test')
model = KAReader(cfg)
model = model.to(torch.device('cuda'))
model_save_path = 'model/{}/{}_best.pt'.format(cfg['name'],
cfg['model_id'])
model.load_state_dict(torch.load(model_save_path))
def __init__(self, model, batcher, vocab):
"""Initialize decoder.
Args:
model: a SentSelector object.
batcher: a Batcher object.
vocab: Vocabulary object
"""
# get the data split set
if "train" in FLAGS.data_path: self._dataset = "train"
elif "val" in FLAGS.data_path: self._dataset = "val"
elif "test" in FLAGS.data_path: self._dataset = "test"
else:
raise ValueError(
"FLAGS.data_path %s should contain one of train, val or test" %
(FLAGS.data_path))
# create the data loader
self._batcher = batcher
if FLAGS.eval_gt_rouge: # no need to load model, default is fasle
# Make a descriptive decode directory name
self._decode_dir = os.path.join(FLAGS.log_root,
'select_gt' + self._dataset)
tf.logging.info('Save evaluation results to ' + self._decode_dir)
if os.path.exists(self._decode_dir):
raise Exception(
"single_pass decode directory %s should not already exist"
% self._decode_dir)
# Make the decode dir
os.makedirs(self._decode_dir)
# Make the dirs to contain output written in the correct format for pyrouge
self._rouge_ref_dir = os.path.join(self._decode_dir, "reference")
if not os.path.exists(self._rouge_ref_dir):
os.mkdir(self._rouge_ref_dir)
self._rouge_gt_dir = os.path.join(self._decode_dir, "gt_selected")
if not os.path.exists(self._rouge_gt_dir):
os.mkdir(self._rouge_gt_dir)
else: # FALSE
self._model = model
self._model.build_graph()
self._vocab = vocab
self._saver = tf.train.Saver(
) # we use this to load checkpoints for decoding
self._sess = tf.Session(config=util.get_config())
# Load an initial checkpoint to use for decoding
print(" eval_ckpt_path ", FLAGS.eval_ckpt_path)
if FLAGS.load_best_eval_model:
tf.logging.info('Loading best eval checkpoint')
ckpt_path = util.load_ckpt(self._saver,
self._sess,
ckpt_dir='eval')
elif FLAGS.eval_ckpt_path:
ckpt_path = util.load_ckpt(self._saver,
self._sess,
ckpt_path=FLAGS.eval_ckpt_path,
ckpt_dir='eval')
else:
tf.logging.info('Loading best train checkpoint')
ckpt_path = util.load_ckpt(self._saver, self._sess)
if FLAGS.single_pass:
# Make a descriptive decode directory name
ckpt_name = "ckpt-" + ckpt_path.split('-')[
-1] # this is something of the form "ckpt-123456"
decode_root_dir, decode_dir = get_decode_dir_name(
ckpt_name, self._dataset)
self._decode_root_dir = os.path.join(FLAGS.log_root,
decode_root_dir)
self._decode_dir = os.path.join(FLAGS.log_root,
decode_root_dir, decode_dir)
tf.logging.info('Save evaluation results to ' +
self._decode_dir)
if os.path.exists(self._decode_dir):
raise Exception(
"single_pass decode directory %s should not already exist"
% self._decode_dir)
else: # Generic decode dir name
self._decode_dir = os.path.join(FLAGS.log_root, "select")
# Make the decode dir if necessary
if not os.path.exists(self._decode_dir):
os.makedirs(self._decode_dir)
if FLAGS.single_pass:
# Make the dirs to contain output written in the correct format for pyrouge
self._rouge_ref_dir = os.path.join(self._decode_dir,
"reference")
if not os.path.exists(self._rouge_ref_dir):
os.mkdir(self._rouge_ref_dir)
self._rouge_dec_dir = os.path.join(self._decode_dir,
"selected")
if not os.path.exists(self._rouge_dec_dir):
os.mkdir(self._rouge_dec_dir)
if FLAGS.save_pkl:
self._result_dir = os.path.join(self._decode_dir,
"select_result")
if not os.path.exists(self._result_dir):
os.mkdir(self._result_dir)
self._probs_pkl_path = os.path.join(self._decode_root_dir,
"probs.pkl")
if not os.path.exists(self._probs_pkl_path):
self._make_probs_pkl = True
else:
self._make_probs_pkl = False
self._precision = []
self._recall = []
self._accuracy = []
self._ratio = []
self._select_sent_num = []
from api import app, zabbix_api, zabbix_Graph_api
import os,sys,logging,logging.config
import db,util
from celery import Celery,platforms
#session使用需要设置secret_key
app.secret_key = 'A0Zr98j/3yX R~XHH!jmN]LWX/,?RT'
#导入自定义的各种配置参数，最终参数以字典形式返回
work_dir = os.path.dirname(os.path.realpath(__file__))
service_conf = os.path.join(work_dir, 'conf/service.conf')
img = os.path.join(work_dir,'web/static/zabbix')
img_url = {"zabbix_img_url":img}
config = util.get_config(service_conf, 'api')
cobbler_config = util.get_config(service_conf, 'cobbler')
zabbix_config = util.get_config(service_conf, 'zabbix')
#将参数追加到app.config字典中，就可以随意调用了
app.config.update(config)
app.config.update(cobbler_config)
app.config.update(img_url)
app.config.update(zabbix_config)
#print app.config
#实例化数据库类，并将实例化的对象导入配置
app.config['cursor'] = db.Cursor(config)
# エキスパートPythonプログラミング 改訂2版 13章
# マルチスレッドその２
# ワーカースレッド内で発生した例外をメインスレッドに返す
import time
from gmaps import Geocoding
from threading import Thread, current_thread
from queue import Queue, Empty
from util import PLACES, get_config
api = Geocoding(api_key=get_config()["geocode_api_key"])
def fetch_place(place):
geocoded = api.geocode(place)[0]
return geocoded, current_thread().name
def present_result(geocoded, thread_name):
print(
f'{geocoded["formatted_address"]:>60s}, {geocoded["geometry"]["location"]["lat"]:6.2f}, {geocoded["geometry"]["location"]["lng"]:6.2f} [{thread_name}]'
)
def worker(work_queue, results_queue):
# 各々のスレッドは、work_queue から取り出して処理する
# どの都市をどのスレッドが処理するかは、その時次第
while not work_queue.empty():
try:
item = work_queue.get(block=False)
except Empty:
my_model = use_cuda(
GraftNet(word_emb_file, entity_emb_file, entity_kge_file,
relation_emb_file, relation_kge_file, cfg['num_layer'],
num_kb_relation, num_entities, num_vocab, cfg['entity_dim'],
cfg['word_dim'], cfg['kge_dim'], cfg['pagerank_lambda'],
cfg['fact_scale'], cfg['lstm_dropout'], cfg['linear_dropout'],
cfg['use_kb'], cfg['use_doc']))
if cfg['load_model_file'] is not None:
print('loading model from', cfg['load_model_file'])
pretrained_model_states = torch.load(cfg['load_model_file'])
if word_emb_file is not None:
del pretrained_model_states['word_embedding.weight']
if entity_emb_file is not None:
del pretrained_model_states['entity_embedding.weight']
my_model.load_state_dict(pretrained_model_states, strict=False)
return my_model
if __name__ == "__main__":
config_file = sys.argv[2]
CFG = get_config(config_file)
if '--train' == sys.argv[1]:
train(CFG)
elif '--test' == sys.argv[1]:
test(CFG)
else:
assert False, "--train or --test?"
def run_eval(model, batcher, re_vocab, embed):
"""Repeatedly runs eval iterations, logging to screen and writing summaries. Saves the model with the best loss seen so far."""
model.build_graph(embed) # build the graph
saver = tf.train.Saver(
max_to_keep=10) # we will keep 3 best checkpoints at a time
sess = tf.Session(config=util.get_config())
eval_dir = os.path.join(
FLAGS.log_root, "eval") # make a subdir of the root dir for eval data
bestmodel_save_path = os.path.join(
eval_dir,
'bestmodel') # this is where checkpoints of best models are saved
summary_writer = tf.summary.FileWriter(eval_dir)
running_avg_loss = 0 # the eval job keeps a smoother, running average loss to tell it when to implement early stopping
best_loss = None # will hold the best loss achieved so far
count = 0
while True:
_ = util.load_ckpt(saver, sess) # load a new checkpoint
batch = batcher.next_batch() # get the next batch
if not os.path.exists(FLAGS.model_dir):
os.makedirs(FLAGS.model_dir)
os.makedirs(FLAGS.system_dir)
# run eval on the batch
t0 = time.time()
# results = model.run_eval_step(sess, batch)
step_output = model.run_eval_step(sess, batch)
t1 = time.time()
#tf.logging.info('seconds for batch: %.2f', t1-t0)
(summaries, loss, train_step) = step_output[0]
(out_decoder_outputs, out_sent_decoder_outputs,
final_dists) = step_output[1]
(step_loss, word_loss, sent_loss, word_loss_null, sent_loss_null,
switch_loss) = step_output[2]
coverage_loss = 0.0
running_avg_loss = calc_running_avg_loss(np.asscalar(loss),
running_avg_loss,
summary_writer, train_step)
if best_loss is None or running_avg_loss < (best_loss):
if best_loss is None:
best_loss = 0.0
tf.logging.info(
'Found new best model with %.3f running_avg_loss. Saving to %s',
running_avg_loss, bestmodel_save_path)
saver.save(sess,
bestmodel_save_path,
global_step=train_step,
latest_filename='checkpoint_best')
best_loss = running_avg_loss
last_step = train_step
tf.logging.info('loss: %f rloss: %f', loss,
rloss) # print the loss to screen
tf.logging.info(
'step_loss: %f word_loss: %f ,sent_loss: %f ,word_loss_null: %f,sent_loss_null: %f ,switch_loss: %f,cover_loss: %f',
step_loss, word_loss, sent_loss, word_loss_null, sent_loss_null,
switch_loss, coverage_loss)
os.system("rm -rf " + FLAGS.model_dir + ' ' + FLAGS.system_dir)
count = count + 1
if train_step % 100 == 0:
summary_writer.flush()
def main():
hooks = [CheckTabs(), CheckIndentation()]
debug = get_config("debug", as_bool=True, default=False)
should_check_pep8 = get_config("check-pep8", as_bool=True, default=True)
if should_check_pep8:
hooks += [CheckPep8()]
incremental = get_config("incremental", as_bool=True, default=False)
incremental_verbose = get_config("incremental.verbose", as_bool=True,
default=False)
if debug:
print "Starting hooks, with pep8 %s, incremental %s, hooks [%s]" % (should_check_pep8, incremental, ", ".join(map(str, hooks)))
# create temp directory for getting copies of files from staging.
# TODO: Make all the hooks operate on strings instead of files, and get rid of this.
temp_dir = tempfile.mkdtemp()
temp_dir_with_slash = temp_dir + os.sep
failure_encountered = False
try:
for filename in changed_files():
if debug:
print "Examining %s" % filename
if not is_python_file(filename):
if debug:
print "Skipping %s, not a python file" % filename
continue
relevant_hooks = [hook for hook in hooks if hook.should_process_file(filename)]
if not relevant_hooks:
if debug:
print "Skipping %s, no relevant hooks" % filename
continue
if incremental:
head_temp_filename = make_temp_copy(temp_dir_with_slash, filename, head=True)
incremental_hooks = copy.copy(relevant_hooks)
if os.path.isfile(head_temp_filename):
# This is not a newly added file, so check whether it used to fail the hooks.
for relevant_hook in relevant_hooks:
head_passes, unused_error_message = relevant_hook.file_passes(head_temp_filename, original_filename=filename)
if not head_passes:
# Incremental checking was requested, and current HEAD doesn't pass,
# so don't bother checking this file with this hook.
incremental_hooks.remove(relevant_hook)
if incremental_verbose:
print "Hook %s failed on current HEAD for file %s"%(relevant_hook, filename)
relevant_hooks = incremental_hooks
if not relevant_hooks:
if debug:
print "Skipping %s, no relevant hooks (after incremental check)" % filename
continue
temp_filename = make_temp_copy(temp_dir_with_slash, filename)
for relevant_hook in relevant_hooks:
passes, error_message = relevant_hook.file_passes(temp_filename, original_filename=filename)
if not passes:
failure_encountered = True
print(error_message)
break
finally:
shutil.rmtree(temp_dir, True)
return int(failure_encountered)
import os
import sys
import subprocess
import logging
import archive
from time import sleep
import util
config = util.get_config()
bash_path = config.get('treasurecolumn', 'bash_path')
filter_shell_script = config.get('treasurecolumn', 'filter_shell_script')
tmp_mp3 = config.get('treasurecolumn', 'tmp_mp3')
tmp_processed_mp3 = config.get('treasurecolumn', 'tmp_processed_mp3')
audio_filter_schell_script = config.get('treasurecolumn', 'audio_filter_schell_script')
ffmpeg_location = config.get('treasurecolumn', 'ffmpeg_location')
def make_movie(input_dir, output_file, frame_count, rate=5):
files = map(lambda f: input_dir + '/' + f,
filter(lambda f: f.endswith('.jpg'), os.listdir(input_dir)))
queue = []
done = 0
for i, jpg in enumerate(files):
parts = os.path.split(jpg)
num = parts[1].split('.')[0]
newfilename = os.path.join(parts[0], 'processed_%s.jpg' % num.zfill(5))
phase = 'diag4' if i % 2 else 'diag42'
p = subprocess.Popen([bash_path, filter_shell_script, jpg, newfilename, phase, '&'])
queue.append(p)
p.communicate()
import os
from datetime import datetime
from enum import Enum
from fabric.api import run, env, task
from fabric.colors import yellow
from fabric.contrib.console import confirm
from fabric.utils import puts, error
from definitions import ROOT_DIR, CONFIG_PATH
from util import adjust_expanduser, get_config
config = get_config(CONFIG_PATH)
env.user = config.get('ssh', 'user')
env.hosts = config.get('hosts', env.dest).split(',') if 'dest' in env else None
env.gateway = config.get('hosts', 'step', fallback=None)
env.ssh_config_path = adjust_expanduser(config.get('ssh', 'config_path'))
env.use_ssh_config = True
MYSQL_EXEC = config.get('mysql', 'exec')
class Style(Enum):
txt = {
'message': 'result as text file',
'format': '.txt',
'encode': 'utf-8'
}
csv = {'message': 'result as csv', 'format': '.csv', 'encode': 'utf-8'}
excel = {
def main(_):
# Configuration.
num_unrolls = FLAGS.num_steps
if FLAGS.seed:
tf.set_random_seed(FLAGS.seed)
# Problem.
# problem, net_config, net_assignments = util.get_config(FLAGS.problem,
# FLAGS.path)
param_dict = {}
param_dict['bs'] = FLAGS.bs
param_dict['m'] = FLAGS.m
param_dict['n'] = FLAGS.n
print(param_dict)
problem, net_config, net_assignments = util.get_config(
FLAGS.problem,
net_name="RNNprop",
mode=FLAGS.mode, #加入mode
num_linear_heads=1,
init=FLAGS.init,
path=FLAGS.path,
param=param_dict)
# Optimizer setup.
if FLAGS.optimizer == "Adam":
cost_op = problem()
problem_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)
problem_reset = tf.variables_initializer(problem_vars)
optimizer = tf.train.AdamOptimizer(FLAGS.learning_rate)
optimizer_reset = tf.variables_initializer(optimizer.get_slot_names())
update = optimizer.minimize(cost_op)
reset = [problem_reset, optimizer_reset]
elif FLAGS.optimizer == "L2L":
if FLAGS.path is None:
logging.warning("Evaluating untrained L2L optimizer")
optimizer = meta.MetaOptimizer(FLAGS.num_mt, FLAGS.beta1, FLAGS.beta2,
**net_config)
# meta_loss = optimizer.meta_loss(problem, 1, net_assignments=net_assignments)
meta_loss, _, _, _, _, seq_step, \
_, _, _, _, _, _ = optimizer.meta_loss(problem, 1, net_assignments=net_assignments)
#这里原来是各种名字的变量的，但是似乎object never used就是指这些，那我就全部用下划线变量代替了
_, update, reset, cost_op, _ = meta_loss
else:
raise ValueError("{} is not a valid optimizer".format(FLAGS.optimizer))
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with ms.MonitoredSession() as sess:
# with tf.Session(config=config) as sess:
sess.run(reset)
# Prevent accidental changes to the graph.
tf.get_default_graph().finalize()
total_time = 0
total_cost = 0
loss_record = []
for ep in xrange(FLAGS.num_epochs):
# Training.
time, cost = util.run_eval_epoch(sess,
cost_op, [update],
num_unrolls,
step=seq_step)
total_time += time
total_cost += sum(cost) / num_unrolls
loss_record += cost
print(ep, cost[-1])
# Results.
util.print_stats("Epoch {}".format(FLAGS.num_epochs), total_cost,
total_time, FLAGS.num_epochs)
with open(
'{}/{}_eval_loss_record.pickle'.format(FLAGS.path,
FLAGS.optimizer),
'wb') as l_record:
pickle.dump(loss_record, l_record)
print("Saving evaluate loss record {}".format(FLAGS.path))
def main(_):
# Configuration.
# Problem.
with tf.variable_scope("problem",
partitioner=tf.min_max_variable_partitioner(
max_partitions=2, min_slice_size=10 << 10)):
problem, net_config, net_assignments = util.get_config(FLAGS.problem)
loss = problem()
global_step = tf.Variable(0, dtype=tf.int64, trainable=False)
# Optimizer setup.
var_list = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
scope='problem')
print(var_list)
#adam_opt = tf.train.GradientDescentOptimizer(FLAGS.learning_rate)
adam_opt = tf.train.AdamOptimizer(FLAGS.learning_rate)
opt = adam_opt.minimize(loss, global_step)
if FLAGS.mode != 1:
optimizer = l2l_optimizer.L2LOptimizer(
internal_optimizer=adam_opt,
loss_func=problem,
opt_last=FLAGS.opt_last,
preprocessor=LogAndSign(10),
co_opt=FLAGS.co_opt,
rnn_layer_cnt=FLAGS.layer,
delta_ratio=FLAGS.delta_ratio,
update_ratio=FLAGS.update_ratio,
dynamic_unroll=FLAGS.dynamic_unroll)
opt = optimizer.minimize(loss,
global_step=global_step,
unroll_len=FLAGS.unroll_len)
if FLAGS.mode == 1:
print('use adam opt')
else:
print('use l2l opt')
slot_reset = tf.no_op()
if FLAGS.mode != 1:
slot_reset = tf.variables_initializer(optimizer._slot_vars +
optimizer._opt_vars)
init = tf.group(
*[tf.global_variables_initializer(),
tf.local_variables_initializer()])
var_list = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)
print(var_list)
#saver = tf.train.Saver(var_list = var_list)
with ms.MonitoredSession() as sess:
#with tf.Session() as sess:
# Prevent accidental changes to the graph.
tf.get_default_graph().finalize()
sess.run(init)
print('trainable variables')
trainable_vars = tf.trainable_variables()
for v in trainable_vars:
print("parameter:", v.name, "device:", v.device, "shape:",
v.get_shape())
best_evaluation = float("inf")
total_time = 0
accum_loss = 0.0
total_cost = 0
for e in xrange(FLAGS.num_epochs):
# Training.
step, curr_loss, _ = sess.run([global_step, loss, opt])
accum_loss += curr_loss
if step % 100 == 0:
print('step:%d,loss:%f' % (step, accum_loss / 100))
accum_loss = 0
if step % FLAGS.reset_interval == 0:
#print('reset')
sess.run(slot_reset)
def main():
#hooks = [CheckTabs(), CheckIndentation()]
hooks = [CheckComment()]
#debug = get_config("debug", as_bool=True, default=False)
debug = False # @TODO: hard code for now, use get_config later on
incremental = get_config("incremental", as_bool=True, default=False)
incremental_verbose = get_config("incremental.verbose", as_bool=True, default=False)
if debug:
print "Starting hooks, incremental %s, hooks [%s]" % (incremental, ", ".join(map(str, hooks)))
# create temp directory for getting copies of files from staging.
# TODO: Make all the hooks operate on strings instead of files, and get rid of this.
temp_dir = tempfile.mkdtemp()
temp_dir_with_slash = temp_dir + os.sep
atexit.register(shutil.rmtree, temp_dir, True) # clean up after ourselves
failure_encountered = False
if incremental:
# Incremental checking requires checking the previous version of a
# file for errors, but if the file was added, we can't do that.
# Get a list of non-added changed files here to check against.
modified_files = frozenset(changed_files(include_added_files=False))
for filename in changed_files():
if debug:
print "Examining %s" % filename
if not is_file_type(filename, 'scala'):
if debug:
print "Skipping %s, not a scala file" % filename
continue
relevant_hooks = [hook for hook in hooks if hook.should_process_file(filename)]
if debug:
print "relevant hooks are: ", relevant_hooks
if not relevant_hooks:
if debug:
print "Skipping %s, no relevant hooks" % filename
continue
if incremental and filename in modified_files:
head_temp_filename = make_temp_copy(temp_dir_with_slash, filename)
incremental_hooks = copy.copy(relevant_hooks)
if os.path.isfile(head_temp_filename):
# This is not a newly added file, so check whether it used to fail the hooks.
for relevant_hook in relevant_hooks:
head_passes, unused_error_message = relevant_hook.file_passes(head_temp_filename, original_filename=filename)
if not head_passes:
# Incremental checking was requested, and current HEAD doesn't pass,
# so don't bother checking this file with this hook.
incremental_hooks.remove(relevant_hook)
if incremental_verbose:
print "Hook %s failed on current HEAD for file %s" % (relevant_hook, filename)
relevant_hooks = incremental_hooks
if not relevant_hooks:
if debug:
print "Skipping %s, no relevant hooks (after incremental check)" % filename
continue
temp_filename = make_temp_copy(temp_dir_with_slash, filename)
for relevant_hook in relevant_hooks:
passes, error_message = relevant_hook.file_passes(temp_filename, original_filename=filename)
print(filename + " --> " + error_message)
if not passes:
failure_encountered = True
break
return int(failure_encountered)
