def check_and_update_feed(feed_list, project_list, cache, debug, regex, multiproject):
"""
Checks every feed entry in the list against project list cache and
then updates the dictionary then writes the cache file to the disk.
- feed_list is a list of feed (from feedparser module)
- project_list is the list of project as read from the yaml
configuration file
- cache is an initialized instance of FileCache
"""
# Lowers the list before searching in it
project_list_low = lower_list_of_strings(project_list)
# Checking every feed entry that are newer than the last check
# and updates the dictionary accordingly
for entry in feed_list:
titles = split_multiproject_title_into_list(entry.title, multiproject)
for title in titles:
(project, version) = cut_title_in_project_version(title, regex)
common.print_debug(debug, u'\tChecking {0:16}: {1}'.format(project, version))
if project.lower() in project_list_low:
cache.print_if_newest_version(project, version, debug)
cache.update_cache_dict(project, version, debug)
cache.write_cache_file()
def retry_tmpfunc():
# Using develop=True to not authenticate to the server
pdc_session = PDCClient(PDC_SERVER,
ssl_verify=True,
develop=True)
print_debug(pdc_session, pdc_query)
return pdc_session(**pdc_query)
def check_versions_for_list_sites(feed_project_list, url, cache_filename,
feed_filename, local_dir, debug, regex,
multiproject):
"""
Checks projects of 'list' type sites such as freshcode's web site's RSS
"""
freshcode_cache = caches.FileCache(local_dir, cache_filename)
feed_info = caches.FeedCache(local_dir, feed_filename)
feed_info.read_cache_feed()
feed = common.get_feed_entries_from_url(url)
if feed is not None:
common.print_debug(debug,
u'\tFound {} entries'.format(len(feed.entries)))
feed_list = common.make_list_of_newer_feeds(feed, feed_info, debug)
common.print_debug(
debug, u'\tFound {} new entries (relative to {})'.format(
len(feed_list), feed_info.date_minutes))
check_and_update_feed(feed_list, feed_project_list, freshcode_cache,
debug, regex, multiproject)
# Updating feed_info with the latest parsed feed entry date
feed_info.update_cache_feed(feed.entries[0].published_parsed)
feed_info.write_cache_feed()
def check_and_update_feed(feed_list, project_list, cache, debug, regex,
multiproject):
"""
Checks every feed entry in the list against project list cache and
then updates the dictionary then writes the cache file to the disk.
- feed_list is a list of feed (from feedparser module)
- project_list is the list of project as read from the yaml
configuration file
- cache is an initialized instance of FileCache
"""
# Lowers the list before searching in it
project_list_low = lower_list_of_strings(project_list)
# Checking every feed entry that are newer than the last check
# and updates the dictionary accordingly
for entry in feed_list:
titles = split_multiproject_title_into_list(entry.title, multiproject)
for title in titles:
(project, version) = cut_title_in_project_version(title, regex)
common.print_debug(
debug, u'\tChecking {0:16}: {1}'.format(project, version))
if project.lower() in project_list_low:
cache.print_if_newest_version(project, version, debug)
cache.update_cache_dict(project, version, debug)
cache.write_cache_file()
def __generateDepModules_solver(self, parentdict):
deps = self.__get_module_requires()
print_debug("tree traverse from %s: %s" % (self.name, deps))
for dep in deps:
if dep not in parentdict:
parentdict[dep] = deps[dep]
a = PDCParser(dep, deps[dep])
a.__generateDepModules_solver(parentdict=parentdict)
def test_loader():
"""
Test general backend loader for complex case
:return:
"""
mt = MetadataLoader(location=__TC_GENERAL_COMPONENT)
print_debug(yaml.dump(mt.get_metadata()))
print_debug(mt.get_backends())
assert 'sanity/generaltest.py' in [x[SOURCE] for x in mt.backend_tests()]
def check_versions(versions_conf, byproject_site_list):
"""
Checks version by checking each project's feed.
"""
for site_name in byproject_site_list:
common.print_debug(versions_conf.options.debug, u'Checking {} projects'.format(site_name))
(project_list, project_url, cache_filename, site_entry) = versions_conf.get_infos_for_site(site_name)
feed_filename = u'{}.feed'.format(site_name)
check_versions_feeds_by_projects(project_list, versions_conf.local_dir, versions_conf.options.debug, project_url, cache_filename, feed_filename, site_entry)
def test_filter_mtf_justtests():
"""
tests load configu and just tests there
"""
out = filtertests(backend="mtf",
location=__TC_MTF_COMPOMENT,
linters=False,
tests=["*.py"],
tags=[],
relevancy="")
print_debug(out)
assert len(out) == 11
def getDockerFile(dir_name):
fromenv = os.environ.get("DOCKERFILE")
if fromenv:
dockerfile = fromenv
dir_name = os.path.dirname(dockerfile)
else:
dockerfile = os.path.join(dir_name, DOCKERFILE)
if not os.path.exists(dockerfile):
dockerfile = None
common.print_debug("Dockerfile should exists in the %s directory." %
dir_name)
return dockerfile
def check_versions(versions_conf, list_site_list):
"""
Checks version by checking each project's feed.
"""
for site_name in list_site_list:
common.print_debug(versions_conf.options.debug, u'Checking {} updates'.format(site_name))
(project_list, project_url, cache_filename, project_entry) = versions_conf.get_infos_for_site(site_name)
regex = versions_conf.extract_regex_from_site(site_name)
multiproject = versions_conf.extract_multiproject_from_site(site_name)
feed_filename = u'{}.feed'.format(site_name)
check_versions_for_list_sites(project_list, project_url, cache_filename, feed_filename, versions_conf.local_dir, versions_conf.options.debug, regex, multiproject)
def test_filter_mtf_filtered_tests_add():
"""
tests load config with filters
:return:
"""
out = filtertests(backend="mtf",
location=__TC_MTF_COMPOMENT,
linters=False,
tests=["*.py"],
tags=["add"],
relevancy="")
print_debug(out)
assert len(out) == 6
def test_filter_general():
"""
tests load config with filters, for general module
:return:
"""
out = filtertests(backend=None,
location=__TC_GENERAL_COMPONENT,
linters=False,
tests=[],
tags=[],
relevancy="")
print_debug(out)
assert len(out) == 6
def test_filter_mtf_nothing():
"""
use config what loads no tests
:return:
"""
out = filtertests(backend="mtf",
location=__TC_MTF_COMPOMENT,
linters=False,
tests=[],
tags=[],
relevancy="")
print_debug(out)
assert len(out) == 0
def test_filter_mtf_justlintes():
"""
test load just linter with simple config
:return:
"""
out = filtertests(backend="mtf",
location=__TC_MTF_COMPOMENT,
linters=True,
tests=[],
tags=[],
relevancy="")
print_debug(out)
assert len(out) > 20
def test_general_config():
"""
test loading general config and check number of tests, linters disabled
:return:
"""
out = (filtertests(backend=None,
location=__TC_GENERAL_CONF,
linters=False,
tests=[],
tags=[],
relevancy=""))
print_debug(out)
assert len(out) == 2
def main(_):
"""
Start either train or eval. Note hardcoded parts of path for training and eval data
"""
hps = LM.get_default_hparams().parse(FLAGS.hpconfig)
hps._set("num_gpus", FLAGS.num_gpus)
print('*****HYPER PARAMETERS*****')
print(hps)
print('**************************')
print_debug('our training DataSetDir=%s , LogDir=%s' %
(FLAGS.datadir, FLAGS.logdir))
#vocab = Vocabulary.from_file(os.path.join(FLAGS.datadir, "1b_word_vocab.txt"))
vocab = Vocabulary.from_file(os.path.join(FLAGS.datadir, "vocabulary.txt"))
FLAGS.mode = "train"
for i in range(10):
print("Iteration ", i, " phase: ", FLAGS.mode)
if FLAGS.mode == "train":
#hps.batch_size = 256
# dataset = Dataset(vocab, os.path.join(FLAGS.datadir,
# "training-monolingual.tokenized.shuffled/*"))
dataset = Dataset(vocab,
os.path.join(FLAGS.datadir, "ptb.train.txt"))
trainlogdir = (
FLAGS.logdir + str("/") + "train"
) #(FLAGS.logdir+str("\\")+"train")#os.path.join(FLAGS.logdir, "train")
print_debug('train log dir=%s' % (trainlogdir))
run_train(dataset, hps, trainlogdir, ps_device="/gpu:0")
print_debug('Finished run_train !!!!!!!!!!!')
elif FLAGS.mode.startswith("eval"):
print_debug('eval mode')
# if FLAGS.mode.startswith("eval_train"):
# data_dir = os.path.join(FLAGS.datadir, "training-monolingual.tokenized.shuffled/*")
# elif FLAGS.mode.startswith("eval_full"):
# data_dir = os.path.join(FLAGS.datadir, "heldout-monolingual.tokenized.shuffled/*")
# else:
# data_dir = os.path.join(FLAGS.datadir, "heldout-monolingual.tokenized.shuffled/news.en.heldout-00000-of-00050")
dataset = Dataset(vocab,
os.path.join(FLAGS.datadir, "ptb.test.txt"),
deterministic=True)
run_eval(dataset, hps, FLAGS.logdir, FLAGS.mode, FLAGS.eval_steps)
print_debug('Finished run_eval !!!!!!!!!!!')
if FLAGS.mode == "train":
FLAGS.mode = "eval_full"
else:
FLAGS.mode = "train"
def print_if_newest_version(self, project, version, debug):
"""
Prints the project and it's version if it is newer than the
one in cache.
"""
try:
version_cache = self.cache_dict[project]
common.print_debug(debug, u'\t\tIn cache: {}'.format(version_cache))
if version != version_cache:
common.print_project_version(project, version)
except KeyError:
common.print_project_version(project, version)
def test_mtf_metadata_linters_and_tests_noconfig():
"""
Test linter only for MTF loader, using no config
:return:
"""
mt = MetadataLoaderMTF(location=__TC_MTF_COMPOMENT, linters=True)
# print yaml.dump(mt.get_metadata())
# print mt.backend_passtrought_args()
# print mt.apply_filters()
case_justlinters_nofilter = mt.apply_filters()
print_debug(case_justlinters_nofilter)
mt._import_tests("*.py")
case_lintersanstests_nofilter = mt.apply_filters()
print_debug(case_lintersanstests_nofilter)
mt.add_filter(tags=["add"])
case_lintersanstests_filter1 = mt.apply_filters()
print_debug(case_lintersanstests_filter1)
mt.add_filter(tags=["-add"])
case_lintersanstests_filter2 = mt.apply_filters()
print_debug(case_lintersanstests_filter2)
assert len(case_justlinters_nofilter) > 20
assert len(case_lintersanstests_nofilter) > len(case_justlinters_nofilter)
assert len(case_lintersanstests_filter1) < len(case_justlinters_nofilter)
assert len(case_lintersanstests_filter1) < len(
case_lintersanstests_nofilter)
assert len(case_lintersanstests_filter1) > len(
case_lintersanstests_filter2)
assert len(case_lintersanstests_filter2) < len(case_justlinters_nofilter)
def update_cache_dict(self, project, version, debug):
"""
Updates cache dictionary if needed. We always keep the latest version.
"""
try:
version_cache = self.cache_dict[project]
common.print_debug(debug, u'\t\tUpdating cache with in cache: {} / new ? version {}'.format(version_cache, version))
if version != version_cache:
self.cache_dict[project] = version
except KeyError:
self.cache_dict[project] = version
def tmpfunc():
a = process.run(
"cd %s; koji download-build %s -a %s -a noarch" %
(dirname, pkgbouid, ARCH),
shell=True,
verbose=is_debug(),
ignore_status=True)
if a.exit_status == 1:
if "packages available for" in a.stdout.strip():
print_debug(
'UNABLE TO DOWNLOAD package (intended for other architectures, GOOD):',
a.command)
else:
raise mtfexceptions.KojiExc(
'UNABLE TO DOWNLOAD package (KOJI issue, BAD):',
a.command)
def test_loader():
"""
Test general backend loader for complex case
:return:
"""
downloaded_test = "downloaded_test.py"
if os.path.exists(downloaded_test):
os.remove(downloaded_test)
mt = MetadataLoader(location=__TC_GENERAL_COMPONENT)
print_debug(yaml.dump(mt.get_metadata()))
print_debug(mt.get_backends())
assert 'sanity/generaltest.py' in [x[SOURCE] for x in mt.backend_tests()]
assert os.path.exists(downloaded_test)
os.remove(downloaded_test)
git_dir_name = "downloaded_git"
assert os.path.exists(git_dir_name)
shutil.rmtree(git_dir_name)
def test_mtf_config():
"""
test real life example and check if proper tests were filtered based on config file
:return:
"""
out = filtertests(backend="mtf",
location=__TC_MTF_CONF,
linters=False,
tests=[],
tags=[],
relevancy="")
tests = [x[SOURCE] for x in out]
print_debug(tests)
assert len(tests) == 6
assert "Rem" not in " ".join(tests)
assert "Add" in " ".join(tests)
assert "DockerFileLint" in " ".join(tests)
def check_versions(versions_conf, list_site_list):
"""
Checks version by checking each project's feed.
"""
for site_name in list_site_list:
common.print_debug(versions_conf.options.debug,
u'Checking {} updates'.format(site_name))
(project_list, project_url, cache_filename,
project_entry) = versions_conf.get_infos_for_site(site_name)
regex = versions_conf.extract_regex_from_site(site_name)
multiproject = versions_conf.extract_multiproject_from_site(site_name)
feed_filename = u'{}.feed'.format(site_name)
check_versions_for_list_sites(project_list, project_url,
cache_filename, feed_filename,
versions_conf.local_dir,
versions_conf.options.debug, regex,
multiproject)
def print_cache_or_check_versions(versions_conf):
"""
Decide to pretty print projects and their associated version that
are already in the cache or to check versions of that projects upon
selections made at the command line
"""
common.print_debug(versions_conf.options.debug,
u'Loading yaml config file')
versions_conf.load_yaml_from_config_file(versions_conf.config_filename)
if versions_conf.options.list_cache is True:
# Pretty prints all caches.
cache_list = versions_conf.make_site_cache_list_name()
caches.print_versions_from_cache(versions_conf.local_dir, cache_list)
else:
# Checks version from online feeds
check_versions(versions_conf)
def __init__(self, dockerfile=None):
if dockerfile is None:
dir_name = os.getcwd()
else:
dir_name = os.path.dirname(dockerfile)
help_md_file = os.path.join(dir_name, HELP_MD)
common.print_debug("help.md path is %s." % help_md_file)
if os.path.exists(help_md_file):
with open(help_md_file, 'r') as f:
lines = f.readlines()
# Count with all lines which begins with #
self.help_md = [x.strip() for x in lines if x.startswith('#')]
# Count with all lines which begins with %
self.help_md.extend(
[x.strip() for x in lines if x.startswith('%')])
else:
common.print_debug("help.md should exists in the %s directory." %
dir_name)
self.help_md = None
def download_tagged(self, dirname):
"""
Downloads packages to directory, based on koji tags
It downloads just ARCH and noarch packages
:param dirname: string
:return: None
"""
print_info("DOWNLOADING ALL packages for %s_%s_%s" %
(self.name, self.stream, self.version))
for foo in process.run("koji list-tagged --quiet %s" %
self.get_pdc_info()["koji_tag"],
verbose=is_debug()).stdout.split("\n"):
pkgbouid = foo.strip().split(" ")[0]
if len(pkgbouid) > 4:
print_debug("DOWNLOADING: %s" % foo)
@Retry(
attempts=DEFAULTRETRYCOUNT * 10,
timeout=DEFAULTRETRYTIMEOUT * 60,
delay=DEFAULTRETRYTIMEOUT,
error=mtfexceptions.KojiExc(
"RETRY: Unbale to fetch package from koji after %d attempts"
% (DEFAULTRETRYCOUNT * 10)))
def tmpfunc():
a = process.run(
"cd %s; koji download-build %s -a %s -a noarch" %
(dirname, pkgbouid, ARCH),
shell=True,
verbose=is_debug(),
ignore_status=True)
if a.exit_status == 1:
if "packages available for" in a.stdout.strip():
print_debug(
'UNABLE TO DOWNLOAD package (intended for other architectures, GOOD):',
a.command)
else:
raise mtfexceptions.KojiExc(
'UNABLE TO DOWNLOAD package (KOJI issue, BAD):',
a.command)
tmpfunc()
print_info("DOWNLOADING finished")
def check_versions_for_list_sites(feed_project_list, url, cache_filename, feed_filename, local_dir, debug, regex, multiproject):
"""
Checks projects of 'list' type sites such as freshcode's web site's RSS
"""
freshcode_cache = caches.FileCache(local_dir, cache_filename)
feed_info = caches.FeedCache(local_dir, feed_filename)
feed_info.read_cache_feed()
feed = common.get_feed_entries_from_url(url)
if feed is not None:
common.print_debug(debug, u'\tFound {} entries'.format(len(feed.entries)))
feed_list = common.make_list_of_newer_feeds(feed, feed_info, debug)
common.print_debug(debug, u'\tFound {} new entries (relative to {})'.format(len(feed_list), feed_info.date_minutes))
check_and_update_feed(feed_list, feed_project_list, freshcode_cache, debug, regex, multiproject)
# Updating feed_info with the latest parsed feed entry date
feed_info.update_cache_feed(feed.entries[0].published_parsed)
feed_info.write_cache_feed()
def get_repo(self):
odcs = ODCS(ODCS_URL,
auth_mech=AuthMech.OpenIDC,
openidc_token=self.auth_token)
print_debug(
"ODCS Starting module composing: %s" % odcs, "%s compose for: %s" %
(self.compose_type, self.get_module_identifier()))
compose_builder = odcs.new_compose(self.get_module_identifier(),
self.compose_type)
timeout_time = 600
print_debug("ODCS Module compose started, timeout set to %ss" %
timeout_time)
compose_state = odcs.wait_for_compose(compose_builder["id"],
timeout=timeout_time)
if compose_state["state_name"] == "done":
compose = "{compose}/{arch}/os".format(
compose=compose_state["result_repo"], arch=ARCH)
print_info("ODCS Compose done, URL with repo file", compose)
return compose
else:
raise mtfexceptions.PDCExc(
"ODCS: Failed to generate compose for module: %s" %
self.get_module_identifier())
def get_latest_release_by_title(project, debug, feed_url, local_dir, feed_filename, site_entry):
"""
Gets the latest release or the releases between the last checked time of
a program on a site of type 'byproject'.
project must be a string that represents the project (user/repository in
github for instance).
Returns a tuple which contains the name of the project, a list of versions
and a boolean that indicates if we checked by last checked time (True) or
by release (False).
"""
feed_list = []
(valued, name, regex, entry) = get_values_from_project(project)
last_checked = is_one_entry_field_value_egal_to_last_check(site_entry, entry)
filename = format_project_feed_filename(feed_filename, name)
url = feed_url.format(name)
feed = common.get_feed_entries_from_url(url)
if feed is not None and len(feed.entries) > 0:
feed_list = get_releases_filtering_feed(debug, local_dir, filename, feed, last_checked)
if valued and regex != '':
# Here we match the whole list against the regex and replace the
# title's entry of the result of that match upon success.
for feed_entry in feed_list:
res = re.match(regex, feed_entry.title)
# Here we should make a new list with the matched entries and leave the other ones
if res:
feed_entry.title = res.group(1)
common.print_debug(debug, u'\tname: {}\n\tversion: {}\n\tregex: {} : {}'.format(name, feed_entry.title, regex, res))
common.print_debug(debug, u'\tProject {}: {}'.format(name, feed.entries[0].title))
return (name, feed_list, last_checked)
def run_eval(dataset, hps, logdir, mode, num_eval_steps):
print_debug('run_eval logdir=%s ' % (logdir))
with tf.variable_scope("model"):
hps.num_sampled = 0 # Always using full softmax at evaluation.
hps.keep_prob = 1.0
#model = LM(hps, "eval", "/cpu:0")
model = LM(hps, "eval", "/gpu:0")
if hps.average_params:
print("Averaging parameters for evaluation.")
saver = tf.train.Saver(model.avg_dict)
else:
saver = tf.train.Saver()
# Use only 4 threads for the evaluation.
#config = tf.ConfigProto(allow_soft_placement=True,
# intra_op_parallelism_threads=20,
# inter_op_parallelism_threads=1)
config = tf.ConfigProto(allow_soft_placement=True)
sess = tf.Session(config=config)
sw = tf.summary.FileWriter(logdir + "/" + mode, sess.graph)
print_debug('run_eval tf.summary.FileWriter=%s ' % (logdir + "/" + mode))
ckpt_loader = CheckpointLoader(saver, model.global_step, logdir + "/train")
print_debug('run_eval ckpt_loader=%s ' % (ckpt_loader.logdir))
with sess.as_default():
print_debug('run_eval sess.as_default iteration')
while ckpt_loader.load_checkpoint():
print_debug('eval load_checkpoint chunk Loader done!')
global_step = ckpt_loader.last_global_step
if mode == "eval_full":
data_iterator = dataset.iterate_forever(
hps.batch_size * hps.num_gpus, hps.num_steps)
else:
data_iterator = dataset.iterate_once(
hps.batch_size * hps.num_gpus, hps.num_steps)
print_debug('eval run local variables initalizer')
#tf.initialize_local_variables().run()
tf.local_variables_initializer().run()
loss_nom = 0.0
loss_den = 0.0
print_debug('eval run for loop of enumerated data iterator mode=' +
mode + ' eval_steps=' + str(num_eval_steps))
#for i, (x, y, w) in enumerate(data_iterator):
for i, (x, y) in enumerate(data_iterator):
if i >= num_eval_steps and mode != "eval_full":
break
#loss = sess.run(model.loss, {model.x: x, model.y: y, model.w: w})
loss = sess.run(model.loss, {model.x: x, model.y: y})
loss_nom += loss
loss_den += 1 # ???
#loss_den += w.mean()
loss = loss_nom / loss_den
#sys.stdout.write("%d: %.3f (%.3f) ... " % (i, loss, np.exp(loss)))
#sys.stdout.flush()
#sys.stdout.write("\n")
log_perplexity = loss_nom / loss_den
print("Results at %d: log_perplexity = %.3f perplexity = %.3f" %
(global_step, log_perplexity, np.exp(log_perplexity)))
summary = tf.Summary()
summary.value.add(tag='eval/log_perplexity',
simple_value=log_perplexity)
summary.value.add(tag='eval/perplexity',
simple_value=np.exp(log_perplexity))
sw.add_summary(summary, global_step)
sw.flush()
#if mode == "eval_full":
# break #we don't need to wait for other checkpoints in this mode
break #we always break
print_debug('run_eval END OF WHILE loader loop')
print_debug('run_eval END OF WHILE session loop')
sess.close()
tf.reset_default_graph()
def run_statistic(dataset, hps, logdir, ps_device, task=0, master=""):
with tf.variable_scope("model"):
print_debug('loading LM model')
model = LM(hps, "train", ps_device)
stime = time.time()
print("Current time: %s" % stime)
print("ALL VARIABLES")
for v in tf.all_variables():
print("%s %s %s %s" % (v.name, v.get_shape(), v.dtype, v.device))
print("TRAINABLE VARIABLES")
for v in tf.trainable_variables():
print("%s %s %s %s" % (v.name, v.get_shape(), v.dtype, v.device))
print("LOCAL VARIABLES")
for v in tf.local_variables():
print("%s %s %s %s" % (v.name, v.get_shape(), v.dtype, v.device))
sv = tf.train.Supervisor(
is_chief=(task == 0),
logdir=logdir, # logdir=None, # logdir=logdir,
summary_op=None, # Automatic summaries don't work with placeholders.
#global_step=model.global_step,
save_summaries_secs=60 * hps.save_summary_every_min,
save_model_secs=60 * hps.save_model_every_min)
# save_summaries_secs=30,
# save_model_secs=120 * 5)
# config = tf.ConfigProto(allow_soft_placement=True,
# intra_op_parallelism_threads=2,
# inter_op_parallelism_threads=20)
config = tf.ConfigProto(allow_soft_placement=True)
close_summary_writer = False
with sv.managed_session(master,
config=config,
start_standard_services=True,
close_summary_writer=False) as sess:
# Slowly increase the number of workers during beginning of the training.
# while not sv.should_stop() and (time.time() - stime) < hps.max_time:
# step = int(sess.run(model.global_step))
# waiting_until_step = task * hps.num_delayed_steps
# if step >= waiting_until_step:
# break
# else:
# print("Current step is %d. Waiting until: %d" % (step, waiting_until_step))
# time.sleep(20.0)
local_step = 0
prev_global_step = sess.run(model.global_step)
cur_global_step = 0
prev_time = time.time()
data_iterator = dataset.iterate_forever(hps.batch_size * hps.num_gpus,
hps.num_steps)
print_debug(
'before looping model, sv.save_path=%s , sv.should_stop()=%d, (time.time() - stime)=%.2fs, hps.max_time=%.2fs '
% (sv.save_path, sv.should_stop(),
(time.time() - stime), hps.max_time))
while not sv.should_stop() and (time.time() - stime) < hps.max_time:
if (int(time.time()) - int(stime)) % 10 == 0:
print_debug(
'While In looping model, sv.should_stop()=%d, (time.time() - stime)=%.2fs, hps.max_time=%.2fs '
% (sv.should_stop(), (time.time() - stime), hps.max_time))
fetches = [model.global_step, model.loss, model.train_op]
# Chief worker computes summaries every 100 steps.
should_compute_summary = (task == 0 and local_step % 100 == 0)
if should_compute_summary:
fetches += [model.summary_op]
# x, y, w = next(data_iterator)
x, y = next(data_iterator)
should_run_profiler = (hps.run_profiler and task == 0
and local_step % 1000 == 13)
if should_run_profiler:
run_options = tf.RunOptions(
trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
# fetched = sess.run(fetches, {model.x: x, model.y: y, model.w: w},
fetched = sess.run(fetches, {
model.x: x,
model.y: y
},
options=run_options,
run_metadata=run_metadata)
# Create the Timeline object, and write it to a json
tl = timeline.Timeline(run_metadata.step_stats)
ctf = tl.generate_chrome_trace_format()
print("Running profiler")
with open(logdir + "/timeline.json", 'w') as f:
f.write(ctf)
print("Finished profiling!")
else:
# fetched = sess.run(fetches, {model.x: x, model.y: y, model.w: w})
fetched = sess.run(fetches, {model.x: x, model.y: y})
cur_global_step = fetched[0]
local_step += 1
if should_compute_summary:
# print_debug('should_compute_summary!!! BUT WE DROPED THIS MODE TO SAVE MEMORY SPACE sv.should_stop()=%d, (time.time() - stime)=%.2fs, hps.max_time=%.2fs ' %(sv.should_stop(), (time.time() - stime), hps.max_time))
sv.summary_computed(sess, fetched[-1])
if local_step < 10 or local_step % 20 == 0:
cur_time = time.time()
num_words = hps.batch_size * hps.num_gpus * hps.num_steps
wps = (cur_global_step -
prev_global_step) * num_words / (cur_time - prev_time)
prev_global_step = cur_global_step
print(
"Iteration %d, time = %.2fs, wps = %.0f, train loss = %.4f"
% (cur_global_step, cur_time - prev_time, wps, fetched[1]))
prev_time = cur_time
# save last model
print_debug('Supervisor Begin Save after training period')
sv._saver.save(sess, sv.save_path, cur_global_step)
print_debug('Supervisor DONE Save after training period')
# close sv with close summery flag
sv.stop(None, close_summary_writer)
def __init__(self, hps, mode="train", ps_device="/gpu:0"):
self.hps = hps
data_size = hps.batch_size * hps.num_gpus
self.x = tf.placeholder(tf.int32, [data_size, hps.num_steps])
self.y = tf.placeholder(tf.int32, [data_size, hps.num_steps])
#self.w = tf.placeholder(tf.int32, [data_size, hps.num_steps])
losses = []
tower_grads = []
#xs = tf.split(0, hps.num_gpus, self.x)
print_debug("creating LM model with %d GPUs" % (hps.num_gpus))
xs = tf.split(self.x, hps.num_gpus, 0)
#ys = tf.split(0, hps.num_gpus, self.y)
ys = tf.split(self.y, hps.num_gpus, 0)
#ws = tf.split(0, hps.num_gpus, self.w)
for i in range(hps.num_gpus):
with tf.device(assign_to_gpu(i, ps_device)), tf.variable_scope(
tf.get_variable_scope(), reuse=True if i > 0 else None):
#loss = self._forward(i, xs[i], ys[i], ws[i])
loss = self._forward(i, xs[i], ys[i])
losses += [loss]
if mode == "train":
cur_grads = self._backward(
loss,
summaries=((i == hps.num_gpus - 1)
and hps.do_summaries))
tower_grads += [cur_grads]
self.loss = tf.add_n(losses) / len(losses)
tf.summary.scalar("model/loss", self.loss)
self.global_step = tf.get_variable("global_step", [],
tf.int32,
trainable=False)
if mode == "train":
grads = average_grads(tower_grads)
if hps.optimizer == 1:
optimizer = tf.train.MomentumOptimizer(hps.learning_rate, 0.9)
elif hps.optimizer == 2:
optimizer = tf.train.AdamOptimizer(hps.learning_rate)
elif hps.optimizer == 3:
optimizer = tf.train.RMSPropOptimizer(
learning_rate=hps.learning_rate)
elif hps.optimizer == 4:
optimizer = tf.train.GradientDescentOptimizer(
hps.learning_rate)
else:
optimizer = tf.train.AdagradOptimizer(
hps.learning_rate,
initial_accumulator_value=1.0 * float(hps.loss_scale) *
float(hps.loss_scale))
self.train_op = optimizer.apply_gradients(
grads, global_step=self.global_step)
self.summary_op = tf.summary.merge_all()
else:
self.train_op = tf.no_op()
if mode in ["train", "eval"] and hps.average_params:
with tf.name_scope(
None
): # This is needed due to EMA implementation silliness.
# Keep track of moving average of LSTM variables.
ema = tf.train.ExponentialMovingAverage(decay=0.999)
variables_to_average = find_trainable_variables("lstm")
self.train_op = tf.group(
*[self.train_op,
ema.apply(variables_to_average)])
self.avg_dict = ema.variables_to_restore(variables_to_average)
"""
import yaml
import os
import sys
from avocado.utils import process
from common import print_info, DEFAULTRETRYCOUNT, DEFAULTRETRYTIMEOUT, \
get_if_remoterepos, BASEPATHDIR, MODULEFILE, print_debug,\
is_debug, ARCH, is_recursive_download, trans_dict, get_odcs_auth
from moduleframework import mtfexceptions
from pdc_client import PDCClient
from timeoutlib import Retry
try:
from odcs.client.odcs import ODCS, AuthMech
except:
print_debug("ODCS library cannot be imported. ODCS is not supported")
PDC_SERVER = "https://pdc.fedoraproject.org/rest_api/v1/modules"
ODCS_URL = "https://odcs.fedoraproject.org"
DEFAULT_MODULE_STREAM = "master"
BASE_REPO_URL = "https://download.fedoraproject.org/pub/fedora/linux/releases/{}/Workstation/{}/os"
def get_module_nsv(name=None, stream=None, version=None):
name = name or os.environ.get('MODULE_NAME')
stream = stream or os.environ.get('MODULE_STREAM') or DEFAULT_MODULE_STREAM
version = version or os.environ.get('MODULE_VERSION')
return {'name': name, 'stream': stream, 'version': version}
def get_base_compose():
