def _make_plugin_section(self, plugin_cls, base_name=None):
section_name = plugin_cls.get_name()
if base_name:
section_name += " [%s]" % base_name
section_obj = section(section_name)
info = plugin_cls.get_info()
if info["title"]:
section_obj.append(paragraph(info["title"]))
if info["description"]:
section_obj.extend(parse_text(info["description"]))
if info["namespace"]:
section_obj.append(
paragraph("**Namespace**: %s" % info["namespace"]))
if info["parameters"]:
if base_name:
ref_prefix = "%s-%s-" % (base_name, plugin_cls.get_name())
else:
ref_prefix = "%s-" % plugin_cls.get_name()
section_obj.extend(
self._make_pretty_parameters(info["parameters"], ref_prefix))
if info["returns"]:
section_obj.extend(
parse_text("**Returns**:\n%s" % info["returns"]))
filename = info["module"].replace(".", "/")
ref = "https://github.com/openstack/rally/blob/master/%s.py" % filename
section_obj.extend(
parse_text("**Module**:\n`%s`__\n\n__ %s" % (info["module"], ref)))
return section_obj
def _make_plugin_section(self, plugin_cls, base_name=None):
section_name = plugin_cls.get_name()
if base_name:
section_name += " [%s]" % base_name
section_obj = utils.section(section_name)
info = plugin_cls.get_info()
if info["title"]:
section_obj.append(utils.paragraph(info["title"]))
if info["description"]:
section_obj.extend(utils.parse_text(info["description"]))
if info["namespace"]:
section_obj.append(
utils.paragraph("**Namespace**: %s" % info["namespace"]))
if base_name:
ref_prefix = "%s-%s-" % (base_name, plugin_cls.get_name())
else:
ref_prefix = "%s-" % plugin_cls.get_name()
if info["parameters"]:
section_obj.extend(
self._make_arg_items(info["parameters"], ref_prefix))
if info["returns"]:
section_obj.extend(
utils.parse_text("**Returns**:\n%s" % info["returns"]))
if info["schema"]:
schema = process_jsonschema(info["schema"])
if "type" in schema:
if "parameters" in schema:
section_obj.extend(
self._make_arg_items(items=schema["parameters"],
ref_prefix=ref_prefix))
elif "patternProperties" in schema:
section_obj.extend(
self._make_arg_items(
items=schema["patternProperties"],
ref_prefix=ref_prefix,
description=[
"*Dictionary is expected. Keys should "
"follow pattern(s) described bellow.*"
]))
elif "oneOf" in schema:
section_obj.append(
utils.note("One of the following groups of "
"parameters should be provided."))
for i, oneOf in enumerate(schema["oneOf"], 1):
description = None
if oneOf.get("doc", None):
description = [oneOf["doc"]]
section_obj.extend(
self._make_arg_items(
items=oneOf["parameters"],
ref_prefix=ref_prefix,
title="Option %s of parameters" % i,
description=description))
else:
section_obj.extend(
self._make_arg_items(items=[schema],
ref_prefix=ref_prefix))
else:
raise Exception("Failed to display provided schema: %s" %
info["schema"])
if issubclass(plugin_cls, validation.ValidatablePluginMixin):
validators = plugin_cls._meta_get("validators", default=[])
platforms = [
kwargs for name, args, kwargs in validators
if name == "required_platform"
]
if platforms:
section_obj.append(
utils.paragraph("**Requires platform(s)**:"))
section = ""
for p in platforms:
section += "* %s" % p["platform"]
admin_msg = "credentials for admin user"
user_msg = ("regular users (temporary users can be created"
" via the 'users' context if admin user is "
"specified for the platform)")
if p.get("admin", False) and p.get("users", False):
section += " with %s and %s." % (admin_msg, user_msg)
elif p.get("admin", False):
section += " with %s." % admin_msg
elif p.get("users", False):
section += " with %s." % user_msg
section += "\n"
section_obj.extend(utils.parse_text(section))
filename = info["module"].replace(".", "/")
ref = "https://github.com/openstack/rally/blob/master/%s.py" % filename
section_obj.extend(
utils.parse_text("**Module**:\n`%s`__\n\n__ %s" %
(info["module"], ref)))
return section_obj
def _make_plugin_section(self, plugin_cls, base_name=None):
section_name = plugin_cls.get_name()
if base_name:
section_name += " [%s]" % base_name
section_obj = section(section_name)
info = plugin_cls.get_info()
if info["title"]:
section_obj.append(paragraph(info["title"]))
if info["description"]:
section_obj.extend(parse_text(info["description"]))
if info["namespace"]:
section_obj.append(
paragraph("**Namespace**: %s" % info["namespace"]))
if base_name:
ref_prefix = "%s-%s-" % (base_name, plugin_cls.get_name())
else:
ref_prefix = "%s-" % plugin_cls.get_name()
if info["parameters"]:
section_obj.extend(
self._make_arg_items(info["parameters"], ref_prefix))
if info["returns"]:
section_obj.extend(parse_text("**Returns**:\n%s" %
info["returns"]))
if info["schema"]:
schema = process_jsonschema(info["schema"])
if "type" in schema:
if "parameters" in schema:
section_obj.extend(
self._make_arg_items(items=schema["parameters"],
ref_prefix=ref_prefix))
elif "patternProperties" in schema:
section_obj.extend(
self._make_arg_items(
items=schema["patternProperties"],
ref_prefix=ref_prefix,
description=[
"*Dictionary is expected. Keys should "
"follow pattern(s) described bellow.*"
]))
elif "oneOf" in schema:
section_obj.append(
note("One of the following groups of "
"parameters should be provided."))
for i, oneOf in enumerate(schema["oneOf"], 1):
description = None
if oneOf.get("doc", None):
description = [oneOf["doc"]]
section_obj.extend(
self._make_arg_items(
items=oneOf["parameters"],
ref_prefix=ref_prefix,
title="Option %s of parameters" % i,
description=description))
else:
section_obj.extend(
self._make_arg_items(items=[schema],
ref_prefix=ref_prefix))
else:
raise Exception("Failed to display provided schema: %s" %
info["schema"])
filename = info["module"].replace(".", "/")
ref = "https://github.com/openstack/rally/blob/master/%s.py" % filename
section_obj.extend(
parse_text("**Module**:\n`%s`__\n\n__ %s" % (info["module"], ref)))
return section_obj
def _make_plugin_section(self, plugin_cls, base_name=None):
section_name = plugin_cls.get_name()
if base_name:
section_name += " [%s]" % base_name
section_obj = utils.section(section_name)
info = plugin_cls.get_info()
if info["title"]:
section_obj.append(utils.paragraph(info["title"]))
if info["description"]:
section_obj.extend(utils.parse_text(info["description"]))
if info["platform"]:
section_obj.append(utils.paragraph(
"**Platform**: %s" % info["platform"]))
if base_name:
ref_prefix = "%s-%s-" % (base_name, plugin_cls.get_name())
else:
ref_prefix = "%s-" % plugin_cls.get_name()
if info["parameters"]:
section_obj.extend(self._make_arg_items(info["parameters"],
ref_prefix))
if info["returns"]:
section_obj.extend(utils.parse_text(
"**Returns**:\n%s" % info["returns"]))
if info["schema"]:
schema = process_jsonschema(info["schema"])
if "type" in schema:
if "parameters" in schema:
section_obj.extend(self._make_arg_items(
items=schema["parameters"],
ref_prefix=ref_prefix))
elif "patternProperties" in schema:
section_obj.extend(self._make_arg_items(
items=schema["patternProperties"],
ref_prefix=ref_prefix,
description=["*Dictionary is expected. Keys should "
"follow pattern(s) described bellow.*"]))
elif "oneOf" in schema:
section_obj.append(utils.note(
"One of the following groups of "
"parameters should be provided."))
for i, oneOf in enumerate(schema["oneOf"], 1):
description = None
if oneOf.get("doc", None):
description = [oneOf["doc"]]
section_obj.extend(self._make_arg_items(
items=oneOf["parameters"],
ref_prefix=ref_prefix,
title="Option %s of parameters" % i,
description=description))
else:
section_obj.extend(self._make_arg_items(
items=[schema], ref_prefix=ref_prefix))
else:
raise Exception("Failed to display provided schema: %s" %
info["schema"])
if issubclass(plugin_cls, validation.ValidatablePluginMixin):
validators = plugin_cls._meta_get("validators", default=[])
platforms = [kwargs for name, args, kwargs in validators
if name == "required_platform"]
if platforms:
section_obj.append(
utils.paragraph("**Requires platform(s)**:"))
section = ""
for p in platforms:
section += "* %s" % p["platform"]
admin_msg = "credentials for admin user"
user_msg = ("regular users (temporary users can be created"
" via the 'users' context if admin user is "
"specified for the platform)")
if p.get("admin", False) and p.get("users", False):
section += " with %s and %s." % (admin_msg, user_msg)
elif p.get("admin", False):
section += " with %s." % admin_msg
elif p.get("users", False):
section += " with %s." % user_msg
section += "\n"
section_obj.extend(utils.parse_text(section))
filename = info["module"].replace(".", "/")
if filename.startswith("rally/"):
project = "rally"
elif filename.startswith("rally_openstack/"):
project = "rally-openstack"
else:
# WTF is it?!
return None
ref = ("https://github.com/openstack/%s/blob/master/%s.py"
% (project, filename))
section_obj.extend(utils.parse_text("**Module**:\n`%s`__\n\n__ %s"
% (info["module"], ref)))
return section_obj
def main():
alphabet_txt = os.path.join(LANG.model_dir, 'alphabet.txt')
raw_txt_gz = os.path.join(LANG.model_dir, 'raw.txt.gz')
unprepared_txt = os.path.join(LANG.model_dir, 'unprepared.txt')
prepared_txt = os.path.join(LANG.model_dir, 'prepared.txt')
vocabulary_txt = os.path.join(LANG.model_dir, 'vocabulary.txt')
unfiltered_arpa = os.path.join(LANG.model_dir, 'unfiltered.arpa')
filtered_arpa = os.path.join(LANG.model_dir, 'filtered.arpa')
lm_binary = os.path.join(LANG.model_dir, 'lm.binary')
kenlm_scorer = os.path.join(LANG.model_dir, 'kenlm.scorer')
temp_prefix = os.path.join(LANG.model_dir, 'tmp')
section('Writing alphabet file', empty_lines_before=1)
with open(alphabet_txt, 'w', encoding='utf-8') as alphabet_file:
alphabet_file.write('\n'.join(LANG.alphabet) + '\n')
redo = ARGS.force_download
section('Downloading text data')
redo = maybe_download(LANG.text_url, raw_txt_gz, force=redo)
section('Unzipping text data')
redo = maybe_ungzip(raw_txt_gz, unprepared_txt, force=redo)
redo = redo or ARGS.force_prepare
section('Preparing text and building vocabulary')
if redo or not os.path.isfile(prepared_txt) or not os.path.isfile(vocabulary_txt):
redo = True
announce('Preparing {} shards of "{}"...'.format(ARGS.workers, unprepared_txt))
counters = Queue(ARGS.workers)
source_bytes = os.path.getsize(unprepared_txt)
aggregator_process = Process(target=aggregate_counters, args=(vocabulary_txt, source_bytes, counters))
aggregator_process.start()
counter_processes = list(map(lambda index: Process(target=count_words, args=(index, counters)),
range(ARGS.workers)))
try:
for p in counter_processes:
p.start()
for p in counter_processes:
p.join()
counters.put(STOP_TOKEN)
aggregator_process.join()
print('')
partials = list(map(lambda i: get_partial_path(i), range(ARGS.workers)))
join_files(partials, prepared_txt)
for partial in partials:
os.unlink(partial)
except KeyboardInterrupt:
aggregator_process.terminate()
for p in counter_processes:
p.terminate()
raise
else:
announce('Files "{}" and \n\t"{}" existing - not preparing'.format(prepared_txt, vocabulary_txt))
redo = redo or ARGS.force_generate
section('Building unfiltered language model')
if redo or not os.path.isfile(unfiltered_arpa):
redo = True
lmplz_args = [
KENLM_BIN + '/lmplz',
'--temp_prefix', temp_prefix,
'--memory', '80%',
'--discount_fallback',
'--limit_vocab_file', vocabulary_txt,
'--text', prepared_txt,
'--arpa', unfiltered_arpa,
'--skip', 'symbols',
'--order', str(LANG.order)
]
if len(LANG.prune) > 0:
lmplz_args.append('--prune')
lmplz_args.extend(list(map(str, LANG.prune)))
subprocess.check_call(lmplz_args)
else:
announce('File "{}" existing - not generating'.format(unfiltered_arpa))
section('Filtering language model')
if redo or not os.path.isfile(filtered_arpa):
redo = True
with open(vocabulary_txt, 'rb') as vocabulary_file:
vocabulary_content = vocabulary_file.read()
subprocess.run([
KENLM_BIN + '/filter',
'single',
'model:' + unfiltered_arpa,
filtered_arpa
], input=vocabulary_content, check=True)
else:
announce('File "{}" existing - not filtering'.format(filtered_arpa))
section('Generating binary representation')
if redo or not os.path.isfile(lm_binary):
redo = True
subprocess.check_call([
KENLM_BIN + '/build_binary',
'-a', '255',
'-q', '8',
'-v',
'trie',
filtered_arpa,
lm_binary
])
else:
announce('File "{}" existing - not generating'.format(lm_binary))
section('Building scorer')
if redo or not os.path.isfile(kenlm_scorer):
redo = True
words = set()
vocab_looks_char_based = True
with open(vocabulary_txt) as vocabulary_file:
for line in vocabulary_file:
for word in line.split():
words.add(word.encode())
if len(word) > 1:
vocab_looks_char_based = False
announce("{} unique words read from vocabulary file.".format(len(words)))
announce(
"{} like a character based model.".format(
"Looks" if vocab_looks_char_based else "Doesn't look"
)
)
if ARGS.alphabet_mode == 'auto':
use_utf8 = vocab_looks_char_based
elif ARGS.alphabet_mode == 'utf8':
use_utf8 = True
else:
use_utf8 = False
serialized_alphabet = get_serialized_utf8_alphabet() if use_utf8 else LANG.get_serialized_alphabet()
from ds_ctcdecoder import Scorer, Alphabet
alphabet = Alphabet()
err = alphabet.deserialize(serialized_alphabet, len(serialized_alphabet))
if err != 0:
announce('Error loading alphabet: {}'.format(err))
sys.exit(1)
scorer = Scorer()
scorer.set_alphabet(alphabet)
scorer.set_utf8_mode(use_utf8)
scorer.reset_params(LANG.alpha, LANG.beta)
scorer.load_lm(lm_binary)
scorer.fill_dictionary(list(words))
shutil.copy(lm_binary, kenlm_scorer)
scorer.save_dictionary(kenlm_scorer, True) # append, not overwrite
announce('Package created in {}'.format(kenlm_scorer))
announce('Testing package...')
scorer = Scorer()
scorer.load_lm(kenlm_scorer)
else:
announce('File "{}" existing - not building'.format(kenlm_scorer))
for view_name, use in views_dict.items():
if use:
views[view_name].use = True
cam = view2cam[view_name]
print(f"Camera '{utils.yellow(cam.name)}' will be used for rendering")
cam.data.type = "PERSP"
cam.data.lens_unit = "FOV"
cam.data.angle = 1.0471975803375244 # 60 degrees in radians
cam.data.clip_start = 0.0001
cam.data.clip_end = 1000
print(f"{cam.name}'s attributes are set to hardcoded values")
# Decorate existing function
rm_directory: Callable = utils.section("Clear data")(utils.rm_directory)
def set_attrs_dir(dir_: str) -> None:
cng.GENERATED_DATA_DIR = dir_
utils.section("Clear data")
def handle_clear(clear: bool, clear_exit: bool, directory: str) -> None:
"""
Handles --clear and --clear-exit options
"""
if (clear or clear_exit) == True:
rm_directory(directory, doublecheck=True)
if clear_exit == True:
print("Exiting")