def _selector(list_to_select_from, title):
if len(list_to_select_from) == 0:
return prompter.prompt(title, default="")
enumerated = list(enumerate(sorted(list_to_select_from)))
print(title + "\n")
for entry in enumerated:
print(" {} / {}".format(entry[0], entry[1]))
def get_valid_input():
input = prompter.prompt("enter selection", default="")
if input.isdigit():
input = int(input)
if input < len(enumerated):
return enumerated[input][1]
else:
print("Invalid number")
return None
else:
return input
print("Enter name, number, type in your own, or leave blank: ")
input = None
while not input:
input = get_valid_input()
return input
def ask_question(question, vars_dict):
default = vars_dict[question['var_name']]
prompt_line = question['prompt_line']
if question['type'] == 'free_text':
return prompt(prompt_line, default=default, suffix=':')
elif question['type'] == 'yesno':
return yesno(prompt_line, default='yes' if default else 'no', suffix=':')
def ask_question(question, vars_dict):
default = vars_dict[question['var_name']]
prompt_line = question['prompt_line']
if question['type'] == 'free_text':
return prompt(prompt_line, default=default, suffix=':')
elif question['type'] == 'yesno':
return yesno(prompt_line, default='yes' if default else 'no', suffix=':')
def _create_config_file():
new_config = editor_ignore_comments(PkgResource.read(PkgResource.DEFAULT_CONFIG))
filename = prompter.prompt("Enter path for saving config", default=CONFIG_FILE)
print("Writing config to {}".format(filename))
with open(filename, 'w') as f:
f.write(new_config)
os.chmod(filename, 0o600)
return filename
def get_valid_input():
input = prompter.prompt("enter selection", default="")
if input.isdigit():
input = int(input)
if input < len(enumerated):
return enumerated[input][1]
else:
print("Invalid number")
return None
else:
return input
def detect_device(self):
"""
Detect the device to burn the image to.
"""
# TODO Implement this
# Windows doesn't need a device (but it needs drives...)
# Mac the device can be parsed from diskutil list
# Linux not sure
# If we need a device and don't know, ask the user
if os_is_mac() or os_is_linux() or os_is_pi():
device = prompt("Please enter the device name to install to:")
else:
device = self.boot_drive
return device
def _create_config_file():
path = 'resources/config_default.yml'
with open(pkg_resources.resource_filename(__name__, path)) as f:
default_cfg = yaml.safe_load(f)
default_text = (
'# Default config for the CFME-QE team.\n'
'# username/password is not required unless "basic_auth" is "true"\n\n'
'{}'.format(yaml.dump(default_cfg, default_flow_style=False)))
new_config = editor_ignore_comments(default_text)
filename = prompter.prompt("Enter path for saving config",
default=CONFIG_FILE)
print("Writing config to {}".format(filename))
with open(filename, 'w') as f:
f.write(new_config)
return filename
def get_variables_data(variables_files):
variables_data = load_cfg_file(variables_files[0])
# Merge multiple config files
if len(variables_files) > 1:
for var_file in variables_files[1:]:
merged_file_data = load_cfg_file(var_file)
variables_data = object_merge(variables_data, merged_file_data)
# Check if there's any variables we need to prompt for
for section, data in variables_data.items():
for var_name, var_value in data.items():
if var_value == "{prompt}":
variables_data[section][var_name] = prompter.prompt(
"Enter value for parameter '{}' in section '{}':".format(
var_name, section))
return variables_data
def get_variables_data(variables_files):
variables_data = load_cfg_file(variables_files[0])
# Merge multiple config files
if len(variables_files) > 1:
for var_file in variables_files[1:]:
merged_file_data = load_cfg_file(var_file)
variables_data = object_merge(variables_data, merged_file_data)
# Check if there's any variables we need to prompt for
for section, data in variables_data.items():
for var_name, var_value in data.items():
if var_value == "{prompt}":
variables_data[section][var_name] = prompter.prompt(
"Enter value for parameter '{}' in section '{}':".format(var_name, section)
)
return variables_data
def update():
util.header('Archiving citations')
execute(archive_all_citations)
util.header('Updating requirements file')
local('pip freeze -r ' + config.root_path + '/devel-req.txt > ' +
config.root_path + '/requirements.txt')
util.header('Updating citations md files')
execute(generate_cite_md)
util.header('Adding to git')
local('git add .')
local('git status')
m = prompt("Commit message:", default='Autoupdate')
# if not yesno('Continue?'):
# return
local('git commit -m \'' + m + "\'")
local('git push origin master')
print colored('............................. Done', 'green')
def test_prompt_strips_whitespace(mock_raw_input):
returned_value = prompt('What is your name?')
assert_equals(returned_value, 'Dave')
def test_prompt_returns_default_for_blank_input(mock_raw_input):
returned_value = prompt('What is your name?', default='Dave')
assert_equals(returned_value, 'Dave')
def test_prompt_strips_whitespace(mock_raw_input):
returned_value = prompt('What is your name?')
assert_equals(returned_value, 'Dave')
def restore_backup_fun(backup_path):
cl.echo(pr.prompt("Restoring backup...", 1, pr.Prompts.Info))
execute(
"dconf load / &>> {} < {}/gnome-settings".format(logfile_path, backup_path),
"tar xf {}/brave.tar.gz -C $HOME/.config/ &>> {}".format(backup_path, logfile_path)
)
def make_backup(directory):
cl.echo(pr.prompt("Backing up...", 1, pr.Prompts.Info))
execute(
"dconf dump / > {}/gnome-settings &>> {}".format(directory, logfile_path),
"tar czf {}/brave.tar.gz ~/.config/BraveSoftware &>> {}".format(directory, logfile_path)
)
def auto_install():
"""Configure everything according to passed options"""
global package_manager
""" prompts """
while True:
packages_to_install = list()
# Initial prompts
# Change shell
change_shell = cl.confirm(pr.prompt("Change shell?", 0, pr.Prompts.Question))
# Dotfiles prompts
restore_dotfiles = cl.confirm(pr.prompt("Restore dotfiles?", 0, pr.Prompts.Question))
if restore_dotfiles:
dotfiles_path = cl.prompt(pr.prompt("Dotfiles repo", 1, pr.Prompts.Question), type=str,
default="https://github.com/noahjutz/dotfiles")
# Backup prompts
restore_backup = cl.confirm(pr.prompt("Restore backups?", 0, pr.Prompts.Question))
if restore_backup:
backup_path = cl.prompt(pr.prompt("Backup path", 1, pr.Prompts.Question),
type=cl.Path(dir_okay=True, exists=True, readable=True),
default="{}/backup".format(env_home))
# Package prompts
install_packages = cl.confirm(pr.prompt("Install packages?", 0, pr.Prompts.Question))
if install_packages:
package_manager = \
cl.prompt(pr.prompt("Package manager", 1, pr.Prompts.Question),
type=cl.Choice(choices=packages.keys(), case_sensitive=False))
package_manager = pacman if package_manager == "PACMAN" else deb
# Prompt each package group
groups = {}
for group in package_manager:
if group == "essential":
continue
groups[group] = cl.confirm(pr.prompt("Install {} packages?", 2, pr.Prompts.Question, bold_text=group))
if groups[group]:
for package in package_manager[group]:
if cl.confirm(pr.prompt("Install {}?", 3, pr.Prompts.Question, bold_text=package)):
packages_to_install.append(package)
cl.echo()
# User decides to do nothing
if not restore_dotfiles and not restore_backup and not install_packages and not change_shell:
cl.echo(pr.prompt("Nothing selected. Starting over.", 0, pr.Prompts.Info))
cl.echo()
continue
# Confirmation screen
cl.echo(pr.prompt("Confirm your input:", 0, pr.Prompts.Info))
# Task list
cl.echo(pr.prompt("Task list:", 1, pr.Prompts.Info))
if change_shell:
cl.echo(pr.prompt("Change shell", 2, pr.Prompts.Info))
if restore_dotfiles:
cl.echo(pr.prompt("Restore dotfiles", 2, pr.Prompts.Info))
cl.echo(pr.prompt(dotfiles_path, 3, pr.Prompts.Info))
if restore_backup:
cl.echo(pr.prompt("Restore backup", 2, pr.Prompts.Info))
cl.echo(pr.prompt(backup_path, 3, pr.Prompts.Info))
if install_packages:
cl.echo(pr.prompt("Install packages", 2, pr.Prompts.Info))
for package in packages_to_install:
cl.echo(pr.prompt(package, 3, pr.Prompts.Info))
cl.echo()
# Confirmation prompt
if cl.confirm(pr.prompt("Start installation?", 0, pr.Prompts.Question)):
cl.echo()
break
else:
cl.echo()
cl.echo(pr.prompt("Starting over.", 0, pr.Prompts.Info))
cl.echo()
""" install """
cl.echo(pr.prompt("Starting installation.", 0, pr.Prompts.Info))
# Restore dotfiles
if restore_dotfiles:
cl.echo(pr.prompt("Restoring dotfiles...", 1, pr.Prompts.Info))
execute(
"git clone --bare {} $HOME/.cfg &>> {}".format(dotfiles_path, logfile_path),
"git --git-dir=$HOME/.cfg/ --work-tree=$HOME checkout -f &>> {}".format(logfile_path),
"echo \".cfg\" >> .gitignore &>> {}".format(logfile_path)
)
# Restore backup
if restore_backup:
restore_backup_fun(backup_path)
# Install packages
if install_packages:
if package_manager == pacman:
# Update
cl.echo(pr.prompt("Updating packages...", 1, pr.Prompts.Info))
execute("sudo pacman -Syu --noconfirm &>> {}".format(logfile_path))
# Install essential packages
cl.echo(pr.prompt("Installing essential packages...", 1, pr.Prompts.Info))
for package in pacman["essential"]:
cl.echo(pr.prompt("Installing {}...", 2, pr.Prompts.Info, bold_text=package))
execute("sudo pacman -S --noconfirm {} &>> {}".format(package, logfile_path))
# Install packages
cl.echo(pr.prompt("Installing packages...", 1, pr.Prompts.Info))
for package in packages_to_install:
cl.echo(pr.prompt("Installing {}...", 2, pr.Prompts.Info, bold_text=package))
execute("yay -S --answerclean None --answerdiff None --ask no {} ".format(package))
elif package_manager == deb:
# Update
cl.echo(pr.prompt("Updating packages...", 1, pr.Prompts.Info))
execute(
"sudo apt-get update &>> {}".format(logfile_path),
"sudo apt-get upgrade &>> {}".format(logfile_path)
)
# Install packages
cl.echo(pr.prompt("Installing packages...", 1, pr.Prompts.Info))
for package in packages_to_install:
cl.echo(pr.prompt("Installing {}...", 2, pr.Prompts.Info, bold_text=package))
execute("sudo apt install -y {} &>> {}".format(package, logfile_path))
# Change shell
if change_shell:
cl.echo(pr.prompt("Changing shell...", 1, pr.Prompts.Info))
execute("sudo chsh -s /usr/bin/fish {} &>> {}".format(env_user, logfile_path))
# Done
cl.echo(pr.prompt("Installation complete.", 0, pr.Prompts.Info))
cl.echo(pr.prompt("Please log out for changes to take effect", 0, pr.Prompts.Info))
def test_prompt_does_not_strip_whitespace(mock_raw_input):
returned_value = prompt('What is your name?', strip=False)
assert_equals(returned_value, ' Dave ')
def test_prompt_returns_default_with_only_whitespace_input(mock_raw_input):
returned_value = prompt('What is your name?', default='Dave')
assert_equals(returned_value, 'Dave')
def create_encoder(parser, context, args):
parser.add_argument('--files', required=True, metavar='FILE', type=str,
nargs='+',
help='File to process')
parser.add_argument('--vocabulary-size', type=int, default=50000,
help='Maximum number of word in the vocabulary')
parser.add_argument('--char-size', type=int, default=200,
help='Maximum number of characters in the vocabulary')
parser.add_argument('--char-count', type=int, default=1,
help='Minimum count of characters in the vocabulary (all characters are\
included by default)')
parser.add_argument('--tokenizer', type=str, default='word',
choices=('space', 'char', 'word'),
help='Tokenizer flag (space, char, word)')
parser.add_argument('--lowercase', action='store_true',
help='Lowercase the data')
parser.add_argument('--hybrid', action='store_true',
help='Create a hybrid word/character vocabulary')
parser.add_argument('--save-to', required=True, metavar='FILE', type=str,
help='Output file name')
args = parser.parse_args(args)
from modules.text import TextEncoder
if args.tokenizer == 'char':
tokenize = lambda s: list(s.strip())
elif args.tokenizer == 'space' or args.tokenizer == 'bpe':
tokenize = str.split
elif args.tokenizer == 'word':
import nltk
from nltk import word_tokenize as tokenize
token_count = Counter()
char_count = Counter()
character = args.tokenizer == 'char'
for filename in args.files:
log.info('Processing %s' % os.path.basename(filename))
with open(filename, 'rt') as f:
for line in f:
line = line.lower() if args.lowercase else line
tokens = tokenize(line)
token_count.update(tokens)
if args.hybrid:
char_count.update(''.join(tokens))
f.close()
log.info('Creating %s encoder' % os.path.splitext(args.save_to)[0])
if args.hybrid:
char_encoder = TextEncoder(
counts = char_count,
min_count = char_count,
max_vocab = args.char_size,
special=('<UNK>',))
encoder = TextEncoder(
counts = token_count,
max_vocab = args.vocabulary_size,
sub_encoder = char_encoder)
else:
encoder = TextEncoder(
counts=token_count,
max_vocab=args.vocabulary_size,
min_count=char_count if args.tokenizer == 'char' else None,
special=('<S>', '</S>') + (() if args.tokenizer == 'char' else ('<UNK>',))
)
if os.path.isfile(os.path.splitext(args.save_to)[0] + ".vocab"):
if not yesno('Encoder already exist. Replace?', default='yes'):
args.save_to = prompt('%s' % log('Enter new encoder name:'))
log.info('Exporting %s encoder' % os.path.basename(args.save_to))
with open(os.path.splitext(args.save_to)[0] + ".vocab", 'wb') as f:
pickle.dump(encoder, f, -1)
f.close()
log.info('Success')
def test_prompt_returns_input(mock_raw_input):
returned_value = prompt('What is your name?')
assert_equals(returned_value, 'Dave')
def test_prompt_returns_default_for_blank_input(mock_raw_input):
returned_value = prompt('What is your name?', default='Dave')
assert_equals(returned_value, 'Dave')
def create_model(parser, context, args):
parser.add_argument('--source-encoder', type=str, metavar='FILE',
default=None, required=True,
help='load source vocabulary ')
parser.add_argument('--target-encoder', type=str, metavar='FILE',
default=None, required=True,
help='load target vocabulary')
parser.add_argument('--source-tokenizer', type=str,
choices=('word', 'space', 'char', 'bpe'), default='word',
help='Type of Preprocessing source text')
parser.add_argument('--target-tokenizer', type=str,
choices=('word', 'space', 'char', 'bpe'), default='char',
help='Type of Preprocessing target text')
parser.add_argument('--alpha', type=float, default=0.01, metavar='X',
help='Length penalty weight during beam translation')
parser.add_argument('--beta', type=float, default=0.4, metavar='X',
help='Coverage penalty weight during beam translation')
parser.add_argument('--gamma', type=float, default=1.0, metavar='X',
help='Over attention penalty weight during beam translation')
parser.add_argument('--decoder-gate', type=str,
choices=('lstm', 'context'), default='lstm',
help='Tyoe of decoder gate (lstm or context)')
parser.add_argument('--len-smooth', type=float, default=5.0, metavar='X',
help='Smoothing constant for length penalty during beam translation')
parser.add_argument('--word-embedding-dims', type=int, metavar='N',
default=256,
help='Size of word embeddings')
parser.add_argument('--target-embedding-dims', type=int, metavar='N',
default=None,
help='Size of target embeddings (default: size of input word or char embedding')
parser.add_argument('--char-embedding-dims', type=int, metavar='N',
default=64,
help='Size of character embeddings')
parser.add_argument('--dropout', type=float, metavar='FRACTION',
default=0.0,
help='Use dropout for non-recurrent connections with the given factor')
parser.add_argument('--encoder-state-dims', type=int, metavar='N',
default=256,
help='Size of encoder state')
parser.add_argument('--decoder-state-dims', type=int, metavar='N',
default=512,
help='Size of decoder state')
parser.add_argument('--attention-dims', type=int, metavar='N',
default=256,
help='Size of attention vectors')
parser.add_argument('--alignment-loss', type=float, metavar='X',
default=0.0,
help='Alignment cross-entropy contribution to loss function (DEPRECATED)')
parser.add_argument('--alignment-decay', type=float, metavar='X',
default=0.9999,
help='Decay factor of alignment cross-entropy contribution (DEPRECATED)')
parser.add_argument('--layer-normalization', action='store_true',
help='Use layer normalization')
parser.add_argument('--recurrent-dropout', type=float, metavar='FRACTION',
default=0.0,
help='Use dropout for recurrent connections with the given factor')
parser.add_argument('--source-lowercase', action='store_true',
help='Convert source text to lowercase before processing')
parser.add_argument('--target-lowercase', action='store_true',
help='convert target text to lowercase before processing')
parser.add_argument('--backwards', action='store_true',
help='Reverse the order (token level) of all input data')
parser.add_argument('--save-model', type=str, metavar='FILE',
default=None, required=True,
help='Output Model file')
args = parser.parse_args(args)
log.info('Loading Source language encoder')
with open(args.source_encoder, 'rb') as f:
args.source_encoder = pickle.load(f)
f.close()
log.info('Loading Target language encoder')
with open(args.target_encoder, 'rb') as f:
args.target_encoder = pickle.load(f)
f.close()
if args.target_embedding_dims is None:
args.target_embedding_dims = (
args.char_embedding_dims
if args.target_tokenizer == 'char'
else args.word_embedding_dims)
log.info('Configuring model')
config = {
'ts_train': 0, # total training time in seconds
'tn_epoch': 0, # total number of epochs
'source_encoder': args.source_encoder, #
'target_encoder': args.target_encoder, #
'source_lowercase': args.source_lowercase, # False
'source_tokenizer': args.source_tokenizer, # word
'target_lowercase': args.target_lowercase, # False
'target_tokenizer': args.target_tokenizer, # char
'source_embedding_dims': args.word_embedding_dims, # 256
'source_char_embedding_dims': args.char_embedding_dims, # 64
'target_embedding_dims': args.target_embedding_dims, # None
'char_embeddings_dropout': args.dropout, # 0.0
'embeddings_dropout': args.dropout, # 0.0
'recurrent_dropout': args.recurrent_dropout, # 0.0
'dropout': args.dropout, # 0.0
'encoder_state_dims': args.encoder_state_dims, # 256
'decoder_state_dims': args.decoder_state_dims, # 512
'attention_dims': args.attention_dims, # 256
'layernorm': args.layer_normalization, # False
'alignment_loss': args.alignment_loss, # 0.0
'alignment_decay': args.alignment_decay, # 0.9999
'backwards': args.backwards, # False
'decoder_gate': args.decoder_gate, # lstm
'alpha': args.alpha, # 0.01
'beta': args.beta, # 0.4
'gamma': args.gamma, # 1.0
'decoder_gate': args.decoder_gate, # lstm
'len_smooth': args.len_smooth, # 5.0
'encoder_layernorm': 'ba2' if args.layer_normalization else False,
'decoder_layernorm': 'ba2' if args.layer_normalization else False
}
if not config['source_encoder'].sub_encoder:
log.warning('Source encoder is not hybrid')
log.info('Checking existence')
if os.path.isfile(args.save_model):
if not yesno(log('Model %s exist, replace? ' % os.path.basename(args.save_model)), default='yes'):
args.save_model = prompt(log('New model name: '))
log.info('Creating model')
model = NMT('nmt', config)
log.info('Saving %s' % os.path.basename(args.save_model))
with open(args.save_model, 'wb') as f:
pickle.dump(config, f)
model.save(f)
f.close()
log.info('Model Saved')
def test_prompt_does_not_strip_whitespace(mock_raw_input):
returned_value = prompt('What is your name?', strip=False)
assert_equals(returned_value, ' Dave ')
def test_prompt_returns_default_with_only_whitespace_input(mock_raw_input):
returned_value = prompt('What is your name?', default='Dave')
assert_equals(returned_value, 'Dave')
def test_prompt_returns_input(mock_raw_input):
returned_value = prompt('What is your name?')
assert_equals(returned_value, 'Dave')
def input(self, *args, **kwargs):
return prompter.prompt(*args, **kwargs)
def get_username(self):
self.username = prompt('Enter your username:'******'\r\n > ')
