def parse(commit):
'''
Parse the commit object into a JSON.
'''
payload, messages = {}, []
cat_file = ['git', 'cat-file', 'commit', commit]
execute = Popen(cat_file, stdout=PIPE, stdin=PIPE)
stdout, _ = execute.communicate()
for line in stdout.split('\n'):
if not re.search(r'^(author|committer|tree|parent|commit)', line):
messages.append(line)
if execute.returncode == 0:
payload.update({'length': len(stdout)})
payload.update({
'raw': stdout,
'length': len(stdout),
'commit': commit,
'message': '\n'.join(messages),
})
if DEBUG_FLAG:
print fill('[parse] Contents of {commit}:\n'.format(commit=commit))
print payload['raw']
print DELIMITER
return payload
def generatePlainText(self):
resultText = "Join the " + self.city + " Sunday Night Film Club "
resultText += "(" + self.clubURL + ") this "
resultText += self.nextSunday.strftime("%A, %b %e") + self.daySuffix
resultText += " at " + self.showTime + " for " + self.film
resultText += ' at the ' + self.location + ". "
resultText += 'Look for ' + self.host + " "
resultText += "wearing "
resultText += self.wearing
resultText += " in the theatre lobby about 15 "
resultText += "minutes before the film. As always, after the film "
resultText += "we will descend on a local establishment for "
resultText += "dinner/drinks/discussion.\n\n"
resultText = textwrap.fill(resultText, 70)
resultText += "\n\n"
# hacky attempt at breaking synopsis up into paragraphs...
r = re.compile("^\r$", re.MULTILINE)
syn = r.split(self.synopsis)
for x in syn:
s = x
s = re.sub("\r", "", s)
s = re.sub("\n", "", s)
s = textwrap.dedent(s).strip()
resultText += textwrap.fill(s, 70)
resultText += "\n\n"
return resultText
def WrapStr(myStr):
LineWrap = int(Prefs['Line_Length'])
if Prefs['Line_Wrap']:
Log.Debug('Wrapped Output is: %s' %(fill(myStr, LineWrap)))
return fill(myStr, LineWrap)
else:
return myStr
def display_pull_request(pull_request):
"""Nicely display_pull_request info about a given pull request"""
display_pull_request_minimal(pull_request)
description_indent = options['description-indent']
print "%s%s" % (description_indent, color_text(pull_request.get('html_url'), 'display-title-url'))
pr_body = pull_request.get('body')
if pr_body and pr_body.strip():
pr_body = re.sub('(<br\s?/?>)', '\n', pr_body.strip())
if options['description-strip-newlines']:
pr_body = fill(pr_body, initial_indent=description_indent, subsequent_indent=description_indent, width=80)
else:
# Normalize newlines
pr_body = re.sub('\r?\n', '\n', pr_body)
pr_body = pr_body.splitlines()
pr_body = [fill(line.strip(), initial_indent=description_indent, subsequent_indent=description_indent, width=80) for line in pr_body]
pr_body = '\n'.join(pr_body)
print pr_body
print
def get_env(suppress_warning=False):
'''
:returns env_vars: mapping of environment variable names to resolved values
:type env_vars: dict
This method looks up the known environment variables, and if they
are not found, then attempts to resolve them by looking in the
file ~/.dnanexus_config/environment, followed by the installed
defaults in /opt/dnanexus/environment.
'''
env_vars = read_conf_dir(get_global_conf_dir())
env_vars.update(read_conf_dir(get_user_conf_dir()))
env_vars.update(read_conf_dir(get_session_conf_dir()))
env_overrides = []
for var in VAR_NAMES:
if var in os.environ:
if var in env_vars and env_vars.get(var) != os.environ[var]:
env_overrides.append(var)
env_vars[var] = os.environ[var]
elif var not in env_vars:
env_vars[var] = None
if sys.stdout.isatty():
if not suppress_warning and len(env_overrides) > 0:
sys.stderr.write(textwrap.fill("WARNING: The following environment variables were found to be different than the values last stored by dx: " + ", ".join(env_overrides), width=80) + '\n')
sys.stderr.write(textwrap.fill('To use the values stored by dx, unset the environment variables in your shell by running "source ~/.dnanexus_config/unsetenv". To clear the dx-stored values, run "dx clearenv".', width=80) + '\n')
return env_vars
def pprint(self, short=False):
"""Returns a pretty-printed version of the entry.
If short is true, only print the title."""
date_str = self.date.strftime(self.journal.config['timeformat'])
if not short and self.journal.config['linewrap']:
title = textwrap.fill(date_str + " " + self.title, self.journal.config['linewrap'])
body = "\n".join([
textwrap.fill(line+" ",
self.journal.config['linewrap'],
initial_indent="| ",
subsequent_indent="| ",
drop_whitespace=False).replace(' ', ' ')
for line in self.body.strip().splitlines()
])
else:
title = date_str + " " + self.title
body = self.body.strip()
# Suppress bodies that are just blanks and new lines.
has_body = len(self.body) > 20 or not all(char in (" ", "\n") for char in self.body)
if short:
return title
else:
return u"{title}{sep}{body}\n".format(
title=title,
sep="\n" if has_body else "",
body=body if has_body else "",
)
def __init__(self, parent, authors_edit, autogen_button, db,
copy_a_to_as_action, copy_as_to_a_action, a_to_as, as_to_a):
EnLineEdit.__init__(self, parent)
self.authors_edit = authors_edit
self.db = db
base = self.TOOLTIP
ok_tooltip = '<p>' + textwrap.fill(base+'<br><br>'+
_(' The green color indicates that the current '
'author sort matches the current author'))
bad_tooltip = '<p>'+textwrap.fill(base + '<br><br>'+
_(' The red color indicates that the current '
'author sort does not match the current author. '
'No action is required if this is what you want.'))
self.tooltips = (ok_tooltip, bad_tooltip)
self.authors_edit.editTextChanged.connect(self.update_state_and_val)
self.textChanged.connect(self.update_state)
self.autogen_button = autogen_button
self.copy_a_to_as_action = copy_a_to_as_action
self.copy_as_to_a_action = copy_as_to_a_action
autogen_button.clicked.connect(self.auto_generate)
copy_a_to_as_action.triggered.connect(self.auto_generate)
copy_as_to_a_action.triggered.connect(self.copy_to_authors)
a_to_as.triggered.connect(self.author_to_sort)
as_to_a.triggered.connect(self.sort_to_author)
self.update_state()
def __init__(self, name, plugins, gui_name, parent=None):
QWidget.__init__(self, parent)
self._layout = QVBoxLayout()
self.setLayout(self._layout)
self.label = QLabel(gui_name)
self.sep = QFrame(self)
self.bf = QFont()
self.bf.setBold(True)
self.label.setFont(self.bf)
self.sep.setFrameShape(QFrame.HLine)
self._layout.addWidget(self.label)
self._layout.addWidget(self.sep)
self.plugins = plugins
self.bar = QToolBar(self)
self.bar.setStyleSheet(
'QToolBar { border: none; background: none }')
self.bar.setIconSize(QSize(32, 32))
self.bar.setMovable(False)
self.bar.setFloatable(False)
self.bar.setToolButtonStyle(Qt.ToolButtonTextUnderIcon)
self._layout.addWidget(self.bar)
self.actions = []
for p in plugins:
target = partial(self.triggered, p)
ac = self.bar.addAction(QIcon(p.icon), p.gui_name, target)
ac.setToolTip(textwrap.fill(p.description))
ac.setWhatsThis(textwrap.fill(p.description))
ac.setStatusTip(p.description)
self.actions.append(ac)
w = self.bar.widgetForAction(ac)
w.setCursor(Qt.PointingHandCursor)
w.setAutoRaise(True)
w.setMinimumWidth(100)
def repo_status(repo, tracking_branch, remote_ref, indent=0):
# Make sure the right branch is checked out
if repo.current_branch() != tracking_branch:
print " " * indent + ("Checked out branch is %s instead of %s" %
(repo.current_branch(), tracking_branch))
# Make sure the branches exist
has_tracking = repo.has_ref(tracking_branch)
has_remote = repo.has_ref(remote_ref)
if not has_tracking:
print " " * indent + "You appear to be missing the tracking branch " + \
tracking_branch
if not has_remote:
print " " * indent + "You appear to be missing the remote branch " + \
remote_ref
if not has_tracking or not has_remote:
return
# Print tracking branch status
(left, right) = repo.tracking_status(tracking_branch, remote_ref)
text = _format_tracking(tracking_branch, remote_ref, left, right)
indent_str = " " * indent
print textwrap.fill(text, initial_indent=indent_str, subsequent_indent=indent_str)
def see(ms):
clear_screen()
if LinkTopicHomeRaw[ms] == "":
update_data_home()
# Tach du lieu trang can xem vao mang
source_linebyline = convert_unicode_2_ascii(sourcetext(LinkTopicHomeRaw[ms])).splitlines()
print "\n\n\n"
print "===================================================="
print "===================================================="
print "========= Enter de xem tiep!! =========="
print "\n\n\n"
print textwrap.fill(" " + "[" + str(ms) + "]:" + TenTopicHome[ms])
print "\n"
for i in range(0, source_linebyline.__len__()):
if source_linebyline[i] != "":
print '\n'
# Wrap lai, xong in tung dong de can chinh le trai
for element in textwrap.wrap(source_linebyline[i], 75):
print " " + element
# Thay the do while
while True:
key = msvcrt.getch()
if ord(key) == 13:
break
continue
print "\n\n\n"
print "===================================================="
print "===================================================="
print "===================================================="
print "\n\n\n"
def show_plugin(self, plugin):
self.showing_widget = plugin.create_widget(self.scroll_area)
self.showing_widget.genesis(self.gui)
self.showing_widget.initialize()
self.set_tooltips_for_labels()
self.scroll_area.setWidget(self.showing_widget)
self.stack.setCurrentIndex(1)
self.showing_widget.show()
self.setWindowTitle(__appname__ + ' - ' + _('Preferences') + ' - ' +
plugin.gui_name)
self.apply_action.setEnabled(False)
self.showing_widget.changed_signal.connect(lambda :
self.apply_action.setEnabled(True))
self.restore_action.setEnabled(self.showing_widget.supports_restoring_to_defaults)
tt = self.showing_widget.restore_defaults_desc
if not self.restore_action.isEnabled():
tt = _('Restoring to defaults not supported for') + ' ' + \
plugin.gui_name
self.restore_action.setToolTip(textwrap.fill(tt))
self.restore_action.setWhatsThis(textwrap.fill(tt))
self.restore_action.setStatusTip(tt)
self.bar_title.show_plugin(plugin)
self.setWindowIcon(QIcon(plugin.icon))
self.bar.setVisible(True)
self.bb.setVisible(False)
def main():
paths = sys.argv[1:] or ['.']
print 'Importing nltk...'
try:
import nltk
except ImportError:
print 'Unable to import nltk -- check your PYTHONPATH.'
sys.exit(-1)
print 'Finding definitions of deprecated funtions & classes in nltk...'
find_deprecated_defs(nltk.__path__[0])
print 'Looking for possible uses of deprecated funcs & classes...'
dep_names = print_deprecated_uses(paths)
if not dep_names:
print 'No deprecated funcs or classes found!'
else:
print "\n"+term.BOLD+"What you should use instead:"+term.NORMAL
for name in sorted(dep_names):
msgs = deprecated_funcs[name].union(
deprecated_classes[name]).union(
deprecated_methods[name])
for msg, prefix, suffix in msgs:
print textwrap.fill(term.RED+prefix+name+suffix+
term.NORMAL+': '+msg,
width=75, initial_indent=' '*2,
subsequent_indent=' '*6)
def _tooltip(self, widget, x, y, keyboard_mode, tooltip):
""" Show current event in tooltip """
# Silver Rain
silver = Gtk.Label()
silver.set_markup("<b>{0}</b>".format(_("Silver Rain")))
# Icon
img = Gtk.Image.new_from_pixbuf(self._event_icon)
# Program
title = Gtk.Label()
str = textwrap.fill(self._event_title, 21)
title.set_markup("<b>" + str + "</b>")
title.set_alignment(0, 0.5)
host = Gtk.Label()
str = textwrap.fill(self._event_host, 21)
host.set_text(str)
host.set_alignment(0, 0.5)
time = Gtk.Label()
time.set_text(self._event_time)
time.set_alignment(0, 0.5)
# Pack
box = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=15)
box.set_border_width(10)
grid = Gtk.Grid()
grid.set_column_spacing(20)
grid.attach(img, 0, 0, 1, 3)
grid.attach(title, 1, 0, 1, 1)
grid.attach(host, 1, 1, 1, 1)
grid.attach(time, 1, 2, 1, 1)
box.pack_start(silver, False, False, 0)
box.pack_start(grid, False, False, 0)
# Show
box.show_all()
tooltip.set_custom(box)
return True
def print_options(option_list):
import textwrap
from auxcodes import bcolors
# expected to be called if a config file is not specified. Print a
# list of options
option_list=sorted(option_list,key=lambda x:x[1])
width,height=_get_terminal_size_linux()
sections=sorted(set(x[0] for x in option_list))
klen=max([len(x[1]) for x in option_list])
tlen=max([len(typename(x[2])) for x in option_list])
fstring='%-'+str(klen)+'s = %-'+str(tlen)+'s (default %s)'
indent=' '*(klen+3)
for s in sections:
print bcolors.OKBLUE+'\n[%s]' % s+bcolors.ENDC
for o in option_list:
if len(o)==4:
(section, name, otype, default)=o
doc=None
elif len(o)==5:
(section, name, otype, default, doc)=o
else:
print 'Oops!',o
continue
if section==s:
print bcolors.BOLD+fstring % (name, typename(otype), str(default))+bcolors.ENDC
if doc is not None:
print textwrap.fill(doc,width-1,initial_indent=indent,subsequent_indent=indent)
def promptForLogicalInfo():
bundle = {"name" : "", "hosts" : [], "bundle_location" : "", "version" : 1}
print "\n" + textwrap.fill("A notary is a 'logical' entity that represents an " \
"arbitrary number of physical hosts. To create a " \
"notary bundle, this script will prompt you for general " \
"information about the logical notary entity, and then for " \
"information about the physical notary hosts.", 78)
print "\n\n" + textwrap.fill("First, please enter the name of the entity managing this notary. " \
"For an individual, this would be an individual's " \
"name (eg: John Smith). For an organization, this " \
"would be the organization's name (eg: Acme).", 78) + "\n"
bundle['name'] = loopingPrompt("Notary name: ")
print "\n\n" + textwrap.fill("Next, please enter the complete URL for where this bundle will " \
"be hosted (eg: https://thoughtcrime.org/thoughtcrime.notary). It must " \
"be an https URL, and the file must have a '.notary' " \
"extension. This location will be periodically checked by clients for " \
"updates to your notary configuration.", 78) + "\n"
bundle['bundle_location'] = loopingPrompt("Bundle location: ")
while (not bundle['bundle_location'].startswith("https://")) or (not bundle['bundle_location'].endswith(".notary")):
print textwrap.fill("Sorry, the bundle location must be an HTTPS URL and have a '.notary' file extension.", 78)
bundle['bundle_location'] = loopingPrompt("Bundle location: ")
return bundle
def print_dict(d, dict_property="Property", wrap=0):
pt = prettytable.PrettyTable([dict_property, 'Value'], print_empty=False)
pt.align = 'l'
for k, v in sorted(six.iteritems(d)):
# convert dict to str to check length
if isinstance(v, dict):
v = jsonutils.dumps(v)
# if value has a newline, add in multiple rows
# e.g. fault with stacktrace
if v and isinstance(v, six.string_types) and r'\n' in v:
lines = v.strip().split(r'\n')
col1 = k
for line in lines:
if wrap > 0:
line = textwrap.fill(six.text_type(line), wrap)
pt.add_row([col1, line])
col1 = ''
else:
if wrap > 0:
v = textwrap.fill(six.text_type(v), wrap)
pt.add_row([k, v])
encoded = encodeutils.safe_encode(pt.get_string())
# FIXME(gordc): https://bugs.launchpad.net/oslo-incubator/+bug/1370710
if six.PY3:
encoded = encoded.decode()
print(encoded)
def show_info(self):
self.info['Path'] = os.path.join('modules', self._modulename) + '.py'
print('')
# meta info
for item in ['Name', 'Path', 'Author', 'Version']:
if item in self.info:
print('%s: %s' % (item.rjust(10), self.info[item]))
#dirs = self._modulename.split('/')
#if dirs[0] == 'recon':
# print('%s: %s => %s' % ('Transform'.rjust(10), dirs[1].upper(), dirs[2].upper()))
print('')
# description
if 'Description' in self.info:
print('Description:')
print('%s%s' % (self.spacer, textwrap.fill(self.info['Description'], 100, subsequent_indent=self.spacer)))
print('')
# options
print('Options:', end='')
self.show_options()
# sources
if hasattr(self, '_default_source'):
print('Source Options:')
print('%s%s%s' % (self.spacer, 'default'.ljust(15), self._default_source))
print('%s%sstring representing a single input' % (self.spacer, '<string>'.ljust(15)))
print('%s%spath to a file containing a list of inputs' % (self.spacer, '<path>'.ljust(15)))
print('%s%sdatabase query returning one column of inputs' % (self.spacer, 'query <sql>'.ljust(15)))
print('')
# comments
if 'Comments' in self.info and self.info['Comments']:
print('Comments:')
for comment in self.info['Comments']:
print('%s%s' % (self.spacer, textwrap.fill('* %s' % (comment), 100, subsequent_indent=self.spacer)))
print('')
def setup(config_class, node_factory):
from .client.node import ClientNodeConfig
config = config_class.get_config()
print
msg = 'Your configuration file is located at %s. ' % config.config_path
msg += 'You can edit the configuration file with a text editor.'
print textwrap.fill(msg)
print
if not config.get('node_id'):
if not prompt_register(config):
raw_node_id = raw_input('Enter the Node ID assigned to you: ')
config.set('node_id', raw_node_id.strip())
if not config.get('recert_code'):
raw_recert_code = raw_input('Enter the Recert Code assigned to you: ')
config.set('recert_code', raw_recert_code.strip())
if config_class is ClientNodeConfig and config.encrypt_passphrase:
set_encryption_passphrase(config, save=False)
config.save()
# See if we have a cert.
if not os.path.exists(config.cert_path):
print 'Fetching your certificate.'
node = node_factory(config=config)
node.request_new_certificate()
if not config.encrypt_passphrase:
set_encryption_passphrase(config)
def setup_basic(config):
msg = 'Your configuration file is located at %s. ' % config.config_path
msg += 'You can edit the configuration file with a text editor.'
print textwrap.fill(msg)
print
def setup_value(value_name, value_text):
change = False
if not config.get(value_name):
change = True
else:
print 'Current value for "%s": %s' % \
(value_text, config.get(value_name))
response = input_yesno('Change %s?' % value_text, 'no')
if response == 'yes':
change = True
if change:
raw_node_id = raw_input('Enter value for %s: ' % value_text)
config.set(value_name, raw_node_id.strip())
config.save()
# Node ID
setup_value('node_id', 'Node ID')
print
# Cert Code
setup_value('recert_code', 'Certificate Code')
print
def demo(train_size=100, test_size=100, java_home=None, mallet_home=None):
from nltk.corpus import brown
import textwrap
# Define a very simple feature detector
def fd(sentence, index):
word = sentence[index]
return dict(word=word, suffix=word[-2:], len=len(word))
# Let nltk know where java & mallet are.
nltk.internals.config_java(java_home)
nltk.classify.mallet.config_mallet(mallet_home)
# Get the training & test corpus. We simplify the tagset a little:
# just the first 2 chars.
def strip(corpus): return [[(w, t[:2]) for (w,t) in sent]
for sent in corpus]
brown_train = strip(brown.tagged_sents(categories='news')[:train_size])
brown_test = strip(brown.tagged_sents(categories='editorial')[:test_size])
crf = MalletCRF.train(fd, brown_train, #'/tmp/crf-model',
transduction_type='VITERBI')
sample_output = crf.tag([w for (w,t) in brown_test[5]])
acc = nltk.tag.accuracy(crf, brown_test)
print '\nAccuracy: %.1f%%' % (acc*100)
print 'Sample output:'
print textwrap.fill(' '.join('%s/%s' % w for w in sample_output),
initial_indent=' ', subsequent_indent=' ')+'\n'
# Clean up
print 'Clean-up: deleting', crf.filename
os.remove(crf.filename)
return crf
def paragraphs(text):
# loosly based on brett cannon's recipe:
# http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/358228
accum = []
blank = ""
plain, quoted = object(), object()
state = plain
if type(text) is list:
for msg in text:
if msg.get_content_type() == "text/plain":
yield msg.get_payload()
raise StopIteration
else:
yield str(msg)[0]
raise StopIteration
for line in text.split("\n"):
if line.strip() == blank:
if accum:
yield textwrap.fill(" ".join(accum))
accum = []
yield "\n"
elif line.startswith(">"):
if accum:
yield textwrap.fill(" ".join(accum)); accum=[]
pass
elif line.startswith("\t") or line.startswith(" "):
yield line
else:
accum.append(line)
def hasher(filename, blocksize=-1):
# Hash the file, returning MD5 and SHA1 in an optimal manner. Most of this code is error handling.
hashmd5 = hashlib.md5()
hashsha1 = hashlib.sha1()
try:
with open(filename, "rb") as f:
for block in iter(lambda: f.read(blocksize), ""):
hashmd5.update(block)
hashsha1.update(block)
except IOError:
err = "Error: Unable to read the file \"%s\". Make sure you have permission to read this file. You may have " \
"to disable or uninstall anti-virus to stop it from denying access to the file. Correct this " \
"problem and try again." % filename
sys.stderr.write(textwrap.fill(err, width=term_width()) + "\n")
sys.exit(-1)
except MemoryError:
# OOM, revert to the smaller blocksize for this file
#print "DEBUG: Reducing block size for the file %s"%filename
if blocksize != -1:
# Blocksize is already small - bail
err = "Error: Unable to read the file \"%s\" into memory. This could be caused by anti-virus, or by " \
"other system instability issues. Kill some running processes before trying again." % filename
sys.stderr.write(textwrap.fill(err, width=term_width()) + "\n")
sys.exit(-1)
# We can't recover if this fails, so no try/except block.
with open(filename, "rb") as f:
for block in iter(lambda: f.read(SMALLBLOCK), ""):
hashmd5.update(block)
hashsha1.update(block)
return (hashmd5.hexdigest(), hashsha1.hexdigest())
def get_spell_info(spell, width=70):
spellDict = lib.registry.lookup_by_name(spell=spell)
spellObj = spellDict['spell']
if not spellDict['spell']:
raise RuntimeError()
yield ' * Name: %s' % spellObj.__name__
yield textwrap.fill(
' * Description: %s' % spellObj.__doc__,
width=width, subsequent_indent=' ' * 4
)
if spellObj.config:
yield ' * Required configs:'
for config in spellObj.config:
yield ' - %s' % config
else:
yield ' * Required configs: None'
if spellDict['test']:
queries = spellDict['test'].collectQueries()
yield ' * Example usage:'
for query, result in queries.iteritems():
yield textwrap.fill(
' >>> %s' % query,
width=width, subsequent_indent=' ' * 10
)
yield textwrap.fill(
' ... %s' % result,
width=width, subsequent_indent=' ' * 10
)
def look(self): print "-" * 42 print self.name.center(42, " ") print "-" * 42 print "" print textwrap.fill(self.ldesc, 42) print ""
def main_parser():
parser = argparse.ArgumentParser(
usage=argparse.SUPPRESS,
description=textwrap.fill(textwrap.dedent("""\
OpenERP Command provides a set of command-line tools around
the OpenERP framework: openobject-server. All the tools are
sub-commands of a single oe executable.""")),
epilog="""Use <command> --help to get information about the command.""",
formatter_class=argparse.RawDescriptionHelpFormatter,
)
description = []
for x in command_list_server:
description.append(x.__name__[len(__package__)+1:])
if x.__doc__:
description.extend([
":\n",
textwrap.fill(str(x.__doc__).strip(),
subsequent_indent=' ',
initial_indent=' '),
])
description.append("\n\n")
subparsers = parser.add_subparsers(
title="Available commands",
help=argparse.SUPPRESS,
description="".join(description[:-1]),
)
# Server-side commands.
for x in command_list_server:
x.add_parser(subparsers)
# Client-side commands. TODO one per .py file.
for x in command_list_client:
x(subparsers)
return parser
def _extensionIndexCommitMessage(self, name, description, update, wrap=True):
args = description.__dict__
args["name"] = name
if update:
template = textwrap.dedent("""\
ENH: Update %(name)s extension
This updates the %(name)s extension to %(scmrevision)s.
""")
if wrap:
paragraphs = (template % args).split("\n")
return "\n".join([textwrap.fill(p, width=76) for p in paragraphs])
else:
return template % args
else:
template = textwrap.dedent("""\
ENH: Add %(name)s extension
Description:
%(description)s
Contributors:
%(contributors)s
""")
if wrap:
for key in args:
args[key] = textwrap.fill(args[key], width=72)
return template % args
def toolTip(self, column):
"""Return tooltip for column."""
try:
ds = self.doc.data[self.data[0]]
except KeyError:
return None
c = self.cols[column]
if c == "name":
text = ds.description()
if text is None:
text = ''
if ds.tags:
text += '\n\n' + _('Tags: %s') % (' '.join(sorted(ds.tags)))
return textwrap.fill(text, 40)
elif c == "size" or (c == 'type' and 'size' not in self.cols):
text = ds.userPreview()
# add preview of dataset if possible
pix = self.getPreviewPixmap(ds)
if pix:
text = text.replace("\n", "<br>")
text = "<html>%s<br>%s</html>" % (text, utils.pixmapAsHtml(pix))
return text
elif c == "linkfile" or c == "type":
return textwrap.fill(ds.linkedInformation(), 40)
return None
def print_dict(d, dict_property="Property", wrap=0):
pt = prettytable.PrettyTable([dict_property, 'Value'],
caching=False, print_empty=False)
pt.align = 'l'
for k, v in sorted(six.iteritems(d)):
# convert dict to str to check length
if isinstance(v, dict):
v = str(v)
if isinstance(v, six.string_types):
v = strutils.safe_encode(v)
# if value has a newline, add in multiple rows
# e.g. fault with stacktrace
if v and isinstance(v, six.string_types) and r'\n' in v:
lines = v.strip().split(r'\n')
col1 = k
for line in lines:
if wrap > 0:
line = textwrap.fill(str(line), wrap)
pt.add_row([col1, line])
col1 = ''
else:
if wrap > 0:
v = textwrap.fill(str(v), wrap)
pt.add_row([k, v])
print(pt.get_string())
def main():
if len(args) < 1:
sys.stderr.write("No rating string provided! Exiting...\n")
sys.exit(1)
rating_string = ' '.join(args)
print "Input composite rating string: ", rating_string
prepped_string = prep_composite_rating(rating_string)
print "Prepped composite rating string:", prepped_string
rating_tables = get_tables(prepped_string)
where_clause = prep_composite_rating(options.where_clause)
print "Where clause:", where_clause
where_tables = get_tables(where_clause)
tables = sorted(list(set(where_tables+rating_tables)))
print "MySQL DB tables to use:", ', '.join(tables)
query = build_query(prepped_string, where=where_clause)
print "MySQL query to compute composite rating:"
print textwrap.fill(query, width=72, initial_indent='\t', subsequent_indent='\t')
produce_report(prepped_string, numbins=options.numbins, \
where=where_clause, norm=options.norm, \
log=options.log)
def getHelpTopic(self, topic, app):
import io
import textwrap
from piecrust.chefutil import print_help_item
with io.StringIO() as tplh:
extensions = app.plugin_loader.getCommandExtensions()
for e in extensions:
if e.command_name == 'prepare' and e.supports(app):
for n in e.getTemplateNames(app):
d = e.getTemplateDescription(app, n)
print_help_item(tplh, n, d)
help_list = tplh.getvalue()
help_txt = (
textwrap.fill(
"Running the 'prepare' command will let "
"PieCrust setup a page for you in the correct place, with "
"some hopefully useful default text.") +
"\n\n" +
textwrap.fill("The following templates are available:") +
"\n\n" +
help_list +
"\n" +
"You can add user-defined templates by creating pages in a "
"`scaffold/pages` sub-directory in your website.")
return help_txt
def write_text(text_file, output):
output_filename = text_file.replace('.txt', '_trigrams.txt')
with open(output_filename, 'w') as outfile:
outfile.write(textwrap.fill(output, 80))
def wrap(string, max_width):
return textwrap.fill(string, max_width)
def prompt(text,
validator,
confirm=False,
helptext='No help text provided',
errortext='Invalid response!'):
"""Prompts for and validates text provided by user
Args:
text (str): the prompt to present to the user
validator (mixed): the dict, list, or string used to validate the
repsonse
confirm (bool): if true, user will be prompted to confirm value
helptext (str): text to show if user response is "?"
errortext (str): text to return if user response does not validate
Return:
Validated response to prompt
"""
# Prepare string
text = '{} '.format(text.rstrip())
# Prepare validator
if isinstance(validator, str):
validator = re.compile(validator, re.U)
elif isinstance(validator, dict) and sorted(
validator.keys()) == ['n', 'y']:
text = '{} '.format(text, '/'.join(list(validator.keys())))
elif isinstance(validator, dict):
keys = list(validator.keys())
keys.sort(key=lambda s: s.zfill(100))
options = ['{} '.format(key, validator[key]) for key in keys]
elif isinstance(validator, list):
options = ['{} '.format(i + 1, val) for i, val in enumerate(validator)]
else:
raise ValueError('Validator must be dict, list, or str')
# Validate response
loop = True
num_loops = 0
while loop:
# Print options
try:
options
except UnboundLocalError:
pass
else:
print('-' * 60 + '\nOPTIONS\n-------')
for option in options:
print(option)
print('-' * 60)
# Prompt for value
val = input(text)
if val.lower() == 'q':
print('User exited prompt')
sys.exit()
elif val.lower() == '?':
print(fill(helptext))
loop = False
elif isinstance(validator, list):
try:
result = validator[int(val) - 1]
except IndexError:
pass
else:
if num_loops >= 0:
loop = False
elif isinstance(validator, dict):
try:
result = validator[val]
except KeyError:
pass
else:
loop = False
else:
try:
validator.search(val).group()
except AttributeError:
pass
else:
result = val
loop = False
# Confirm value, if required
if confirm and not loop:
try:
result = str(result)
except UnicodeEncodeError:
result = str(result)
loop = prompt('Is this value correct: "{}"?'.format(result), {
'y': False,
'n': True
},
confirm=False)
elif loop:
print(fill(errortext))
num_loops += 1
# Return value as unicode
return result
class ConstraintToVarBoundTransform(IsomorphicTransformation):
"""Change constraints to be a bound on the variable.
Looks for constraints of form k*v + c1 <= c2. Changes bound on v to match
(c2 - c1)/k if it results in a tighter bound. Also does the same thing for
lower bounds.
"""
alias('core.constraints_to_var_bounds',
doc=textwrap.fill(textwrap.dedent(__doc__.strip())))
def __init__(self, *args, **kwargs):
"""Initialize the transformation."""
super(ConstraintToVarBoundTransform, self).__init__(*args, **kwargs)
def _create_using(self, model):
"""Create new model, applying transformation."""
m = model.clone()
self._apply_to(m)
return m
def _apply_to(self, model):
"""Apply the transformation to the given model."""
m = model
for constr in m.component_data_objects(ctype=Constraint,
active=True,
descend_into=True):
# Check if the constraint is k * x + c1 <= c2 or c2 <= k * x + c1
if constr.body.polynomial_degree() == 1:
repn = generate_canonical_repn(constr.body)
if repn.variables is not None and len(repn.variables) == 1:
var = repn.variables[0]
const = repn.constant if repn.constant is not None else 0
coef = float(repn.linear[0])
if coef == 0:
# This can happen when a one element of a bilinear term
# is fixed to zero. Obviously, do not divide by zero.
pass
else:
if constr.upper is not None:
newbound = (value(constr.upper) - const) / coef
if coef > 0:
var.setub(
min(var.ub, newbound) if var.
ub is not None else newbound)
elif coef < 0:
var.setlb(
max(var.lb, newbound) if var.
lb is not None else newbound)
if constr.lower is not None:
newbound = (value(constr.lower) - const) / coef
if coef > 0:
var.setlb(
max(var.lb, newbound) if var.
lb is not None else newbound)
elif coef < 0:
var.setub(
min(var.ub, newbound) if var.
ub is not None else newbound)
constr.deactivate()
# Sometimes deactivating the constraint will remove a
# variable from all active constraints, so that it won't be
# updated during the optimization. Therefore, we need to
# shift the value of var as necessary in order to keep it
# within its implied bounds, as the constraint we are
# deactivating is not an invalid constraint, but rather we
# are moving its implied bound directly onto the variable.
if (var.has_lb() and var.value is not None
and var.value < var.lb):
var.set_value(var.lb)
if (var.has_ub() and var.value is not None
and var.value > var.ub):
var.set_value(var.ub)
def maybe_fill(t):
if all(len(l) < width for l in t.splitlines()):
return t
else:
return textwrap.fill(t, width, **kwargs)
def _hanging_indent(self, paragraph: str, indent=" ", width: int = DEFAULT_WIDTH) -> str:
return textwrap.indent(textwrap.fill(paragraph, width=width - len(indent)), prefix=indent).lstrip()
import textwrap
s = "Python can be easy to pick up whether you're a first time programmer or you're experienced with other languages"
s_wrap_list = textwrap.wrap(s, 40)
print(s_wrap_list)
# ['Python can be easy to pick up whether', "you're a first time programmer or you're", 'experienced with other languages']
print('\n'.join(s_wrap_list))
# Python can be easy to pick up whether
# you're a first time programmer or you're
# experienced with other languages
print(textwrap.fill(s, 40))
# Python can be easy to pick up whether
# you're a first time programmer or you're
# experienced with other languages
print(textwrap.wrap(s, 40, max_lines=2))
# ['Python can be easy to pick up whether', "you're a first time programmer or [...]"]
print(textwrap.fill(s, 40, max_lines=2))
# Python can be easy to pick up whether
# you're a first time programmer or [...]
print(textwrap.fill(s, 40, max_lines=2, placeholder=' ~'))
# Python can be easy to pick up whether
# you're a first time programmer or ~
print(textwrap.fill(s, 40, max_lines=2, placeholder=' ~', initial_indent=' '))
# Python can be easy to pick up whether
# HTTP Package
# https://www.googleapis.com/books/v1/volumes?q=isbn:1101904224
import urllib.request
import json
import textwrap
with urllib.request.urlopen(
'https://www.googleapis.com/books/v1/volumes?q=isbn:1101904224') as f:
text = f.read()
decoded = text.decode('utf-8')
print(textwrap.fill(decoded, width=50))
print()
obj = json.loads(decoded)
print(obj['kind'])
print(obj['items'][0]['searchInfo']['textSnippet'])
def send_by_mail(self, to, fmts, delete_from_library, subject='', send_ids=None,
do_auto_convert=True, specific_format=None):
ids = [self.library_view.model().id(r) for r in self.library_view.selectionModel().selectedRows()] if send_ids is None else send_ids
if not ids or len(ids) == 0:
return
files, _auto_ids = self.library_view.model().get_preferred_formats_from_ids(ids,
fmts, set_metadata=True,
specific_format=specific_format,
exclude_auto=do_auto_convert,
use_plugboard=plugboard_email_value,
plugboard_formats=plugboard_email_formats)
if do_auto_convert:
nids = list(set(ids).difference(_auto_ids))
ids = [i for i in ids if i in nids]
else:
_auto_ids = []
full_metadata = self.library_view.model().metadata_for(ids,
get_cover=False)
bad, remove_ids, jobnames = [], [], []
texts, subjects, attachments, attachment_names = [], [], [], []
for f, mi, id in zip(files, full_metadata, ids):
t = mi.title
if not t:
t = _('Unknown')
if f is None:
bad.append(t)
else:
remove_ids.append(id)
jobnames.append(t)
attachments.append(f)
if not subject:
subjects.append(_('E-book:')+ ' '+t)
else:
components = get_components(subject, mi, id)
if not components:
components = [mi.title]
subjects.append(os.path.join(*components))
a = authors_to_string(mi.authors if mi.authors else
[_('Unknown')])
texts.append(_('Attached, you will find the e-book') +
'\n\n' + t + '\n\t' + _('by') + ' ' + a + '\n\n' +
_('in the %s format.') %
os.path.splitext(f)[1][1:].upper())
if mi.comments and gprefs['add_comments_to_email']:
from calibre.utils.html2text import html2text
texts[-1] += '\n\n' + _('About this book:') + '\n\n' + textwrap.fill(html2text(mi.comments))
prefix = ascii_filename(t+' - '+a)
if not isinstance(prefix, str):
prefix = prefix.decode(preferred_encoding, 'replace')
attachment_names.append(prefix + os.path.splitext(f)[1])
remove = remove_ids if delete_from_library else []
to_s = list(repeat(to, len(attachments)))
if attachments:
send_mails(jobnames,
Dispatcher(partial(self.email_sent, remove=remove)),
attachments, to_s, subjects, texts, attachment_names,
self.job_manager)
self.status_bar.show_message(_('Sending email to')+' '+to, 3000)
auto = []
if _auto_ids != []:
for id in _auto_ids:
if specific_format is None:
dbfmts = self.library_view.model().db.formats(id, index_is_id=True)
formats = [f.lower() for f in (dbfmts.split(',') if dbfmts else
[])]
if list(set(formats).intersection(available_input_formats())) != [] and list(set(fmts).intersection(available_output_formats())) != []:
auto.append(id)
else:
bad.append(self.library_view.model().db.title(id, index_is_id=True))
else:
if specific_format in list(set(fmts).intersection(set(available_output_formats()))):
auto.append(id)
else:
bad.append(self.library_view.model().db.title(id, index_is_id=True))
if auto != []:
format = specific_format if specific_format in list(set(fmts).intersection(set(available_output_formats()))) else None
if not format:
for fmt in fmts:
if fmt in list(set(fmts).intersection(set(available_output_formats()))):
format = fmt
break
if format is None:
bad += auto
else:
autos = [self.library_view.model().db.title(id, index_is_id=True) for id in auto]
if self.auto_convert_question(
_('Auto convert the following books to %s before sending via '
'email?') % format.upper(), autos):
self.iactions['Convert Books'].auto_convert_mail(to, fmts, delete_from_library, auto, format, subject)
if bad:
bad = '\n'.join('%s'%(i,) for i in bad)
d = warning_dialog(self, _('No suitable formats'),
_('Could not email the following books '
'as no suitable formats were found:'), bad)
d.exec_()
def data_str(self, include_og=True):
"""
Get description of all data, including information for OG setting.
:return: str
"""
# __str__() omits information on OG setting to reduce confusion
# as to which set of symops are active, this property gives
# all stored data including OG setting
desc = {} # dictionary to hold description strings
# parse data into strings
desc['magtype'] = self._data['magtype']
desc['bns_number'] = ".".join(map(str, self._data["bns_number"]))
desc['bns_label'] = self._data["bns_label"]
desc['og_id'] = ("\t\tOG: " +
".".join(map(str, self._data["og_number"])) + " " +
self._data["og_label"] if include_og else '')
desc['bns_operators'] = ' '.join(
[op_data['str'] for op_data in self._data['bns_operators']])
desc['bns_lattice'] = (' '.join([
lattice_data['str']
for lattice_data in self._data['bns_lattice'][3:]
]) if len(self._data['bns_lattice']) > 3 else ''
) # don't show (1,0,0)+ (0,1,0)+ (0,0,1)+
desc['bns_wyckoff'] = '\n'.join([
textwrap.fill(wyckoff_data['str'],
initial_indent=wyckoff_data['label'] + " ",
subsequent_indent=" " *
len(wyckoff_data['label'] + " "),
break_long_words=False,
break_on_hyphens=False)
for wyckoff_data in self._data['bns_wyckoff']
])
desc['og_bns_transformation'] = ('OG-BNS Transform: ({})\n'.format(
self._data['og_bns_transform']) if desc['magtype'] == 4
and include_og else '')
bns_operators_prefix = "Operators{}: ".format(
' (BNS)' if desc['magtype'] == 4 and include_og else '')
bns_wyckoff_prefix = "Wyckoff Positions{}: ".format(
' (BNS)' if desc['magtype'] == 4 and include_og else '')
# apply textwrap on long lines
desc['bns_operators'] = textwrap.fill(
desc['bns_operators'],
initial_indent=bns_operators_prefix,
subsequent_indent=" " * len(bns_operators_prefix),
break_long_words=False,
break_on_hyphens=False)
description = ("BNS: {d[bns_number]} {d[bns_label]}{d[og_id]}\n"
"{d[og_bns_transformation]}"
"{d[bns_operators]}\n"
"{bns_wyckoff_prefix}{d[bns_lattice]}\n"
"{d[bns_wyckoff]}").format(
d=desc, bns_wyckoff_prefix=bns_wyckoff_prefix)
if desc['magtype'] == 4 and include_og:
desc['og_operators'] = ' '.join(
[op_data['str'] for op_data in self._data['og_operators']])
# include all lattice vectors because (1,0,0)+ (0,1,0)+ (0,0,1)+
# not always present in OG setting
desc['og_lattice'] = ' '.join([
lattice_data['str']
for lattice_data in self._data['og_lattice']
])
desc['og_wyckoff'] = '\n'.join([
textwrap.fill(wyckoff_data['str'],
initial_indent=wyckoff_data['label'] + " ",
subsequent_indent=" " *
len(wyckoff_data['label'] + " "),
break_long_words=False,
break_on_hyphens=False)
for wyckoff_data in self._data['og_wyckoff']
])
og_operators_prefix = "Operators (OG): "
og_wyckoff_prefix = "Wyckoff Positions (OG): "
# apply textwrap on long lines
desc['og_operators'] = textwrap.fill(
desc['og_operators'],
initial_indent=og_operators_prefix,
subsequent_indent=" " * len(og_operators_prefix),
break_long_words=False,
break_on_hyphens=False)
description += ("\n{d[og_operators]}\n"
"Wyckoff Positions (OG): {d[og_lattice]}\n"
"{d[og_wyckoff]}").format(d=desc)
elif desc['magtype'] == 4:
description += '\nAlternative OG setting exists for this space group.'
return description
def format_description(self, description):
paras = description.split('\n')
formatted_paras = [
textwrap.fill(para, self.width) for para in paras
]
return '\n'.join(formatted_paras) + '\n'
def draw_figure(self, *args):
self._update_selection()
if not self._validate_inputs():
return
start_time, end_time = self.time_selector.get_inputs()
plot_type = self.vars_button.text
results = self.dfs
if plot_type in ['price of electricity (box)']:
self._reinit_graph(has_legend=False)
else:
self._reinit_graph(has_legend=True)
fig = self.current_fig
ax = self.current_ax
ax.clear()
plt.xticks(rotation=0)
plt.grid(True)
if plot_type == 'arbitrage activity':
for key in results:
df = results[key]
ax.plot((df['q_r'] - df['q_d'])[start_time:end_time],
drawstyle='steps-post',
label=textwrap.fill(key, 50))
ax.set_ylabel('MWh')
ax.set_xlabel('ending hour')
ax.set_title('Energy Charged/Discharged (MWh)')
elif plot_type == 'regulation capacity offered':
for key in results:
df = results[key]
if (df['q_reg'] != 0).any():
ax.plot(df['q_reg'][start_time:end_time],
drawstyle='steps-post',
label=textwrap.fill(key, 50))
else:
ax.plot((df['q_rd'] - df['q_ru'])[start_time:end_time],
drawstyle='steps-post',
label=textwrap.fill(key, 50))
ax.set_ylabel('MWh')
ax.set_xlabel('ending hour')
ax.set_title('Regulation Capacity Offered (MWh)')
elif plot_type == 'revenue':
for key in results:
df = results[key]
ax.plot(df['revenue'][start_time:end_time],
drawstyle='steps-post',
label=textwrap.fill(key, 50))
ax.set_ylabel('$')
ax.set_xlabel('ending hour')
ax.set_title('Cumulative Revenue Generated ($)')
ax.get_yaxis().set_major_formatter(
mpl.ticker.FuncFormatter(lambda x, p: format(int(x), ',')))
elif plot_type == 'state of charge':
for key in results:
df = results[key]
ax.plot(df['state of charge'][start_time:end_time],
drawstyle='steps-post',
label=textwrap.fill(key, 50))
ax.set_ylabel('MWh')
ax.set_xlabel('ending hour')
ax.set_title('State of Charge (MWh)')
elif plot_type == 'price of electricity':
for key in results:
df = results[key]
ax.plot(df['price of electricity'][start_time:end_time],
drawstyle='steps-post',
label=textwrap.fill(key, 50))
ax.set_ylabel('$/MWh')
ax.set_xlabel('ending hour')
ax.set_title('Price of Electricity ($/MWh)')
elif plot_type == 'price of electricity (box)':
ax.boxplot([
results[key]['price of electricity'][start_time:end_time]
for key in results
],
labels=[
textwrap.fill(' '.join(key.split()[:3]), 8)
for key in results
])
ax.set_ylabel('$/MWh')
ax.set_title('Price of Electricity ($/MWh)')
else:
for key in results:
df = results[key]
ax.plot(df[plot_type][start_time:end_time],
drawstyle='steps-post',
label=textwrap.fill(key, 50))
ax.set_title(plot_type)
if plot_type in ['price of electricity (box)']:
pass
else:
ax.legend(bbox_to_anchor=(1.02, 0.5),
loc="center left",
borderaxespad=0,
shadow=False,
labelspacing=1.8)
self.plotbox.children[0].draw()
def fill_paragraph( text, width, indent ):
""
text = whitespace_regex.sub(' ', text).strip()
return textwrap.fill( text, width, initial_indent=indent,
subsequent_indent=indent )
Values to consider as True.
false_values : list, default None
Values to consider as False.
skiprows : list-like, int, or callable, optional
Line numbers to skip (0-indexed) or number of lines to skip (int) at the
start of the file. If callable, the callable function will be evaluated
against the row indices, returning True if the row should be skipped and
False otherwise. An example of a valid callable argument would be ``lambda
x: x in [0, 2]``.
nrows : int, default None
Number of rows to parse.
na_values : scalar, str, list-like, or dict, default None
Additional strings to recognize as NA/NaN. If dict passed, specific
per-column NA values. By default the following values are interpreted
as NaN: '"""
+ fill("', '".join(sorted(STR_NA_VALUES)), 70, subsequent_indent=" ")
+ """'.
keep_default_na : bool, default True
Whether or not to include the default NaN values when parsing the data.
Depending on whether `na_values` is passed in, the behavior is as follows:
* If `keep_default_na` is True, and `na_values` are specified, `na_values`
is appended to the default NaN values used for parsing.
* If `keep_default_na` is True, and `na_values` are not specified, only
the default NaN values are used for parsing.
* If `keep_default_na` is False, and `na_values` are specified, only
the NaN values specified `na_values` are used for parsing.
* If `keep_default_na` is False, and `na_values` are not specified, no
strings will be parsed as NaN.
Note that if `na_filter` is passed in as False, the `keep_default_na` and
metavar='SYN2',
type=str,
help='a Processing syntax definition for Vim')
args = parser.parse_args()
if not os.path.isfile(args.syn1):
sys.exit("Could not open syntax file %s. Does the file exist?" %
args.syn1)
if not os.path.isfile(args.syn2):
sys.exit("Could not open syntax file %s. Does the file exist?" %
args.syn2)
for key in patterns:
w1 = get_words(args.syn1, patterns[key])
w2 = get_words(args.syn2, patterns[key])
removed = w1 - w2
added = w2 - w1
print "- %s" % key
print "Added:"
print textwrap.fill(' '.join(added),
initial_indent=' ',
subsequent_indent=' ')
print "Removed:"
print textwrap.fill(' '.join(removed),
initial_indent=' ',
subsequent_indent=' ')
print
def format_job(run_name, job):
job_id = job['job_id']
status = job['status']
description = job['description']
duration = seconds_to_hms(int(job['duration'] or 0))
# Every job gets a link to e.g. pulpito's pages
info_url = misc.get_results_url(run_name, job_id)
if info_url:
info_line = email_templates['info_url_templ'].format(info=info_url)
else:
info_line = ''
if status in UNFINISHED_STATUSES:
format_args = dict(
job_id=job_id,
desc=description,
time=duration,
info_line=info_line,
)
return email_templates['running_templ'].format(**format_args)
if status == 'pass':
return email_templates['pass_templ'].format(
job_id=job_id,
desc=description,
time=duration,
info_line=info_line,
)
else:
log_dir_url = job['log_href'].rstrip('teuthology.yaml')
if log_dir_url:
log_line = email_templates['fail_log_templ'].format(
log=log_dir_url)
else:
log_line = ''
sentry_event = job.get('sentry_event')
if sentry_event:
sentry_line = email_templates['fail_sentry_templ'].format(
sentry_event=sentry_event)
else:
sentry_line = ''
if job['failure_reason']:
# 'fill' is from the textwrap module and it collapses a given
# string into multiple lines of a maximum width as specified.
# We want 75 characters here so that when we indent by 4 on the
# next line, we have 79-character exception paragraphs.
reason = fill(job['failure_reason'] or '', 75)
reason = \
'\n'.join((' ') + line for line in reason.splitlines())
reason_lines = email_templates['fail_reason_templ'].format(
reason=reason).rstrip()
else:
reason_lines = ''
format_args = dict(
job_id=job_id,
desc=description,
time=duration,
info_line=info_line,
log_line=log_line,
sentry_line=sentry_line,
reason_lines=reason_lines,
)
return email_templates['fail_templ'].format(**format_args)
async def py_constraints(
addresses: Addresses,
console: Console,
py_constraints_subsystem: PyConstraintsSubsystem,
python_setup: PythonSetup,
registered_target_types: RegisteredTargetTypes,
union_membership: UnionMembership,
) -> PyConstraintsGoal:
if py_constraints_subsystem.summary:
if addresses:
console.print_stderr(
"The `py-constraints --summary` goal does not take file/target arguments. Run "
"`help py-constraints` for more details.")
return PyConstraintsGoal(exit_code=1)
all_expanded_targets, all_explicit_targets = await MultiGet(
Get(Targets, AddressSpecs([DescendantAddresses("")])),
Get(UnexpandedTargets, AddressSpecs([DescendantAddresses("")])),
)
all_python_targets = sorted(
{
t
for t in (*all_expanded_targets, *all_explicit_targets)
if t.has_field(InterpreterConstraintsField)
},
key=lambda tgt: cast(Address, tgt.address),
)
constraints_per_tgt = [
PexInterpreterConstraints.create_from_targets([tgt], python_setup)
for tgt in all_python_targets
]
transitive_targets_per_tgt = await MultiGet(
Get(TransitiveTargets, TransitiveTargetsRequest([tgt.address]))
for tgt in all_python_targets)
transitive_constraints_per_tgt = [
PexInterpreterConstraints.create_from_targets(
transitive_targets.closure, python_setup)
for transitive_targets in transitive_targets_per_tgt
]
dependees_per_root = await MultiGet(
Get(
Dependees,
DependeesRequest(
[tgt.address], transitive=True, include_roots=False))
for tgt in all_python_targets)
data = [{
"Target": tgt.address.spec,
"Constraints": str(constraints),
"Transitive Constraints": str(transitive_constraints),
"# Dependencies": len(transitive_targets.dependencies),
"# Dependees": len(dependees),
} for tgt, constraints, transitive_constraints, transitive_targets,
dependees in zip(
all_python_targets,
constraints_per_tgt,
transitive_constraints_per_tgt,
transitive_targets_per_tgt,
dependees_per_root,
)]
with py_constraints_subsystem.output_sink(console) as stdout:
writer = csv.DictWriter(
stdout,
fieldnames=[
"Target",
"Constraints",
"Transitive Constraints",
"# Dependencies",
"# Dependees",
],
)
writer.writeheader()
for entry in data:
writer.writerow(entry)
return PyConstraintsGoal(exit_code=0)
transitive_targets = await Get(TransitiveTargets,
TransitiveTargetsRequest(addresses))
final_constraints = PexInterpreterConstraints.create_from_targets(
transitive_targets.closure, python_setup)
if not final_constraints:
target_types_with_constraints = sorted(
tgt_type.alias for tgt_type in registered_target_types.types
if tgt_type.class_has_field(InterpreterConstraintsField,
union_membership))
logger.warning(
"No Python files/targets matched for the `py-constraints` goal. All target types with "
f"Python interpreter constraints: {', '.join(target_types_with_constraints)}"
)
return PyConstraintsGoal(exit_code=0)
constraints_to_addresses = defaultdict(set)
for tgt in transitive_targets.closure:
constraints = PexInterpreterConstraints.create_from_targets(
[tgt], python_setup)
if not constraints:
continue
constraints_to_addresses[constraints].add(tgt.address)
with py_constraints_subsystem.output(console) as output_stdout:
output_stdout(f"Final merged constraints: {final_constraints}\n")
if len(addresses) > 1:
merged_constraints_warning = (
"(These are the constraints used if you were to depend on all of the input "
"files/targets together, even though they may end up never being used together in "
"the real world. Consider using a more precise query or running "
"`./pants py-constriants --summary`.)\n")
output_stdout(indent(fill(merged_constraints_warning, 80), " "))
for constraint, addrs in sorted(constraints_to_addresses.items()):
output_stdout(f"\n{constraint}\n")
for addr in sorted(addrs):
output_stdout(f" {addr}\n")
return PyConstraintsGoal(exit_code=0)
def print_modules(self):
basepath = os.path.abspath(
os.path.join(os.path.dirname(os.path.realpath(__file__)), ".."))
rst = {}
for m in all_modules():
try:
module_type = "core"
filename = os.path.join(basepath, "modules", "core",
"{}.py".format(m))
if not os.path.exists(filename):
filename = os.path.join(basepath, "modules", "contrib",
"{}.py".format(m))
module_type = "contrib"
if not os.path.exists(filename):
log.warning("module {} not found".format(m))
continue
doc = None
with open(filename) as f:
tree = ast.parse(f.read())
doc = ast.get_docstring(tree)
if not doc:
log.warning("failed to find docstring for {}".format(m))
continue
if self._format == "rst":
if os.path.exists(
os.path.join(basepath, "..", "screenshots",
"{}.png".format(m))):
doc = "{}\n\n.. image:: ../screenshots/{}.png".format(
doc, m)
rst[module_type] = rst.get(module_type, [])
rst[module_type].append({"module": m, "content": doc})
else:
print(
textwrap.fill(
"{}:".format(m),
80,
initial_indent=self._indent * 2,
subsequent_indent=self._indent * 2,
))
for line in doc.split("\n"):
print(
textwrap.fill(
line,
80,
initial_indent=self._indent * 3,
subsequent_indent=self._indent * 6,
))
except Exception as e:
log.warning(e)
if self._format == "rst":
print("List of modules\n===============")
for k in ["core", "contrib"]:
print("\n{}\n{}\n".format(k, "-" * len(k)))
for mod in rst[k]:
print("\n{}\n{}\n".format(mod["module"],
"~" * len(mod["module"])))
print(mod["content"])
def find_jar_iter(
name_pattern,
path_to_jar=None,
env_vars=(),
searchpath=(),
url=None,
verbose=False,
is_regex=False,
):
"""
Search for a jar that is used by nltk.
:param name_pattern: The name of the jar file
:param path_to_jar: The user-supplied jar location, or None.
:param env_vars: A list of environment variable names to check
in addition to the CLASSPATH variable which is
checked by default.
:param searchpath: List of directories to search.
:param is_regex: Whether name is a regular expression.
"""
assert isinstance(name_pattern, str)
assert not isinstance(searchpath, str)
if isinstance(env_vars, str):
env_vars = env_vars.split()
yielded = False
# Make sure we check the CLASSPATH first
env_vars = ["CLASSPATH"] + list(env_vars)
# If an explicit location was given, then check it, and yield it if
# it's present; otherwise, complain.
if path_to_jar is not None:
if os.path.isfile(path_to_jar):
yielded = True
yield path_to_jar
else:
raise LookupError("Could not find %s jar file at %s" %
(name_pattern, path_to_jar))
# Check environment variables
for env_var in env_vars:
if env_var in os.environ:
if env_var == "CLASSPATH":
classpath = os.environ["CLASSPATH"]
for cp in classpath.split(os.path.pathsep):
if os.path.isfile(cp):
filename = os.path.basename(cp)
if (is_regex and re.match(name_pattern, filename) or
(not is_regex and filename == name_pattern)):
if verbose:
print("[Found %s: %s]" % (name_pattern, cp))
yielded = True
yield cp
# The case where user put directory containing the jar file in the classpath
if os.path.isdir(cp):
if not is_regex:
if os.path.isfile(os.path.join(cp, name_pattern)):
if verbose:
print("[Found %s: %s]" %
(name_pattern, cp))
yielded = True
yield os.path.join(cp, name_pattern)
else:
# Look for file using regular expression
for file_name in os.listdir(cp):
if re.match(name_pattern, file_name):
if verbose:
print("[Found %s: %s]" % (
name_pattern,
os.path.join(cp, file_name),
))
yielded = True
yield os.path.join(cp, file_name)
else:
jar_env = os.environ[env_var]
jar_iter = ((os.path.join(jar_env, path_to_jar)
for path_to_jar in os.listdir(jar_env))
if os.path.isdir(jar_env) else (jar_env, ))
for path_to_jar in jar_iter:
if os.path.isfile(path_to_jar):
filename = os.path.basename(path_to_jar)
if (is_regex and re.match(name_pattern, filename) or
(not is_regex and filename == name_pattern)):
if verbose:
print("[Found %s: %s]" %
(name_pattern, path_to_jar))
yielded = True
yield path_to_jar
# Check the path list.
for directory in searchpath:
if is_regex:
for filename in os.listdir(directory):
path_to_jar = os.path.join(directory, filename)
if os.path.isfile(path_to_jar):
if re.match(name_pattern, filename):
if verbose:
print("[Found %s: %s]" % (filename, path_to_jar))
yielded = True
yield path_to_jar
else:
path_to_jar = os.path.join(directory, name_pattern)
if os.path.isfile(path_to_jar):
if verbose:
print("[Found %s: %s]" % (name_pattern, path_to_jar))
yielded = True
yield path_to_jar
if not yielded:
# If nothing was found, raise an error
msg = "NLTK was unable to find %s!" % name_pattern
if env_vars:
msg += " Set the %s environment variable" % env_vars[0]
msg = textwrap.fill(msg + ".",
initial_indent=" ",
subsequent_indent=" ")
if searchpath:
msg += "\n\n Searched in:"
msg += "".join("\n - %s" % d for d in searchpath)
if url:
msg += "\n\n For more information, on %s, see:\n <%s>" % (
name_pattern,
url,
)
div = "=" * 75
raise LookupError("\n\n%s\n%s\n%s" % (div, msg, div))
def slim_makescript(script_file, trees_file, demographic_model, contig,
samples, mutation_types, extended_events, scaling_factor,
burn_in):
pop_names = [
pc.metadata["id"] for pc in demographic_model.population_configurations
]
# Use copies of these so that the time frobbing below doesn't have
# side-effects in the caller's model.
demographic_events = copy.deepcopy(demographic_model.demographic_events)
if extended_events is None:
extended_events = []
else:
extended_events = copy.deepcopy(extended_events)
# Reassign event times according to integral SLiM generations.
# This collapses the time deltas used in HomSap/AmericanAdmixture_4B11,
# and calculates times for GenerationAfter objects.
def fix_time(event):
for attr in ("time", "start_time", "end_time"):
if not hasattr(event, attr):
continue
t = getattr(event, attr)
t_rounded = round(float(t) / scaling_factor) * scaling_factor
if isinstance(t, stdpopsim.ext.GenerationAfter):
t_rounded -= scaling_factor
if t_rounded < 0:
raise ValueError(f"Bad {attr}: {getattr(event, attr)}")
setattr(event, attr, t_rounded)
for event in demographic_events:
fix_time(event)
for event in extended_events:
fix_time(event)
# The demography debugger constructs event epochs, which we use
# to define the forwards-time events.
dd = msprime.DemographyDebugger(
population_configurations=demographic_model.population_configurations,
migration_matrix=demographic_model.migration_matrix,
demographic_events=demographic_events)
epochs = sorted(dd.epochs, key=lambda e: e.start_time, reverse=True)
T = [
round(e.start_time * demographic_model.generation_time) for e in epochs
]
migration_matrices = [e.migration_matrix for e in epochs]
N = np.empty(shape=(dd.num_populations, len(epochs)), dtype=int)
growth_rates = np.empty(shape=(dd.num_populations, len(epochs)),
dtype=float)
for j, epoch in enumerate(epochs):
for i, pop in enumerate(epoch.populations):
N[i, j] = int(pop.end_size)
growth_rates[i, j] = pop.growth_rate
admixture_pulses = []
subpopulation_splits = []
for i, epoch in enumerate(epochs):
for de in epoch.demographic_events:
if isinstance(de, msprime.MassMigration):
if de.proportion < 1:
# Calculate remainder of population after previous
# MassMigration events in this epoch.
rem = 1 - np.sum([
ap[3] for ap in admixture_pulses
if ap[0] == i and ap[1] == de.source
])
admixture_pulses.append((
i,
de.source, # forwards-time dest
de.dest, # forwards-time source
rem * de.proportion))
continue
# Backwards: de.source is being merged into de.dest.
# Forwards: de.source is being created, taking individuals
# from de.dest.
#
# If the proportion==1, we can use SLiM function:
# sim.addSubpopSplit(newpop, size, oldpop),
# which we trigger by adding a row to subpopulation_splits.
# This SLiM function creates newpop (=de.source), under the
# assumption that it doesn't already exist.
subpopulation_splits.append(
(f"_T[{i}]", de.source, f"_N[{i+1},{de.source}]", de.dest))
# Zero out the population size for generations before this
# epoch, to avoid simulating invididuals that contribute no
# genealogy.
N[de.source, 0:(i + 1)] = 0
growth_rates[de.source, 0:(i + 1)] = 0
# Ensure there are no migrations to or from de.source before
# this epoch.
for j in range(i + 1):
for k in range(dd.num_populations):
migration_matrices[j][k][de.source] = 0
migration_matrices[j][de.source][k] = 0
drawn_mutations = []
fitness_callbacks = []
condition_on_allele_frequency = []
op_id = stdpopsim.ext.ConditionOnAlleleFrequency.op_id
for ee in extended_events:
if hasattr(ee, "mutation_type_id"):
mt_id = getattr(ee, "mutation_type_id")
cls_name = ee.__class__.__name__
if mutation_types is None:
raise ValueError(
f"Invalid {cls_name} event. No mutation types defined.")
if not (0 < ee.mutation_type_id <= len(mutation_types)):
# FIXME: use zero-based indexes
raise ValueError(
f"Invalid {cls_name} event with mutation type id {mt_id}.")
if hasattr(ee, "start_time") and hasattr(ee, "end_time"):
# Now that GenerationAfter times have been accounted for, we can
# properly catch invalid start/end times.
start_time = getattr(ee, "start_time")
end_time = getattr(ee, "end_time")
stdpopsim.ext.validate_time_range(start_time, end_time)
if isinstance(ee, stdpopsim.ext.DrawMutation):
time = ee.time * demographic_model.generation_time
save = 1 if ee.save else 0
drawn_mutations.append((time, ee.mutation_type_id,
ee.population_id, ee.coordinate, save))
elif isinstance(ee, stdpopsim.ext.ChangeMutationFitness):
start_time = ee.start_time * demographic_model.generation_time
end_time = ee.end_time * demographic_model.generation_time
fitness_callbacks.append(
(start_time, end_time, ee.mutation_type_id, ee.population_id,
ee.selection_coeff, ee.dominance_coeff))
elif isinstance(ee, stdpopsim.ext.ConditionOnAlleleFrequency):
start_time = ee.start_time * demographic_model.generation_time
end_time = ee.end_time * demographic_model.generation_time
save = 1 if ee.save else 0
condition_on_allele_frequency.append(
(start_time, end_time, ee.mutation_type_id, ee.population_id,
op_id(ee.op), ee.allele_frequency, save))
else:
raise ValueError(f"Unknown extended event type {type(ee)}")
# Check that drawn mutations exist for extended events that need them.
drawn_mut_type_ids = {mt_id for _, mt_id, _, _, _ in drawn_mutations}
for ee in extended_events:
if (isinstance(ee, stdpopsim.ext.ChangeMutationFitness)
or isinstance(ee, stdpopsim.ext.ConditionOnAlleleFrequency)):
if ee.mutation_type_id not in drawn_mut_type_ids:
cls_name = ee.__class__.__name__
raise ValueError(
f"Invalid {cls_name} event. No drawn mutation for "
"mutation type id {ee.mutation_type_id}")
printsc = functools.partial(print, file=script_file)
# Header
printsc('/*')
printsc(' * stdpopsim ' + stdpopsim.__version__)
printsc(' *')
printsc(' * Demographic model: ' + demographic_model.id)
printsc(' * ' + "\n * ".join(
[line.strip() for line in demographic_model.description.split('\n')]))
for citation in demographic_model.citations:
printsc(' * ' + str(citation))
printsc(' */')
recomb_rates, recomb_ends = msprime_rm_to_slim_rm(contig.recombination_map)
indent = 8 * " "
recomb_rates_str = ("c(\n" +
textwrap.fill(", ".join(map(str, recomb_rates)),
width=80,
initial_indent=indent,
subsequent_indent=indent) + ")")
recomb_ends_str = ("c(\n" + textwrap.fill(", ".join(map(str, recomb_ends)),
width=80,
initial_indent=indent,
subsequent_indent=indent) + ")")
pop_names_str = ', '.join(map(lambda x: f'"{x}"', pop_names))
printsc(
string.Template(_slim_upper).substitute(
scaling_factor=scaling_factor,
burn_in=float(burn_in),
chromosome_length=int(contig.recombination_map.get_length()),
recombination_rates=recomb_rates_str,
recombination_ends=recomb_ends_str,
mutation_rate=contig.mutation_rate,
generation_time=demographic_model.generation_time,
trees_file=trees_file,
pop_names=f"c({pop_names_str})"))
def matrix2str(matrix,
row_comments=None,
col_comment=None,
indent=2,
fmt="",
dim=(None, None)):
"""
Return an Eidos representation of the matrix as a string.
"""
if row_comments is not None:
assert len(matrix) == len(row_comments)
if len(matrix) == 0:
return "c()"
s = ["array(c(\n"]
if col_comment is not None:
s.append(indent * 4 * ' ' + '// ' + col_comment + '\n')
for i in range(len(matrix)):
s.append(indent * 4 * " ")
s.append('c({})'.format(", ".join(
[format(x, fmt) for x in matrix[i]])))
if i != len(matrix) - 1:
s.append(",")
if row_comments is not None:
s.append(" // " + row_comments[i])
s.append("\n")
s.append((indent - 1) * 4 * " ")
if dim[0] is None:
dim = (len(matrix[0]), dim[1])
if dim[1] is None:
dim = (dim[0], len(matrix))
s.append(f'), c({dim[0]}, {dim[1]}))')
return "".join(s)
if mutation_types is None:
mutation_types = [stdpopsim.ext.MutationType()]
# Mutation type; genomic elements.
# FIXME: Change this to use zero-based indices---one-based indices are
# inconsistent with everything else, e.g. population IDs.
for i, m in enumerate(mutation_types, 1):
distrib_args = [str(arg) for arg in m.distribution_args]
distrib_args[
m.Q_scaled_index] = "Q * " + distrib_args[m.Q_scaled_index]
distrib_args = ", ".join(distrib_args)
printsc(f' initializeMutationType("m{i}", {m.dominance_coeff}, ' +
f'"{m.distribution_type}", {distrib_args});')
if not m.convert_to_substitution:
# T is the default for WF simulations.
printsc(f' m{i}.convertToSubstitution = F;')
mut_weights = ", ".join(str(m.weight) for m in mutation_types)
printsc(' initializeGenomicElementType("g1", ' +
f'seq(1, {len(mutation_types)}), c({mut_weights}));')
printsc()
# Epoch times.
printsc(' // Time of epoch boundaries, in years before present.')
printsc(' // The first epoch spans from INF to _T[0].')
printsc(' defineConstant("_T", c({}));'.format(", ".join(map(str, T))))
printsc()
# Population sizes.
printsc(' // Population sizes in each epoch.')
printsc(' _N = ' + matrix2str(
N, row_comments=pop_names, col_comment="INF:_T[0], _T[0]:_T[1], etc.")
+ ';')
printsc()
printsc(' defineConstant("num_epochs", length(_T));')
printsc(' defineConstant("num_populations", ncol(_N));')
printsc()
# Growth rates.
printsc(' // Population growth rates for each epoch.')
printsc(' defineConstant("growth_rates", ' +
matrix2str(growth_rates,
row_comments=pop_names,
col_comment="INF:_T[0], _T[0]:_T[1], etc.",
dim=("num_epochs", "num_populations")) + ');')
printsc()
printsc(' no_migration = rep(0, num_populations*num_populations);')
printsc()
# Migration rates.
printsc(' // Migration rates for each epoch.')
printsc(
' // Migrations involving a population with size=0 are ignored.')
printsc(' // XXX: document what the rows & cols correspond to.')
printsc(' defineConstant("migration_matrices", array(c(')
for i in range(len(migration_matrices)):
epoch_str = f"INF:_T[{i}]" if i == 0 else f"_T[{i}]:_T[{i+1}]"
printsc()
printsc(2 * 4 * ' ' + '// ' + epoch_str)
end = ",\n" if i != len(migration_matrices) - 1 else "\n"
if np.all(np.array(migration_matrices[i]) == 0):
printsc(2 * 4 * ' ' + 'no_migration', end=end)
else:
printsc(2 * 4 * ' ' +
matrix2str(migration_matrices[i],
indent=3,
fmt="g",
dim=("num_populations", "num_populations")),
end=end)
printsc()
printsc(4 * ' ' + '), c(num_populations, num_populations, num_epochs)));')
printsc()
# Population splits.
printsc(' // Population splits, one row for each event.')
printsc(' defineConstant("subpopulation_splits", ' + matrix2str(
subpopulation_splits, col_comment="time, newpop, size, oldpop") + ');')
printsc()
# Admixture pulses.
# Output _T[...] variable rather than an index.
admixture_pulses = [(f"_T[{ap[0]}]", *ap[1:]) for ap in admixture_pulses]
printsc(' // Admixture pulses, one row for each pulse.')
printsc(
' defineConstant("admixture_pulses", ' +
matrix2str(admixture_pulses, col_comment="time, dest, source, rate") +
');')
printsc()
# Drawn mutations.
printsc(' // Drawn mutations, one row for each mutation.')
printsc(' defineConstant("drawn_mutations", ' + matrix2str(
drawn_mutations,
col_comment="time, mut_type, pop_id, genomic_coordinate, save") + ');')
printsc()
# Fitness callbacks.
printsc(' // Fitness callbacks, one row for each callback.')
printsc(' defineConstant("fitness_callbacks", ' +
matrix2str(fitness_callbacks,
col_comment="start_time, end_time, mut_type, pop_id, "
"selection_coeff, dominance_coeff") + ');')
printsc()
# Allele frequency conditioning
op_types = ", ".join(
f"\"{op}\""
for op in stdpopsim.ext.ConditionOnAlleleFrequency.op_types)
printsc(f' defineConstant("op_types", c({op_types}));')
printsc(' // Allele frequency conditioning, one row for each.')
printsc(' defineConstant("condition_on_allele_frequency", ' +
matrix2str(condition_on_allele_frequency,
col_comment="start_time, end_time, mut_type, pop_id, "
"op, allele_frequency, save") + ');')
printsc()
# Sampling episodes.
sample_counts = collections.Counter([
(sample.population,
round(sample.time * demographic_model.generation_time))
for sample in samples
])
sampling_episodes = []
for (pop, time), count in sample_counts.items():
# SLiM can only sample individuals, which we assume are diploid.
n_inds = (count + 1) // 2
if count % 2 != 0:
pop_id = pop_names[pop]
gen = time / demographic_model.generation_time
warnings.warn(
stdpopsim.SLiMOddSampleWarning(
f"SLiM simulates diploid individuals, so {n_inds} "
f"individuals will be sampled for the {count} haploids "
f"requested from population {pop_id} at time {gen}. "
"See #464."))
sampling_episodes.append((pop, n_inds, time))
printsc(' // One row for each sampling episode.')
printsc(' defineConstant("sampling_episodes", ' +
matrix2str(sampling_episodes, col_comment='pop, n_inds, time') +
');')
printsc(_slim_lower)
return epochs[0]
def paragraph(self, text):
if self.previous:
self.file.write("\n")
self.file.write(textwrap.fill(text, self.width))
self.file.write("\n")
self.previous = True
def add_node(lbl, G):
nid = str(uuid.uuid4())
G.add_node(nid)
n = G.get_node(nid)
n.attr['label'] = textwrap.fill(escape_str(lbl), width=20)
return nid
exit()
else:
while (float(rw) < 1):
print(Fore.RED + Style.BRIGHT +
'Invalid entry. Let\'s do that room again.')
oneRoom(floortotal, htotal)
#print('rw = ' + str(rw) + ' in')
rw = numParse(rw)
rtotal = int(rl) * int(rw)
#print ('rtotal is %g sq in' %rtotal)
print('\t\t\tRoom total: ' + inchesToFeet(rtotal))
floortotal += rtotal
oneRoom(floortotal, htotal)
print('\nJUMPVISUAL SQ FOOTAGE CALCULATOR v2.1')
print('~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~')
#print(' created by john gallino')
print('\n' + textwrap.fill(quotes.quotes[qnum], 65) + '\n')
#choice = 'x'
#choice = input('Are you including inches in your measurements? (y/n): ')
#
#
#while (choice.lower() != 'y' and choice.lower() != 'n'):
# choice = int(input('\nPlease enter y or n :'))
print(
'\nINSTRUCTIONS\n1. Starting with the first floor, enter the length and width of each room.\n2. If you are including inches, separate with a space. (e.g. \'8 3\' for 8ft 3in)\n3. Decimals work too. (e.g. \'10.5\' for 10ft 6in)\n4. When finished with a floor, enter \'-1\'\n'
)
oneRoom(floortotal, htotal)
def test_candidateLogUpToDate(self):
"""
Is the candidate up to date? The term and the index together
have to be at least as advanced as that of the current machine.
"""
currentTerm = 100
log = []
for x in xrange(10):
log.append(rpc_objects.LogEntry(term=currentTerm, command=x))
results = self.state.persister.setCurrentTerm(currentTerm)
results.addCallback(
dropResult(self.state.persister.matchAndAppendNewLogEntries, -1,
log))
# Index and term are equal
results.addCallback(
dropResult(self.state.candidateLogUpToDate,
lastLogIndex=len(log) - 1,
lastLogTerm=currentTerm))
indexAndTermOKFailureMessage = fill(
'''candidateLogUpToDate should return True when lastLogIndex and
lastLogTerm are equal to the state's last log index and the
term of the last log entry.''')
results.addCallback(self.assertTrue, msg=indexAndTermOKFailureMessage)
results.addCallback(
dropResult(self.state.candidateLogUpToDate,
lastLogIndex=len(log),
lastLogTerm=currentTerm))
indexIsHigherFailureMessage = fill(
'''candidateLogUpToDate should return True when lastLogIndex is
greater than state's last log index and the lastLogTerm is
equal to the term of the last log entry.''')
results.addCallback(self.assertTrue, msg=indexIsHigherFailureMessage)
results.addCallback(
dropResult(self.state.candidateLogUpToDate,
lastLogIndex=len(log) - 1,
lastLogTerm=currentTerm + 1))
termIsHigherFailureMessage = fill(
'''candidateLogUpToDate should return True when lastLogIndex is
equal to the state's last log index and the lastLogTerm is
greater than the term of the last log entry.''')
results.addCallback(self.assertTrue, msg=termIsHigherFailureMessage)
results.addCallback(
dropResult(self.state.candidateLogUpToDate,
lastLogIndex=len(log) - 2,
lastLogTerm=currentTerm))
indexIsLowerFailureMessage = fill(
'''candidateLogUpToDate should return False when lastLogIndex is
less than the state's last log index''')
results.addCallback(self.assertFalse, msg=indexIsLowerFailureMessage)
# # Term is lower
results.addCallback(
dropResult(self.state.candidateLogUpToDate,
lastLogIndex=len(log),
lastLogTerm=currentTerm - 1))
termIsLowerFailureMessage = fill(
'''candidateLogUpToDate should return False when lastLogTerm is
less than the term of the last log entry.''')
results.addCallback(self.assertFalse, msg=termIsLowerFailureMessage)
return results
def wrap_text(
text,
indent,
width=79,
initial_indent=4,
multi=False,
preserve_line_breaks=True,
quote=True,
):
text = text.strip()
if not multi:
if not quote:
quote = ""
elif len(text
) + initial_indent + 8 > width or "\n" in text or '"' in text:
quote = '"""'
else:
quote = '"'
if text[0] == '"':
text = "{} {}".format(quote, text)
else:
text = "{}{}".format(quote, text)
if text[-1] == '"':
text = "{} {}".format(text, quote)
else:
text = "{}{}".format(text, quote)
else:
width -= 3
wrapped_lines = []
first = True
for l in text.splitlines():
if first:
w = width - initial_indent
first = False
else:
if multi:
w = width - indent
else:
w = width
lines = textwrap.fill(
" ".join(l.strip().split()),
width=w,
subsequent_indent=" " * indent,
replace_whitespace=False,
).splitlines()
wrapped_lines += [i.strip() for i in lines]
text = ("\n" + " " * indent).join(wrapped_lines)
if not multi:
return text
else:
lines = [
'"{}"'.format(i.strip().replace('"', "'"))
for i in text.splitlines()
]
lines2 = []
c = 0
for l in lines:
if preserve_line_breaks:
if c != 0:
l = '"\\n{}'.format(l[1:100])
elif c != len(lines) - 1:
l = '%s "' % (l[0:-1])
c += 1
lines2.append(l)
text = ("\n{}".format(" " * indent)).join(lines2)
if "\n" in text:
text = "{}".format(text)
return text
def comment(s): print(textwrap.indent(textwrap.fill(s, width=100, initial_indent='', subsequent_indent=' '), ' // '), file=typings)
def addfield(name, tpe='any'): add(f'{name}: {types.pop(name, tpe)}')
def output(text1,
col1=None,
text2=None,
col2=None,
wrap=True,
beg=None,
end='\n',
sep=' '):
print('', end=beg)
ptoolkit = use_ptoolkit() and ptcolors['displaymethod'] == "prompt-toolkit"
if not ptoolkit:
col1 = colors[col1] if col1 and colors[col1] and colors[col1][
0].isdigit() else None
col2 = colors[col2] if col2 and colors[col2] and colors[col2][
0].isdigit() else None
clb1 = "\x1B[{}m".format(col1) if col1 else ""
clb2 = "\x1B[{}m".format(col2) if col2 else ""
cle1 = "\x1B[0m" if col1 else ""
cle2 = "\x1B[0m" if col2 else ""
text1 = clb1 + text1 + cle1
if text2 is not None:
text2 = clb2 + text2 + cle2
if wrap:
width = settings.getint("text-wrap-width")
width = 999999999 if width < 2 else width
width = min(width, termWidth)
text1 = textwrap.fill(text1,
width,
replace_whitespace=False,
drop_whitespace=False)
if text2:
if len(text1) + 1 + len(text2) >= width:
if not re.match("^\n+$", sep):
sep += '\n'
text2 = textwrap.fill(text2,
width,
replace_whitespace=False,
drop_whitespace=False)
if ptoolkit:
col1 = ptcolors[col1] if col1 and ptcolors[col1] else ""
col2 = ptcolors[col2] if col2 and ptcolors[col2] else ""
print_formatted_text(to_formatted_text(text1, col1), end='')
if text2:
print_formatted_text(to_formatted_text(sep), end='')
print_formatted_text(to_formatted_text(text2, col2), end='')
print('', end=end)
else:
if not text2:
print(text1, end=end)
else:
print(text1, end='')
print(sep, end='')
print(text2, end=end)
linecount = 1
if text1:
linecount += text1.count('\n')
if end:
linecount += end.count('\n')
if sep:
linecount += sep.count('\n')
if text2:
linecount += text2.count('\n')
return linecount
def main(language: str) -> None:
"""The hypermodern Python project."""
page = wikipedia.random_page(language=language)
click.secho(page.title, fg="green")
click.echo(textwrap.fill(page.extract))
import textwrap
from textwrap_example import sample_text
def should_indent(line):
print('Indent {!r}?'.format(line))
return len(line.strip()) % 2 == 0
dedented_text = textwrap.dedent(sample_text)
wrapped = textwrap.fill(dedented_text, width=50)
final = textwrap.indent(wrapped, 'EVEN ',
predicate=should_indent)
print('\nQuoted block:\n')
print(final)
def main(argv=None):
""" Scan a binary file for certain open source libraries that may have CVEs """
if argv is None:
argv = sys.argv
# Reset logger level to info
LOGGER.setLevel(logging.INFO)
parser = argparse.ArgumentParser(
prog="cve-bin-tool",
description=textwrap.dedent("""
The CVE Binary Tool scans for a number of common, vulnerable open source
components (openssl, libpng, libxml2, expat and a few others) to let you know
if a given directory or binary file includes common libraries with known
vulnerabilities.
"""),
epilog=textwrap.fill(
f'Available checkers: {", ".join(Scanner.available_checkers())}') +
"\n\nPlease disclose issues responsibly!",
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument("directory", help="directory to scan")
output_group = parser.add_argument_group("Output")
output_group.add_argument("-q",
"--quiet",
action="store_true",
help="suppress output")
output_group.add_argument(
"-l",
"--log",
help="log level (default: info)",
dest="log_level",
action=LogAction,
choices=["debug", "info", "warning", "error", "critical"],
)
output_group.add_argument(
"-o",
"--output-file",
action="store",
default=sys.stdout,
help="provide output filename (default: output to stdout)",
)
output_group.add_argument(
"-f",
"--format",
action="store",
choices=["csv", "json", "console", "html"],
default="console",
help="update output format (default: console)",
)
parser.add_argument(
"-V",
"--version",
action="version",
version=VERSION,
)
parser.add_argument(
"-u",
"--update",
action="store",
choices=["now", "daily", "never", "latest"],
default="daily",
help="update schedule for NVD database (default: daily)",
)
parser.add_argument("-m",
"--multithread",
action="store_true",
help="enable multithread")
parser.add_argument("-x",
"--extract",
action="store_true",
help="autoextract compressed files")
checker_group = parser.add_argument_group("Checkers")
checker_group.add_argument(
"-s",
"--skips",
dest="skips",
action="store",
type=str,
help="comma-separated list of checkers to disable",
)
checker_group.add_argument(
"-r",
"--runs",
dest="checkers",
action="store",
type=str,
help="comma-separated list of checkers to enable",
)
if len(argv) <= 1:
parser.print_help()
return 0
try:
args = parser.parse_args(argv[1:])
except SystemExit:
# override default argparse exit(2) behaviour so positive numbers can indicate
sys.exit(-2)
if args.log_level:
LOGGER.setLevel(args.log_level)
if args.quiet:
LOGGER.setLevel(logging.CRITICAL)
if platform.system() != "Linux":
warning_nolinux = """
**********************************************
Warning: this utility was developed for Linux.
You may need to install additional utilities
to use it on other operating systems.
**********************************************
"""
LOGGER.warning(warning_nolinux)
if not os.path.isfile(args.directory) and not os.path.isdir(
args.directory):
LOGGER.error("Error: directory/file invalid")
parser.print_usage()
return -1
exclude_folders = [".git"]
walker = DirWalk(
# pattern='*.*',
# folder_include_pattern='*',
folder_exclude_pattern=";".join(exclude_folders),
# file_exclude_pattern=';'.join(exclude_files)
).walk
# Connect to the database
cvedb_orig = CVEDB()
# if OLD_CACHE_DIR (from cvedb.py) exists, print warning
if os.path.exists(OLD_CACHE_DIR):
LOGGER.warning(
f"Obsolete cache dir {OLD_CACHE_DIR} is no longer needed and can be removed."
)
# Clear data if -u now is set
if args.update == "now":
cvedb_orig.clear_cached_data()
if args.update == "latest":
cvedb_orig.refresh_cache_and_update_db()
# update db if needed
if args.update != "never":
cvedb_orig.get_cvelist_if_stale()
skips = ""
if args.skips:
skips = args.skips
if args.checkers:
checkers = args.checkers.split(",")
skips = ",".join(
map(
lambda checker: checker.name,
filter(
lambda checker: checker.name not in checkers,
pkg_resources.iter_entry_points("cve_bin_tool.checker"),
),
))
# Single-thread mode
if not args.multithread:
# Close database when done
cvedb = CVEDB()
cvedb.open()
with cvedb:
extractor = Extractor()
scanner = Scanner(cvedb)
scanner.remove_skiplist(skips)
LOGGER.info(scanner.print_checkers())
with extractor() as ectx:
if os.path.isdir(args.directory):
for filepath in walker([args.directory]):
scan_and_or_extract_file(scanner, ectx, walker,
args.extract, filepath)
elif os.path.isfile(args.directory):
scan_and_or_extract_file(scanner, ectx, walker,
args.extract, args.directory)
LOGGER.info("")
LOGGER.info("Overall CVE summary: ")
LOGGER.info(
f"There are {scanner.files_with_cve} files with known CVEs detected"
)
if scanner.files_with_cve > 0:
affected_string = ", ".join(
map(
lambda module_version: "".join(str(module_version)),
scanner.affected(),
))
LOGGER.info(f"Known CVEs in {affected_string}:")
# Creates a Object for OutputEngine
output = OutputEngine(modules=scanner.all_cves,
filename=args.output_file)
if not args.quiet or args.output_file != sys.stdout:
output.output_file(args.format)
# Use the number of files with known cves as error code
# as requested by folk planning to automate use of this script.
# If no files found, then the program exits cleanly.
return scanner.files_with_cve
# Enable multithread
else:
def worker():
cvedb = CVEDB()
cvedb.open()
with cvedb:
scanner = Scanner(cvedb)
scanner.remove_skiplist(skips)
while True:
scan_target = q.get()
if not scan_target:
q.task_done()
break
scanner.scan_file(scan_target)
q.task_done()
cves.put(scanner.files_with_cve)
# using queue
q = queue.Queue()
cves = queue.Queue()
# Extract all files first, save the path to a list
extractor = Extractor()
file_list = []
with extractor() as ectx:
if os.path.isdir(args.directory):
for filepath in walker([args.directory]):
extract_file(ectx, walker, args.extract, filepath,
file_list)
elif os.path.isfile(args.directory):
extract_file(ectx, walker, args.extract, args.directory,
file_list)
binary_list = []
pool = multiprocessing.Pool(multiprocessing.cpu_count() * 4)
try:
for i, file_is_binary in enumerate(
pool.map(is_binary, file_list)):
if file_is_binary:
binary_list.append(file_list[i])
# Get all of the binary files
scanning_list = binary_list if binary_list else file_list
if len(scanning_list) == 1:
return scan_files(scanning_list[0], vars(args))
# create threads
threads = []
for i in range(multiprocessing.cpu_count() * 4):
t = threading.Thread(target=worker)
t.start()
threads.append(t)
for scanning_target in scanning_list:
q.put(scanning_target)
for i in range(multiprocessing.cpu_count() * 4):
q.put(None)
# wait until all works done
q.join()
for t in threads:
t.join()
cve_list = []
while not cves.empty():
cve_list.append(cves.get())
return cve_list
finally:
pool.terminate()
pool.join()
