def __str__(self):
if self.left or self.right:
return '{}({},\n{},\n{}\n)'.format(type(self).__name__,
self.data,
textwrap.indent(str(self.left), ' '*4),
textwrap.indent(str(self.right), ' '*4))
return repr(self)
def print_scrape_results_http(results):
"""Print the results obtained by "http" method.
Args:
results: The results to be printed to stdout.
"""
for t in results:
for result in t:
logger.info('{} links found. The search with the keyword "{}" yielded the result: "{}"'.format(
len(result['results']), result['search_keyword'], result['num_results_for_kw']))
import textwrap
for result_set in ('results', 'ads_main', 'ads_aside'):
if result_set in result.keys():
print('### {} link results for "{}" ###'.format(len(result[result_set]), result_set))
for link_title, link_snippet, link_url, link_position in result[result_set]:
try:
print(' Link: {}'.format(unquote(link_url.geturl())))
except AttributeError as ae:
print(ae)
if Config['GLOBAL'].getint('verbosity') > 1:
print(
' Title: \n{}'.format(textwrap.indent('\n'.join(textwrap.wrap(link_title, 50)), '\t')))
print(
' Description: \n{}\n'.format(
textwrap.indent('\n'.join(textwrap.wrap(link_snippet, 70)), '\t')))
print('*' * 70)
print()
def _format_loop_exception(self, context, n):
message = context.get('message', 'Unhandled exception in event loop')
exception = context.get('exception')
if exception is not None:
exc_info = (type(exception), exception, exception.__traceback__)
else:
exc_info = None
lines = []
for key in sorted(context):
if key in {'message', 'exception'}:
continue
value = context[key]
if key == 'source_traceback':
tb = ''.join(traceback.format_list(value))
value = 'Object created at (most recent call last):\n'
value += tb.rstrip()
else:
try:
value = repr(value)
except Exception as ex:
value = ('Exception in __repr__ {!r}; '
'value type: {!r}'.format(ex, type(value)))
lines.append('[{}]: {}\n\n'.format(key, value))
if exc_info is not None:
lines.append('[exception]:\n')
formatted_exc = textwrap.indent(
''.join(traceback.format_exception(*exc_info)), ' ')
lines.append(formatted_exc)
details = textwrap.indent(''.join(lines), ' ')
return '{:02d}. {}:\n{}\n'.format(n, message, details)
def training_desc(info, verbosity, precision):
"""
Textifies a single training history record. Returns a list of lines.
"""
desc = [
"Date: {}".format(date_desc(info['date'])),
"Bot: {bot}".format(**info) +
", Trainer: {trainer} {config}".format(**info),
"Dists: {}".format(indent(dists_desc(info['dists']), " " * 7).strip()),
"Seeds: {}".format(seeds_desc(info['seeds'], verbosity)),
"Level: {}".format(level_desc(info['level'])) +
", Runs: {}".format(info['runs']) +
", Time: {}".format(time_desc(info['time'], precision)),
"Output: {output}, PRNGs: {prngs_seed}".format(**info) +
", Scores: {}".format(scores_desc(info['scores'], verbosity,
precision))
]
if info['phases'] is not None:
desc.insert(3, "Phases: {}".format(
phases_desc(info['phases'], precision)))
if info['emphases']:
desc.insert(3, "Emphases: {}".format(emphases_desc(info['emphases'])))
if info['param_scale']:
desc.insert(5, "Scaled params: {}".format(
param_scale_desc(info['param_scale'])))
if info['param_freeze']:
desc.insert(5, "Frozen params: {}".format(
", ".join(info['param_freeze'])))
if info['param_map']:
desc.insert(5, "Params map: {}".format(
indent(param_map_desc(info['param_map']), " " * 12).strip()))
return desc
def write_options_group(self, write, group):
def is_positional_group(group):
return any(not o.option_strings for o in group._group_actions)
if is_positional_group(group):
for option in group._group_actions:
write(option.metavar)
write(textwrap.indent(option.help or '', ' ' * 4))
return
opts = OrderedDict()
for option in group._group_actions:
if option.metavar:
option_fmt = '%s ' + option.metavar
else:
option_fmt = '%s'
option_str = ', '.join(option_fmt % s for s in option.option_strings)
option_desc = textwrap.dedent((option.help or '') % option.__dict__)
opts[option_str] = textwrap.indent(option_desc, ' ' * 4)
padding = len(max(opts)) + 1
for option, desc in opts.items():
write(option.ljust(padding), desc)
def write_options_group(self, group, fp, with_title=True, base_indent=4):
def is_positional_group(group):
return any(not o.option_strings for o in group._group_actions)
indent = ' ' * base_indent
if is_positional_group(group):
for option in group._group_actions:
fp.write(option.metavar + '\n')
fp.write(textwrap.indent(option.help or '', ' ' * base_indent) + '\n')
return
if not group._group_actions:
return
if with_title:
fp.write('\n\n')
fp.write(group.title + '\n')
opts = OrderedDict()
for option in group._group_actions:
if option.metavar:
option_fmt = '%s ' + option.metavar
else:
option_fmt = '%s'
option_str = ', '.join(option_fmt % s for s in option.option_strings)
option_desc = textwrap.dedent((option.help or '') % option.__dict__)
opts[option_str] = textwrap.indent(option_desc, ' ' * 4)
padding = len(max(opts)) + 1
for option, desc in opts.items():
fp.write(indent + option.ljust(padding) + desc + '\n')
def play(self):
selection = None
current_sleep = self.start
current_round = 0
while True:
selection = choice_unduplicated(available_notes, selection)
octave, note = selection.split('.')
flute = getFlute(notes[selection])
figlet_note = figlet('{} {}'.format(octave, note))
header = ' Interval: {:.2f} | Sleep: {:.2f} | Round {:d}/{:d}'.format(
self.step, current_sleep, current_round, self.rounds)
flute_padded = indent(flute, ' ' * 14)
figlet_width = len(max(figlet_note.splitlines(), key=len))
note_padded = indent(figlet_note, ' ' * (17 - int(figlet_width/2)))
system('clear')
print(header, "\n\n", note_padded, flute_padded, sep='')
sleep(current_sleep)
current_round += 1
if (current_round >= self.rounds):
current_round = 0
current_sleep -= self.step
if (current_sleep <= self.stop):
break
def check():
name = f.__name__
message = ""
if f.__doc__:
message = "\n" + textwrap.indent(f.__doc__, ' """ ')
try:
f()
return True
except AssertionError as e:
if e.args:
message = e.args[0].strip()
exception_class, exception, trace = sys.exc_info()
frames = traceback.extract_tb(trace)
last = frames[len(frames) - 1]
message_hr = textwrap.indent(message, " ")
assertion = "{3}".format(*last)
position = "{0}:{1}".format(*last)
report("{} ({}):".format(name, position))
if message_hr:
report(" --------------------------------")
report("{}".format(message_hr))
report(" --------------------------------")
report(" {}".format(assertion))
report("")
return False
except Exception as e:
report_exc("{}:{}".format(name, message), traceback.format_exc())
return False
def print_scrape_results_http(results, verbosity=1, view=False):
"""Print the results obtained by "http" method."""
for t in results:
for result in t:
logger.info('{} links found! The search with the keyword "{}" yielded the result:{}'.format(
len(result['results']), result['search_keyword'], result['num_results_for_kw']))
if view:
import webbrowser
webbrowser.open(result['cache_file'])
import textwrap
for result_set in ('results', 'ads_main', 'ads_aside'):
if result_set in result.keys():
print('### {} link results for "{}" ###'.format(len(result[result_set]), result_set))
for link_title, link_snippet, link_url, link_position in result[result_set]:
try:
print(' Link: {}'.format(urllib.parse.unquote(link_url.geturl())))
except AttributeError as ae:
print(ae)
if verbosity > 1:
print(
' Title: \n{}'.format(textwrap.indent('\n'.join(textwrap.wrap(link_title, 50)), '\t')))
print(
' Description: \n{}\n'.format(
textwrap.indent('\n'.join(textwrap.wrap(link_snippet, 70)), '\t')))
print('*' * 70)
print()
def galaxy(_galaxy):
systems = []
debug('showing the galaxy...')
for c, s in _galaxy._systems.items():
sys = indent(system(s), ' ')[2:]
systems.append('{} {}'.format(c, sys))
return 'Galaxy:\n{}'.format(indent('\n'.join(systems), ' '))
def check_negative(dirname, results, expected):
"""Checks the results with the expected results."""
inputs = (set(results['inputs']), set(expected['!inputs']))
outputs = (set(results['outputs']), set(expected['!outputs']))
if not inputs[1].isdisjoint(inputs[0]):
print(bcolors.ERROR + ': Found inputs that should not exist:')
s = pprint.pformat(inputs[1] & inputs[1], width=1)
print(textwrap.indent(s, ' '))
return False
if not outputs[1].isdisjoint(outputs[0]):
print(bcolors.ERROR + ': Found outputs that should not exist:')
s = pprint.pformat(outputs[1] & outputs[1], width=1)
print(textwrap.indent(s, ' '))
return False
existing = [p for p in (os.path.join(dirname, p) for p in outputs[1])
if os.path.exists(p)]
if existing:
print(bcolors.ERROR + ': Outputs exist on from file system, but should not:')
pprint.pprint(existing)
return False
return True
def check_positive(dirname, results, expected):
"""Checks the results with the expected results."""
inputs = (set(results['inputs']), set(expected['inputs']))
outputs = (set(results['outputs']), set(expected['outputs']))
if not inputs[1].issubset(inputs[0]):
print(bcolors.ERROR + ': Expected inputs are not a subset of the results.')
print(' The following were not found in the results:')
s = pprint.pformat(inputs[1] - inputs[0], width=1)
print(textwrap.indent(s, ' '))
print(' Instead, these were found:')
s = pprint.pformat(inputs[0], width=1)
print(textwrap.indent(s, ' '))
return False
if not outputs[1].issubset(outputs[0]):
print(bcolors.ERROR + ': Expected outputs are not a subset of the results')
print(' The following were not found in the results:')
s = pprint.pformat(outputs[1] - outputs[0], width=1)
print(textwrap.indent(s, ' '))
print(' Instead, these were found:')
s = pprint.pformat(outputs[0], width=1)
print(textwrap.indent(s, ' '))
return False
missing = [p for p in (os.path.join(dirname, p) for p in outputs[0])
if not os.path.exists(p)]
if missing:
print(bcolors.ERROR + ': Result outputs missing from file system:')
pprint.pprint(missing)
return False
return True
def pretty_print(self,m=True,f=False):
#Print the files and folders under root, with their memory usage if m(with_memory)=True
#Only print folders is f(folders_only)=True
base=3
width=self.depth*base+50
DFS=[self.root]
while DFS:
cur_node=DFS.pop()
indent_size=cur_node.dist*base
name=os.path.basename(cur_node.name)
star_size=width-indent_size-len(name)
try:
if m:
print(textwrap.indent(name,''.join([' ']*indent_size)),''.join(['.']*star_size),cur_node.memsize)
else:
print(textwrap.indent(name,''.join([' ']*indent_size)))
except UnicodeEncodeError as err:
logging.debug(err)
for k in cur_node.kiddir:
DFS.append(k)
if not f:
for k in cur_node.kidfile:
indent_size=k.dist*base
name=os.path.basename(k.name)
star_size=width-indent_size-len(name)
try:
if m:
print(textwrap.indent(name,''.join([' ']*indent_size)),''.join(['.']*star_size),k.memsize)
else:
print(textwrap.indent(name,''.join([' ']*indent_size)))
except UnicodeEncodeError as err:
print ('Handling unicode encode error',err)
def __str__(self):
o = self.fmt.format(**self.__dict__)
for k, l in self.lines.items():
o += textwrap.indent(str(l), prefix=k + ' ')
for txt in self.texts:
o += 'with string\n'
o += textwrap.indent(str(txt), prefix=' ')
return o
def build_cpp_api_main(outputdir, rst_header, components):
"""Parse existing rst files (one for each class,
+ those for functions) generated for C++ API
and collect them into cpp_api.rst
in sphinx/reference directory.
Parameters
----------
outputdir : Path()
sphinx directory which contains rst files
generated for the api (e.g. by doxy2swig)
rst_header : string
text to put on top of the python_api file.
"""
mainrst_filename = Path(outputdir, 'cpp_api.rst')
# list documented (cpp) packages
docpp_dir = Path(outputdir, 'cpp')
packages = [f for f in docpp_dir.glob('*')]
packages = [p.name for p in packages if os.listdir(p)]
# trick to print components in the expected order.
packages = [p for p in components if p in packages]
indent = 4 * ' '
class_diag = 'Class diagrams (UML view)'
class_diag += '\n' + len(class_diag) * '=' + '\n\n'
class_diag += ':doc:`/reference/class_diagrams`\n\n'
with open(mainrst_filename, 'w') as f:
label = '.. _siconos_cpp_reference:\n\n\n'
title = 'Siconos C/C++ API reference'
title += '\n' + len(title) * '#' + '\n\n'
title += 'This is the documentation of C/C++ interface to Siconos.\n\n\n'
f.write(label)
f.write(title)
f.write(rst_header)
tab_directive = '.. csv-table::\n'
tab_directive += textwrap.indent(':widths: 60 40\n\n', indent)
column_titles = '**Classes and structs**, **Files**\n'
tab_directive += textwrap.indent(column_titles, indent)
f.write(class_diag)
for p in packages:
title = p.title() + ' component\n'
title += len(title) * '=' + '\n\n'
ppath = 'cpp/' + p
f.write(title)
pgm_listings = 'Check :ref:`' + p + '_pgm_listings`'
pgm_listings += ' for a complete list of headers for this component.'
f.write(pgm_listings + '\n\n')
#f.write(tab_directive)
directive = '.. include:: ' + ppath + '/autodoc_classes.rst'
directive += '\n'#','
directive += '.. include:: ' + ppath + '/autodoc_files.rst\n'
indent = ''
f.write(textwrap.indent(directive, indent))
f.write('\n')
def status(self, workspacePath):
status = ScmStatus()
try:
onCorrectBranch = False
output = self.callGit(workspacePath, 'ls-remote' ,'--get-url')
if output != self.__url:
status.add(ScmTaint.switched,
"> URL: configured: '{}', actual: '{}'".format(self.__url, output))
if self.__commit:
output = self.callGit(workspacePath, 'rev-parse', 'HEAD')
if output != self.__commit:
status.add(ScmTaint.switched,
"> commit: configured: '{}', actual: '{}'".format(self.__commit, output))
elif self.__tag:
output = self.callGit(workspacePath, 'tag', '--points-at', 'HEAD').splitlines()
if self.__tag not in output:
actual = ("'" + ", ".join(output) + "'") if output else "not on any tag"
status.add(ScmTaint.switched,
"> tag: configured: '{}', actual: '{}'".format(self.__tag, actual))
elif self.__branch:
output = self.callGit(workspacePath, 'rev-parse', '--abbrev-ref', 'HEAD')
if output != self.__branch:
status.add(ScmTaint.switched,
"> branch: configured: '{}', actual: '{}'".format(self.__branch, output))
else:
output = self.callGit(workspacePath, 'log', '--oneline',
'refs/remotes/origin/'+self.__branch+'..HEAD')
if output:
status.add(ScmTaint.unpushed_main,
joinLines("> unpushed commits on {}:".format(self.__branch),
indent(output, ' ')))
onCorrectBranch = True
# Check for modifications wrt. checked out commit
output = self.callGit(workspacePath, 'status', '--porcelain')
if output:
status.add(ScmTaint.modified, joinLines("> modified:",
indent(output, ' ')))
# The following shows all unpushed commits reachable by any ref
# (local branches, stash, detached HEAD, etc).
# Exclude HEAD if the configured branch is checked out to not
# double-count them. Does not mark the SCM as dirty.
what = ['--all', '--not', '--remotes']
if onCorrectBranch: what.append('HEAD')
output = self.callGit(workspacePath, 'log', '--oneline', '--decorate',
*what)
if output:
status.add(ScmTaint.unpushed_local,
joinLines("> unpushed local commits:", indent(output, ' ')))
except BuildError as e:
status.add(ScmTaint.error, e.slogan)
return status
def slope_a(a, cell_size=1, kern=None, degrees=True, verb=False, keep=False):
"""Return slope in degrees for an input array using 3rd order
finite difference method for a 3x3 moing window view into the array.
Requires:
---------
- a : an input 2d array. X and Y represent coordinates of the Z values
- cell_size : cell size, must be in the same units as X and Y
- kern : kernel to use
- degrees : True, returns degrees otherwise radians
- verb : True, to print results
- keep : False, to remove/squeeze extra dimensions
- filter :
np.array([[1, 2, 1], [2, 0, 2], [1, 2, 1]]) **current default
Notes:
------
::
dzdx: sum(col2 - col0)/8*cellsize
dzdy: sum(row2 - row0)/8*celsize
Assert the array is ndim=4 even if (1,z,y,x)
general dzdx + dzdy = dxyz
[[a, b, c], [[1, 0, 1], [[1, 2, 1] [[1, 2, 1]
[d, e, f] [2, 0, 2], + [0, 0, 0] = [2, 0, 2],
[g, h, i] [1, 0, 1]] [1, 2, 1]] [1, 2, 1]]
"""
frmt = """\n :----------------------------------------:
:{}\n :input array...\n {}\n :slope values...\n {!r:}
:----------------------------------------:
"""
# ---- stride the data and calculate slope for 3x3 sliding windows ----
np.set_printoptions(edgeitems=10, linewidth=100, precision=1)
a_s = stride(a, win=(3, 3), stepby=(1, 1))
if a_s.ndim < 4:
new_shape = (1,) * (4-len(a_s.shape)) + a_s.shape
a_s = a_s.reshape(new_shape)
#
kern = kernels(kern) # return the kernel if specified
# ---- default filter, apply the filter to the array ----
#
dz_dx, dz_dy = filter_a(a_s, a_filter=kern, cell_size=cell_size)
#
s = np.sqrt(dz_dx**2 + dz_dy**2)
if degrees:
s = np.rad2deg(np.arctan(s))
if not keep:
s = np.squeeze(s)
if verb:
p = " "
args = ["Results for slope_a... ",
indent(str(a), p), indent(str(s), p)]
print(dedent(frmt).format(*args))
return s
def __init__(self, module, objs, pckg_tree=None):
self.name = module.split('.')
if pckg_tree:
pckg_tree.walk(self.name[1:], self)
self._extractors = {}
self._var_names = set()
code = StringIO()
for obj in objs:
if isinstance(obj, RustFuncGen.Func):
self._extractors[obj.name] = extractors = []
params, pyconstructors, argnames = [], [], []
pn = [x[0] for x in obj.parameters]
for x, param in enumerate(obj.parameters):
argnames.append(param[0] + ', ')
pyarg_constructor = self._fn_arg.format(param[0])
p, _ = self.unwrap_types(param[1], "", "")
if p[0] != '(':
p = p[1:-1]
sig = param[0] + ': ' + p
if x + 1 != len(obj.parameters):
sig += ",\n"
pyarg_constructor += "\n"
params.append(indent(sig, tab * 3))
pyconstructors.append(indent(pyarg_constructor, tab * 3))
signature, return_val = self.unwrap_types(
obj.rsreturn, "", "",
extractors=extractors,
arg_names=pn,
init=True,
parent="result")
if signature[0] == '(':
return_type = "-> " + signature
elif len(obj.rsreturn) == 1 \
and obj.rsreturn[0] == 'PyObject::None':
return_type, return_val = "", ""
else:
return_type = "-> " + signature[1:-1]
if len(argnames) > 0:
args = "(" + "".join(argnames) + "),"
else:
args = "NoArgs,"
body = self._fn_body.substitute(
name=obj.name,
convert="".join(pyconstructors),
args=args,
extract_values="".join(extractors),
return_value=return_val,
)
code.write(self._function.substitute(
name=obj.name,
parameters="".join(params),
return_type=return_type,
body=body,
))
self._generated_functions = code
def fmt_field(key, value="", level=0):
if "\n" in value:
value = indent(dedent(value), get_setting("indent")).strip("\n")
sep = "\n"
elif value:
sep = " "
else:
sep = ""
key = str(key).replace("_", " ")
leader = level * get_setting("indent")
return indent(LabelColor(key + ":") + sep + value, leader)
def __call__(self, content=None, wd=None, sudo=False, sudoPwd=None):
if self.path is not None:
try:
with open(self.path, 'r') as input_:
self._content = input_.read()
except FileNotFoundError:
pass
content = self._content if content is None else content
if isinstance(self.seek, str):
rObj = re.compile(self.seek, re.DOTALL | re.MULTILINE)
match = rObj.search(content)
if match:
tgt = match.string[match.start():match.end()]
msg = log_message(
logging.DEBUG,
msg="Pattern {} matched {}".format(
rObj.pattern, tgt),
name=self._name)
self._rv = "\n".join(
[rObj.sub(textwrap.indent(self.data, ' ' * self.indent),
content, count=0)] +
[""] * self.newlines
)
else:
msg = log_message(
logging.WARNING,
msg="Pattern {} unmatched.".format(
rObj.pattern),
name=self._name)
self._rv = ""
yield msg
elif self.seek:
args = (
[content] +
[textwrap.indent(self.data, ' ' * self.indent)] +
[""] * self.newlines
)
self._rv = "\n".join(args)
else:
args = (
[textwrap.indent(self.data, ' ' * self.indent)] +
[""] * self.newlines + [content]
)
self._rv = "\n".join(args)
# yield log_message(logging.DEBUG, msg=self._rv, name=self._name)
if self._rv is not None and self.path is not None:
with open(self.path, 'w') as output:
output.write(self._rv)
output.flush()
def __str__(self):
s = ""
s += "frac_train = %s" % self.frac_train
s += "\nrandomized = %s" % self.randomized
s += "\nnum_trainings = %s" % self.num_trainings
s += "\nreplace = %s" % self.replace
s += "\nresults = %s" % self.results
s += "\nmean = %s" % self.mean
textwrap.indent(s, " ")
s = "CrossValidationResults:\n" + s
return s
def visit_If(self, node):
test = self.visit(node.test)
spaces = ' ' * 4
body = textwrap.indent('\n'.join(map(self.visit, node.body)), spaces)
if node.orelse:
orelse = textwrap.indent(
'\n'.join(map(self.visit, node.orelse)),
spaces
)
return 'if {}:\n{}\nelse:\n{}'.format(test, body, orelse)
return 'if {}:\n{}'.format(test, body)
def write_options(self, parser, fp):
def is_positional_group(group):
return any(not o.option_strings for o in group._group_actions)
# HTML output:
# A table using some column-spans
def html_write(s):
for line in s.splitlines():
fp.write(' ' + line + '\n')
rows = []
for group in parser._action_groups:
if group.title == 'Common options':
# (no of columns used, columns, ...)
rows.append((1, '.. class:: borg-common-opt-ref\n\n:ref:`common_options`'))
else:
if not group._group_actions:
continue
group_header = '**%s**' % group.title
if group.description:
group_header += ' — ' + group.description
rows.append((1, group_header))
if is_positional_group(group):
for option in group._group_actions:
rows.append((3, '', '``%s``' % option.metavar, option.help or ''))
else:
for option in group._group_actions:
if option.metavar:
option_fmt = '``%s ' + option.metavar + '``'
else:
option_fmt = '``%s``'
option_str = ', '.join(option_fmt % s for s in option.option_strings)
option_desc = textwrap.dedent((option.help or '') % option.__dict__)
rows.append((3, '', option_str, option_desc))
fp.write('.. only:: html\n\n')
table = io.StringIO()
table.write('.. class:: borg-options-table\n\n')
self.rows_to_table(rows, table.write)
fp.write(textwrap.indent(table.getvalue(), ' ' * 4))
# LaTeX output:
# Regular rST option lists (irregular column widths)
latex_options = io.StringIO()
for group in parser._action_groups:
if group.title == 'Common options':
latex_options.write('\n\n:ref:`common_options`\n')
latex_options.write(' |')
else:
self.write_options_group(group, latex_options)
fp.write('\n.. only:: latex\n\n')
fp.write(textwrap.indent(latex_options.getvalue(), ' ' * 4))
def __repr__(self):
lines = [
'building_id: {}'.format(self.building_id),
'name: {}'.format(self.name),
'container: \n{}'.format(indent(str(self.container), ' '))
]
for (index, resource_plant) in enumerate(self.resource_plants):
lines.append('resource_plant {}:')
lines.append(indent(str(resource_plant), ' '))
for (index, unit_factory) in enumerate(self.unit_factories):
lines.append('unit_factory {}:')
lines.append(indent(str(unit_factory), ' '))
return '\n'.join(lines)
def make_codeblock(s):
"""Make a codeblock that will render nicely in sphinx"""
block = ["AreaDetector Component", "::", ""]
lines = s.split("\n", 1)
header, lines = lines[0], lines[1:]
block.append(textwrap.indent(textwrap.dedent(header), prefix=" " * 4))
lines = "\n".join(lines)
block.append(textwrap.indent(textwrap.dedent(lines), prefix=" " * 4))
block.append("")
return "\n".join(block)
def visit_If(self, node):
ts = self.visit(node.test)
ys = []
for n in node.body:
ys.append(self.visit(n))
yes = indent('\n'.join(ys), ' ')
no = []
for n in node.orelse:
no.append(self.visit(n))
if no:
n = indent('\n'.join(no), ' ')
return 'if (%s) {\n%s\n}\nelse {\n%s\n}\n' % (ts, yes, n)
else:
return 'if (%s) {\n%s\n}\n' % (ts, yes)
def run_test(self, *, source, spec, expected):
ir = compiler.compile_to_ir(source, self.schema)
path_scope = textwrap.indent(ir.scope_tree.pformat(), ' ')
expected_scope = textwrap.indent(
textwrap.dedent(expected).strip(' \n'), ' ')
if path_scope != expected_scope:
diff = '\n'.join(difflib.context_diff(
expected_scope.split('\n'), path_scope.split('\n')))
self.fail(
f'Scope tree does not match the expected result.'
f'\nEXPECTED:\n{expected_scope}\nACTUAL:\n{path_scope}'
f'\nDIFF:\n{diff}')
def check_pickleable(self, test, err):
# Ensure that sys.exc_info() tuples are picklable. This displays a
# clear multiprocessing.pool.RemoteTraceback generated in the child
# process instead of a multiprocessing.pool.MaybeEncodingError, making
# the root cause easier to figure out for users who aren't familiar
# with the multiprocessing module. Since we're in a forked process,
# our best chance to communicate with them is to print to stdout.
try:
pickle.dumps(err)
except Exception as exc:
original_exc_txt = repr(err[1])
original_exc_txt = textwrap.fill(original_exc_txt, 75)
original_exc_txt = textwrap.indent(original_exc_txt, ' ')
pickle_exc_txt = repr(exc)
pickle_exc_txt = textwrap.fill(pickle_exc_txt, 75)
pickle_exc_txt = textwrap.indent(pickle_exc_txt, ' ')
if tblib is None:
print("""
{} failed:
{}
Unfortunately, tracebacks cannot be pickled, making it impossible for the
parallel test runner to handle this exception cleanly.
In order to see the traceback, you should install tblib:
pip install tblib
""".format(test, original_exc_txt))
else:
print("""
{} failed:
{}
Unfortunately, the exception it raised cannot be pickled, making it impossible
for the parallel test runner to handle it cleanly.
Here's the error encountered while trying to pickle the exception:
{}
You should re-run this test without the --parallel option to reproduce the
failure and get a correct traceback.
""".format(test, original_exc_txt, pickle_exc_txt))
raise
def mk_build(targets, rule, inputs, implicit=(), order=(), **kwargs):
targets = targets if isinstance(targets, str) else ' '.join(targets)
inputs = inputs if isinstance(inputs, str) else ' '.join(inputs)
implicit = ' | %s' % ' '.join(implicit) if implicit else ''
order = ' || %s' % ' '.join(order) if order else ''
bindings = '\n' + textwrap.indent(mk_bindings(**kwargs), ' ') if len(kwargs) > 0 else ''
return 'build %s: %s %s%s%s%s' % (targets, rule, inputs, implicit, order, bindings)
def convert_docstring(docstring):
indentation = re.search(r'^\s*', docstring).group()
data = parse_docstring(docstring)
lines = []
lines.append(data['description'])
if len(data['params']) > 0:
lines.append("")
lines.append("Args:")
for param_name, param_dict in data['params'].items():
if len(param_dict['type']) > 0:
lines.append(format_hanging(
"%s (%s): %s" % (
param_name, param_dict['type'], param_dict['doc'])))
else:
lines.append(format_hanging(
"%s: %s" % (param_name, param_dict['doc'])))
if len(data['returns']) > 0 or len(data['rtype']) > 0:
lines.append("")
lines.append("Returns:")
if len(data['rtype']) > 0:
lines.append(format_hanging(
"%s: %s" % (data['rtype'], data['returns'])))
else:
lines.append(format_hanging("%s" % data['returns']))
lines.append(data['epilogue'])
res = (data['delimiters'][0] + "\n".join(lines) + data['delimiters'][1])
while "\n\n\n" in res:
res = res.replace("\n\n\n", "\n\n")
res = textwrap.indent(res, indentation)
return res
def main():
parser = argparse.ArgumentParser(
description=
"Collects all hive-managed resources from a cluster and outputs them as yaml documents"
)
parser.add_argument(
"--path",
"-p",
required=True,
help="Destination file path for artifact [required]",
)
parser.add_argument(
"--output",
"-o",
required=True,
choices=["yaml", "configmap"],
help="Output format of managed resource list [required]",
)
parser.add_argument(
"--all",
action="store_true",
help=
"Output ALL managed resources. By default, it only outputs namespaces.",
)
parser.add_argument(
"--namespace",
"-ns",
default="openshift-monitoring",
help="The namespace for the generated ConfigMap",
)
parser.add_argument(
"--name",
default="managed-namespaces",
help="The name of the generated ConfigMap",
)
arguments = parser.parse_args()
try:
r = subprocess.check_output(["oc", "whoami"],
stderr=subprocess.DEVNULL)
except subprocess.CalledProcessError as e:
sys.exit(
"Must be logged into an OSD cluster to gather list of managed resources"
)
print("Collecting a list of hive-managed resources from cluster...")
if arguments.all:
kinds = get_api_resource_kinds()
else:
kinds = ["namespaces"]
managed_resource_dict = collect_managed_resources(kinds)
managed_resource_yaml = yaml.dump(managed_resource_dict)
with open(arguments.path, "w") as f:
if arguments.output == "yaml":
f.write(managed_resource_yaml)
else:
# indent the yaml document into the configmap template
f.write(
CONFIGMAP_TEMPLATE.format(
arguments.name,
arguments.namespace,
textwrap.indent(managed_resource_yaml, " "),
))
def write_sym_rst(sym, out_dir):
# Writes documentation for 'sym' to <out_dir>/CONFIG_<sym.name>.rst
kconf = sym.kconfig
# List all prompts on separate lines
prompt_str = "\n\n".join("*{}*".format(node.prompt[0])
for node in sym.nodes if node.prompt) \
or "*(No prompt -- not directly user assignable.)*"
# String with the RST for the symbol page
#
# - :orphan: suppresses warnings for the symbol RST files not being
# included in any toctree
#
# - '.. title::' sets the title of the document (e.g. <title>). This seems
# to be poorly documented at the moment.
sym_rst = ":orphan:\n\n" \
".. title:: {0}\n\n" \
".. option:: CONFIG_{0}\n\n" \
"{1}\n\n" \
"Type: ``{2}``\n\n" \
.format(sym.name, prompt_str, kconfiglib.TYPE_TO_STR[sym.type])
# Symbols with multiple definitions can have multiple help texts
for node in sym.nodes:
if node.help is not None:
sym_rst += "Help\n" \
"====\n\n" \
"{}\n\n" \
.format(node.help)
if sym.direct_dep is not kconf.y:
sym_rst += "Direct dependencies\n" \
"===================\n\n" \
"{}\n\n" \
"*(Includes any dependencies from if's and menus.)*\n\n" \
.format(kconfiglib.expr_str(sym.direct_dep))
if sym.defaults:
sym_rst += "Defaults\n" \
"========\n\n"
for value, cond in sym.defaults:
default_str = kconfiglib.expr_str(value)
if cond is not kconf.y:
default_str += " if " + kconfiglib.expr_str(cond)
sym_rst += " - {}\n".format(default_str)
sym_rst += "\n"
def add_select_imply_rst(type_str, expr):
# Writes a link for each selecting symbol (if 'expr' is sym.rev_dep) or
# each implying symbol (if 'expr' is sym.weak_rev_dep). Also adds a
# heading at the top, derived from type_str ("select"/"imply").
nonlocal sym_rst
heading = "Symbols that ``{}`` this symbol".format(type_str)
sym_rst += "{}\n{}\n\n".format(heading, len(heading)*"=")
# The reverse dependencies from each select/imply are ORed together
for select in kconfiglib.split_expr(expr, kconfiglib.OR):
# - 'select/imply A if B' turns into A && B
# - 'select/imply A' just turns into A
#
# In both cases, we can split on AND and pick the first
# operand.
sym_rst += " - :option:`CONFIG_{}`\n".format(
kconfiglib.split_expr(select, kconfiglib.AND)[0].name)
sym_rst += "\n"
if sym.rev_dep is not kconf.n:
add_select_imply_rst("select", sym.rev_dep)
if sym.weak_rev_dep is not kconf.n:
add_select_imply_rst("imply", sym.weak_rev_dep)
def menu_path(node):
path = ""
menu = node.parent
while menu is not kconf.top_node:
# Fancy Unicode arrow. Added in '93, so ought to be pretty
# safe.
path = " → " + menu.prompt[0] + path
menu = menu.parent
return "(top menu)" + path
heading = "Kconfig definition"
if len(sym.nodes) > 1:
heading += "s"
sym_rst += "{}\n{}\n\n".format(heading, len(heading)*"=")
sym_rst += "\n\n".join(
"At ``{}:{}``, in menu ``{}``:\n\n"
".. parsed-literal::\n\n"
"{}".format(node.filename, node.linenr, menu_path(node),
textwrap.indent(str(node), " "*4))
for node in sym.nodes)
sym_rst += "\n\n*(Definitions include propagated dependencies, " \
"including from if's and menus.)*"
write_if_updated(os.path.join(out_dir, "CONFIG_{}.rst".format(sym.name)),
sym_rst)
def wiki_file_writer(elem, myFile, prefix):
global instance_id
t = '\t'
Instance = t + t + "<Instance "
for ch_elem in elem:
if (('id' in ch_elem.tag) and ('parentid' not in ch_elem.tag)):
Instance = Instance + "Id=" + '"' + str(
wikiConverter.instance_id
) + '"' + " InstanceType=" + '"' + "Revision/Wiki" + '"' + " RevisionId=" + '"' + str(
ch_elem.text) + '"' + ">\n"
myFile.write(Instance)
'''
RevisionId = t+t+t+"<RevisionId>"+ch_elem.text+"</RevisionId>\n"
myFile.write(RevisionId)
'''
'''
if(ch_elem.tag==prefix+'parentid'):
ParentId = t+t+t+"<ParentId>"+ch_elem.text+"</ParentId>\n"
myFile.write(ParentId)
'''
'''
Timestamp Information
'''
if ('timestamp' in ch_elem.tag):
'''
if(f_p!=1):
Instance = Instance+" InstanceType= "+'"'+"wiki/text"+'"'+">\n"
myFile.write(Instance)
'''
Timestamp = t + t + t + "<TimeStamp>\n"
myFile.write(Timestamp)
CreationDate = t + t + t + t + "<CreationDate>" + ch_elem.text[:
-1] + '.0' + "</CreationDate>\n"
myFile.write(CreationDate)
Timestamp = t + t + t + "</TimeStamp>\n"
myFile.write(Timestamp)
'''
Contributors information
'''
if ('contributor' in ch_elem.tag):
Contributors = t + t + t + "<Contributors>\n"
myFile.write(Contributors)
for contrib in ch_elem:
if ('ip' in contrib.tag):
LastEditorUserName = t + t + t + t + "<OwnerUserName>" + html.escape(
contrib.text) + "</OwnerUserName>\n"
myFile.write(LastEditorUserName)
else:
if ('username' in contrib.tag):
try:
LastEditorUserName = t + t + t + t + "<OwnerUserName>" + html.escape(
contrib.text) + "</OwnerUserName>\n"
except:
LastEditorUserName = t + t + t + t + "<OwnerUserName>None</OwnerUserName>\n"
myFile.write(LastEditorUserName)
if (('id' in contrib.tag)
and ('parentid' not in contrib.tag)):
LastEditorUserId = t + t + t + t + "<OwnerUserId>" + contrib.text + "</OwnerUserId>\n"
myFile.write(LastEditorUserId)
Contributors = t + t + t + "</Contributors>\n"
myFile.write(Contributors)
'''
Body/Text Information
'''
if ('text' in ch_elem.tag):
Body = t + t + t + "<Body>\n"
myFile.write(Body)
if (ch_elem.attrib.get('bytes') != None):
text_field = t + t + t + t + "<Text Type=" + '"' + "wiki/text" + '"' + " Bytes=" + '"' + ch_elem.attrib[
'bytes'] + '">\n'
elif (ch_elem.text != None):
text_field = t + t + t + t + "<Text Type=" + '"' + "wiki/text" + '"' + " Bytes=" + '"' + str(
len(ch_elem.text)) + '">\n'
else:
text_field = t + t + t + t + "<Text Type=" + '"' + "wiki/text" + '"' + " Bytes=" + '"' + str(
0) + '">\n'
myFile.write(text_field)
if (ch_elem.text == None):
text_body = ""
else:
text_body = textwrap.indent(text=ch_elem.text,
prefix=t + t + t + t + t)
text_body = html.escape(text_body)
Body_text = text_body + "\n"
myFile.write(Body_text)
text_field = t + t + t + t + "</Text>\n"
myFile.write(text_field)
Body = t + t + t + "</Body>\n"
myFile.write(Body)
if ('comment' in ch_elem.tag):
Edit = t + t + t + "<EditDetails>\n"
myFile.write(Edit)
if (ch_elem.text == None):
text_body = ""
else:
text_body = textwrap.indent(text=ch_elem.text,
prefix=t + t + t + t + t)
text_body = html.escape(text_body)
EditType = t + t + t + t + "<EditType>\n" + text_body + "\n" + t + t + t + t + "</EditType>\n"
#Body_text = text_body+"\n"
myFile.write(EditType)
Edit = t + t + t + "</EditDetails>\n"
myFile.write(Edit)
if ('sha1' in ch_elem.tag):
sha = ch_elem.text
if (type(sha) != type(None)):
shaText = t + t + t + '<Knowl key="sha">' + sha + '</Knowl>\n'
myFile.write(shaText)
else:
shaText = ''
Instance = t + t + "</Instance>\n"
myFile.write(Instance)
wikiConverter.instance_id += 1
def run(options):
datadir = 'meson-private'
if options.builddir is not None:
datadir = os.path.join(options.builddir, datadir)
if not os.path.isdir(datadir):
print('Current directory is not a build dir. Please specify it or '
'change the working directory to it.')
return 1
all_backends = options.all
print('This is a dump of the internal unstable cache of meson. This is for debugging only.')
print('Do NOT parse, this will change from version to version in incompatible ways')
print('')
coredata = cdata.load(options.builddir)
backend = coredata.get_option(OptionKey('backend'))
for k, v in sorted(coredata.__dict__.items()):
if k in ('backend_options', 'base_options', 'builtins', 'compiler_options', 'user_options'):
# use `meson configure` to view these
pass
elif k in ['install_guid', 'test_guid', 'regen_guid']:
if all_backends or backend.startswith('vs'):
print(k + ': ' + v)
elif k == 'target_guids':
if all_backends or backend.startswith('vs'):
print(k + ':')
dump_guids(v)
elif k in ['lang_guids']:
if all_backends or backend.startswith('vs') or backend == 'xcode':
print(k + ':')
dump_guids(v)
elif k == 'meson_command':
if all_backends or backend.startswith('vs'):
print('Meson command used in build file regeneration: ' + ' '.join(v))
elif k == 'pkgconf_envvar':
print('Last seen PKGCONFIG environment variable value: ' + v)
elif k == 'version':
print('Meson version: ' + v)
elif k == 'cross_files':
if v:
print('Cross File: ' + ' '.join(v))
elif k == 'config_files':
if v:
print('Native File: ' + ' '.join(v))
elif k == 'compilers':
for for_machine in MachineChoice:
print('Cached {} machine compilers:'.format(
for_machine.get_lower_case_name()))
dump_compilers(v[for_machine])
elif k == 'deps':
def print_dep(dep_key, dep):
print(' ' + dep_key[0][1] + ": ")
print(' compile args: ' + repr(dep.get_compile_args()))
print(' link args: ' + repr(dep.get_link_args()))
if dep.get_sources():
print(' sources: ' + repr(dep.get_sources()))
print(' version: ' + repr(dep.get_version()))
for for_machine in iter(MachineChoice):
items_list = list(sorted(v[for_machine].items()))
if items_list:
print(f'Cached dependencies for {for_machine.get_lower_case_name()} machine')
for dep_key, deps in items_list:
for dep in deps:
print_dep(dep_key, dep)
else:
print(k + ':')
print(textwrap.indent(pprint.pformat(v), ' '))
def create_breathe_files(headers, srcdir, component_name, sphinx_directory,
doxygen_config_filename):
"""Create rst files for sphinx from xml (doxygen) outputs generated from headers.
Parameters
----------
headers : list (cmake like)
headers files to parse
srcdir : string
absolute path to c/c++ sources (CMAKE_SOURCE_DIR)
component_name : string
component (numerics, kernel, ...) of interest
sphinx_directory : string
directory where rst files will be written
doxygen_config_filename : string
name (full path) of the doxygen configuration file
Notes:
* for each header, rst files (class, struct, file and source codes) will be generated
* three other 'main' rst files will be produced :
* breathe_api.rst, with the toctree for all classes and structs
* files_list.rst, with the toctree for all files documentation
* sources_list.rst with the toctree for all program listings
"""
# Get all headers for the current component
headers = bt.parse_cmake_list(headers)
rst_files = []
rst_programs = []
# Parse doxygen config
doxyconf = parse_doxygen_config(doxygen_config_filename)
xmlconf = {}
sphinx_directory = os.path.join(sphinx_directory, component_name)
if not os.path.exists(sphinx_directory):
os.makedirs(sphinx_directory)
xmlconf['CASE_SENSE_NAMES'] = doxyconf['CASE_SENSE_NAMES'].find('YES') > -1
xmlconf['XML_OUTPUT'] = os.path.join(doxyconf['OUTPUT_DIRECTORY'].lstrip(),
doxyconf['XML_OUTPUT'].lstrip())
all_index = {}
# -- Create rst files for classes, structs and files found in xml directory --
for hfile in headers:
xml2rst(hfile, srcdir, component_name, sphinx_directory, xmlconf,
all_index)
# -- Create rst files to collect list of classes and files (i.e. files just created above) --
class_and_struct_files = glob.glob(
os.path.join(sphinx_directory, 'class*.rst'))
class_and_struct_files += glob.glob(
os.path.join(sphinx_directory, 'struct*.rst'))
class_and_struct_files.sort()
pgm_files = glob.glob(os.path.join(sphinx_directory, 'pgm_*.rst'))
pgm_files.sort()
rst_files = glob.glob(os.path.join(sphinx_directory, 'file_*.rst'))
rst_files.sort()
all_files = class_and_struct_files + rst_files
all_files.sort()
# -- Name of the main rst files for the current component --
# usually : docs/sphinx/reference/cpp/component_name/autodoc_all.rst
outputname = os.path.join(sphinx_directory, 'autodoc_all.rst')
#title = component_name + ' component\n'
#title += len(title) * '=' + '\n\n'
indent = 4 * ' '
basename = '/reference/cpp/' + component_name + '/'
with open(outputname, 'wt') as out:
opt = ':maxdepth: 4\n'
for f in all_files:
name = os.path.basename(f).split('.')[0]
if name.find('class') > -1:
shorttitle = name.split('class')[-1]
text = '* :class:`' + shorttitle + '` : '
elif name.find('struct') > -1:
shorttitle = name.split('struct')[-1]
text = '* :class:`' + shorttitle + '` : '
elif name.find('file_') > -1:
shorttitle = name.split('file_')[-1].replace('_', '.')
name = basename + name
text = '* :doc:`' + shorttitle + '<' + name + '>` : '
try:
text += all_index[shorttitle] + '\n'
except:
text += '\n'
name = basename + name
#gen = textwrap.indent(text, 4 * ' ')
out.write(text)
out.write('\n\n')
# Classes and structs
outputname = os.path.join(sphinx_directory, 'autodoc_classes.rst')
subtitle = 'Classes and structs'
subtitle += '\n' + len(subtitle) * '-' + '\n\n'
with open(outputname, 'wt') as out:
#out.write(title)
out.write(subtitle)
#out.write('.. toctree::\n')
opt = ':maxdepth: 4\n'
#out.write(textwrap.indent(opt, indent))
#out.write('\n')
for f in class_and_struct_files:
name = os.path.basename(f).split('.')[0]
if name.find('class') > -1:
shorttitle = name.split('class')[-1]
elif name.find('struct') > -1:
shorttitle = name.split('struct')[-1]
text = '* :class:`' + shorttitle + '` : '
try:
text += all_index[shorttitle] + '\n'
except:
text += '\n'
name = basename + name
#gen = textwrap.indent(text, 4 * ' ')
out.write(text)
out.write('\n\n')
# Files doc
outputname = os.path.join(sphinx_directory, 'autodoc_files.rst')
subtitle = 'Files documentation\n'
subtitle += len(subtitle) * '-' + '\n\n'
with open(outputname, 'wt') as out:
out.write(subtitle)
#out.write('.. toctree::\n :maxdepth: 2\n\n')
for f in rst_files:
name = os.path.basename(f).split('.')[0]
shorttitle = name.split('file_')[-1].replace('_', '.')
name = basename + name
text = '* :doc:`' + shorttitle + '<' + name + '>` : '
try:
text += all_index[shorttitle] + '\n'
except:
text += '\n'
#gen = textwrap.indent(shorttitle + '<' + name + '>\n', 4 * ' ')
out.write(text)
out.write('\n\n')
# Programs listings
outputname = os.path.join(sphinx_directory, 'autodoc_pgm.rst')
label = '.. _' + component_name + '_pgm_listings:\n\n'
title = component_name.title() + ' programs listings\n'
title += len(title) * '-' + '\n\n'
title = label + title
with open(outputname, 'wt') as out:
out.write(title)
out.write('.. toctree::\n :maxdepth: 2\n\n')
for f in pgm_files:
name = os.path.basename(f).split('.')[0]
shorttitle = f.split(sphinx_directory)[-1].split('.')[0]
splits = shorttitle.split('_')
shorttitle = '/'.join(splits[2:-1]) + '.' + splits[-1]
name = basename + name
gen = textwrap.indent(shorttitle + '<' + name + '>\n', 4 * ' ')
out.write(gen)
def process_solutions(self, doctree: nodes.document, src: str) -> None:
"""Handle any solutions contained in the document.
This ensures that a ``*.py`` file is created in the ``resources`` directory
containing the actual solution.
It then also rewrites the given doctree to output a pair of code cells in
the resulting notebook. The first is a prompt for the user to input their
solution and the second contains a :magic:`ipython:load` declaration to
give the user the option to load in the solution if they wish to see it.
Parameters
----------
doctree:
The doctree to process
src:
The path to the file containing the document being processed
"""
docpath = pathlib.Path(src)
logger.debug("[tutorial]: processing solutions for: %s", docpath)
basename = f"{docpath.stem}-soln"
for idx, soln in enumerate(doctree.traverse(condition=solution)):
name = f"{basename}-{idx+1:02d}.py"
destination = pathlib.Path("resources", docpath.with_suffix(""), name)
refuri = relative_uri(src, str(destination))
# Convert the solution to a valid Python document that can be executed.
document = new_document("<solution>")
document += soln
# Rather than go through the trouble of maintaining 2 document translators,
# one for notebooks and another for Python files. Let's just use the notebook
# translator and do some post-processing on the result - much easier.
translator = NotebookTranslator(document)
document.walkabout(translator)
notebook = translator.asnotebook()
blocks = []
for cell in notebook.cells:
source = cell.source
# Comment out the lines containing markdown.
if cell.cell_type == "markdown":
source = textwrap.indent(source, "# ")
blocks.append(source)
self.resources[str(destination)] = ("create", "\n".join(blocks))
# TODO: Expose config options for these
# TODO: Translations?
your_soln = nodes.literal_block(
"", "# Write your solution here...\n", language="python"
)
load_soln = nodes.literal_block(
"",
f"# Execute this cell to load the example solution\n%load {refuri}\n",
language="python",
)
# Replace the actual solution with the 2 cells defined above.
soln.children = [your_soln, load_soln]
def print_new_expected(s: str) -> None:
print(f'{"":>12}"""\\', end="")
for l in s.splitlines(keepends=False):
print("\n" + textwrap.indent(repr(l)[1:-1], " " * 12), end="")
print('"""\n')
def gen_ctypes_wrappers(fcninfo, ofname):
file_header = file_docstring.format(
datetime=time.strftime("%d %b %Y %H:%M:%S", time.localtime()))
contents = ''
function_pointer_string = ''
namedtuples_header = ' ' * 8 + '# Named tuples to contain the outputs of DLL calls\n'
for fcn, data in sorted(six.iteritems(fcninfo)):
function_pointer_string += ' ' * 8 + 'self._{name:s} = self._getfcn(self.dll, \'{name:s}\')\n'.format(
name=fcn)
i, o = [], []
for arg in data['argnames']:
if arg in data['input_args']:
i.append(arg)
elif arg in data['output_args']:
o.append(arg)
elif arg in data['inout_args']:
i.append(arg)
o.append(arg)
proto = ','.join(o) + ' = ' + fcn + '(' + ','.join(i) + ')'
headline = 'def {fname:s}(self,{inargs:s}):\n'.format(
inargs=','.join(i), fname=fcn)
body = '\"\"\"\n{proto:s}\n\"\"\"\n'.format(proto=proto)
def gen_val(typ, dim, default=''):
if typ == 'int' and dim == 0:
return 'ct.c_long({default:s})'.format(default=default)
elif typ == 'int' and dim > 0:
return '({dim:d}*ct.c_long)()'.format(default=default, dim=dim)
elif typ == 'double' and dim == 0:
return 'ct.c_double({default:s})'.format(default=default)
elif typ == 'double' and dim < 0:
return '(len({default:s})*ct.c_double)(*{default:s})'.format(
default=default)
elif typ == 'double' and dim > 0:
return '({dim:d}*ct.c_double)()'.format(default=default,
dim=dim)
elif typ == 'char' and dim != 0 and default:
return 'ct.create_string_buffer({default:s}.encode(\'utf-8\'),{dim:d})'.format(
default=default, dim=abs(dim))
elif typ == 'char' and dim != 0 and not default:
return 'ct.create_string_buffer({dim:d})'.format(dim=abs(dim))
else:
raise KeyError('Invalid type/dim pair:' + str((typ, dim)))
# Add the type definitions
for arg in data['argnames']:
if arg in data['input_args'] or arg in data['inout_args']:
typ, dim = data['input_args'].get(
arg, None) or data['inout_args'].get(arg, None)
if dim == '*':
body += '{name:s} = '.format(
name=arg
) + 'ct.create_string_buffer({default:s},len{default:s})'.format(
default=arg) + '\n'
else:
body += '{name:s} = '.format(name=arg) + gen_val(
typ, dim, default=arg) + '\n'
elif arg in data['output_args']:
typ, dim = data['output_args'][arg]
body += '{name:s} = '.format(name=arg) + gen_val(
typ, dim, default='') + '\n'
# Add the function call
string_lengths = []
arg_strings = []
for arg in data['argnames']:
typ, dim = data['input_args'].get(
arg, None) or data['inout_args'].get(
arg, None) or data['output_args'].get(arg, None)
if dim == 0 and typ in ['int', 'double']:
arg_strings.append('ct.byref({arg:s})'.format(arg=arg))
else:
arg_strings.append(arg)
if typ == 'char':
if dim == '*':
string_lengths.append(str(10000))
else:
string_lengths.append(str(dim))
body += '\nif self._{fcnname:s} is None: raise ValueError("The function {fcnname:s} could not be loaded from the shared library.") \n'.format(
fcnname=fcn)
if len(string_lengths) == 0:
string_args = ''
else:
string_args = ',' + ','.join(string_lengths)
body += 'self._{fcnname:s}({args:s}{string_args:s})\n'.format(
fcnname=fcn, args=','.join(arg_strings),
string_args=string_args) + '\n'
# Add the return call
arg_strings = []
arg_names = []
for arg in data['argnames']:
if arg in data['output_args'] or arg in data['inout_args']:
typ, dim = data['output_args'].get(
arg, None) or data['inout_args'].get(arg, None)
arg_names.append('"' + arg + '"')
if dim == 0 and typ in ['int', 'double']:
arg_strings.append('{arg:s}.value'.format(arg=arg))
elif dim > 0 and typ in ['int', 'double']:
arg_strings.append('list({arg:s})'.format(arg=arg))
elif typ == 'char':
arg_strings.append('trim({arg:s}.raw)'.format(arg=arg))
else:
arg_strings.append(arg)
if len(arg_strings) > 1:
namedtuples_header += ' ' * 8 + 'self._{fcn:s}output_tuple = namedtuple(\'{fcn:s}output\',[{argnames:s}])\n'.format(
fcn=fcn, argnames=','.join(arg_names))
body += 'return self._{fcn:s}output_tuple({args:s})'.format(
fcn=fcn, args=','.join(arg_strings)) + '\n\n'
elif len(arg_strings) == 1:
body += 'return {arg:s}'.format(arg=arg_strings[0]) + '\n\n'
else:
body += 'return\n\n'
contents += textwrap.indent(headline, ' ' * 4) + textwrap.indent(
body, ' ' * 8)
# Write it into the output file
with open(ofname, 'w') as fp:
fp.write(file_header)
fp.write(wrapper_header)
fp.write(namedtuples_header + '\n')
fp.write(function_pointer_string + '\n')
fp.write(contents)
def codegen_construct(class_header, class_name, class_name_elaborated, fields, has_post_ctor, meth=False):
body = []
for kind, field in fields:
name = field.spelling
spec = parse_codegen_spec(kind)
if spec.is_meta:
tpl = construct_meta_tpl
if spec.is_plain:
if spec.star:
tpl = construct_plain_star_tpl
else:
tpl = construct_plain_tpl
if spec.is_list:
if spec.star:
tpl = construct_list_star_tpl
else:
tpl = construct_list_tpl
if spec.is_dlist:
if spec.star:
tpl = construct_dlist_star_tpl
else:
tpl = construct_dlist_tpl
if spec.is_set:
tpl = construct_set_tpl
if spec.is_dict:
tpl = construct_dict_tpl
if spec.is_dict:
key_type = spec.element_type[0]
value_type = spec.element_type[1]
else:
key_type = None
value_type = None
body.append(
tpl.format(name=name,
element_type=spec.element_type,
key_type=key_type,
value_type=value_type,
deref=spec.deref))
if meth:
function_tpl = construct_meth_tpl
function_body_indent = 4
if has_post_ctor:
post_ctor = post_construct_meth_tpl
else:
post_ctor = ''
else:
function_tpl = construct_tpl
function_body_indent = 0
if has_post_ctor:
post_ctor = post_construct_tpl
else:
post_ctor = ''
code = function_tpl.format(class_header=class_header,
class_name=class_name,
class_name_elaborated=class_name_elaborated,
construct_body=textwrap.indent(''.join(body), ' ' * function_body_indent).strip(),
post_construct=post_ctor)
return code
def create_dockerfile_object(self, directory: str) -> str:
"""
Writes a dockerfile to the provided directory using the specified
arguments on this Docker storage object.
In order for the docker python library to build a container it needs a
Dockerfile that it can use to define the container. This function takes the
specified arguments then writes them to a temporary file called Dockerfile.
*Note*: if `files` are added to this container, they will be copied to this directory as well.
Args:
- directory (str, optional): A directory where the Dockerfile will be created,
if no directory is specified is will be created in the current working directory
Returns:
- str: the absolute file path to the Dockerfile
"""
# Generate single pip install command for python dependencies
pip_installs = "RUN pip install "
if self.python_dependencies:
for dependency in self.python_dependencies:
pip_installs += "{} ".format(dependency)
# Generate ENV variables to load into the image
env_vars = ""
if self.env_vars:
white_space = " " * 20
env_vars = "ENV " + " \ \n{}".format(white_space).join(
"{k}={v}".format(k=k, v=v) for k, v in self.env_vars.items())
# Copy user specified files into the image
copy_files = ""
if self.files:
for src, dest in self.files.items():
fname = os.path.basename(src)
full_fname = os.path.join(directory, fname)
if os.path.exists(full_fname) and filecmp.cmp(
src, full_fname) is False:
raise ValueError(
"File {fname} already exists in {directory}".format(
fname=full_fname, directory=directory))
else:
shutil.copy2(src, full_fname)
copy_files += "COPY {fname} {dest}\n".format(
fname=full_fname if self.dockerfile else fname, dest=dest)
# Write all flows to file and load into the image
copy_flows = ""
for flow_name, flow_location in self.flows.items():
clean_name = slugify(flow_name)
flow_path = os.path.join(directory, "{}.flow".format(clean_name))
with open(flow_path, "wb") as f:
cloudpickle.dump(self._flows[flow_name], f)
copy_flows += "COPY {source} {dest}\n".format(
source=flow_path
if self.dockerfile else "{}.flow".format(clean_name),
dest=flow_location,
)
# Write all extra commands that should be run in the image
extra_commands = ""
for cmd in self.extra_commands:
extra_commands += "RUN {}\n".format(cmd)
# Write a healthcheck script into the image
with open(os.path.join(os.path.dirname(__file__), "_healthcheck.py"),
"r") as healthscript:
healthcheck = healthscript.read()
healthcheck_loc = os.path.join(directory, "healthcheck.py")
with open(healthcheck_loc, "w") as health_file:
health_file.write(healthcheck)
if self.dockerfile:
with open(self.dockerfile, "r") as contents:
base_commands = textwrap.indent("\n" + contents.read(),
prefix=" " * 16)
else:
base_commands = "FROM {base_image}".format(
base_image=self.base_image)
file_contents = textwrap.dedent("""\
{base_commands}
RUN pip install pip --upgrade
{extra_commands}
{pip_installs}
RUN mkdir -p /root/.prefect/
{copy_flows}
COPY {healthcheck_loc} /root/.prefect/healthcheck.py
{copy_files}
{env_vars}
RUN python /root/.prefect/healthcheck.py '[{flow_file_paths}]' '{python_version}'
""".format(
base_commands=base_commands,
extra_commands=extra_commands,
pip_installs=pip_installs,
copy_flows=copy_flows,
healthcheck_loc=healthcheck_loc
if self.dockerfile else "healthcheck.py",
copy_files=copy_files,
env_vars=env_vars,
flow_file_paths=", ".join(
['"{}"'.format(k) for k in self.flows.values()]),
python_version=(sys.version_info.major, sys.version_info.minor),
))
file_contents = "\n".join(line.lstrip()
for line in file_contents.split("\n"))
dockerfile_path = os.path.join(directory, "Dockerfile")
with open(dockerfile_path, "w+") as dockerfile:
dockerfile.write(file_contents)
return dockerfile_path
def _clean_output(output):
output = output.replace("{", "{{").replace( "}", "}}")
return textwrap.indent(output, ' | ')
def append(self, txt: str, sep='\n') -> 'MultiLineFormatter':
self.text += sep
if txt:
self.text += indent(dedent(txt), self.current_indent)
return self
def show_not_passed(e):
print('\033[91m', " Not passsed", '\033[0m')
print(textwrap.indent(str(e), " "))
rst.write("""\
:drc_rule:`{rt.template}`
{hd}
""".format(rt=rt, hd='-'*(len(rt.template)+len(':drc_rule:``'))))
if rt.notes:
rst.write("""\
.. note::
{}
""".format(textwrap.indent(rt.notes, prefix=' ')))
headers = ('Name', 'Description', 'Flags', 'Value')
headers_fmt = (':drc_rule:`Name`', 'Description', ':drc_flag:`Flags`', 'Value')
rst.write("""\
.. list-table:: {rt.description}
:header-rows: 1
:stub-columns: 1
:widths: 10 75 5 10
* - {h}
""".format(rt=rt,h='\n - '.join(headers_fmt)))
for r in rt.rules:
def _fill_text(self, text, width, indent): return "\n".join([textwrap.fill(line, width) for line in textwrap.indent(textwrap.dedent(text), indent).splitlines()])
async def _eval(ctx, *, body: str):
'''Run python scripts on discord!'''
await to_code_block(ctx, body)
env = {
'bot': bot,
'ctx': ctx,
'channel': ctx.message.channel,
'author': ctx.message.author,
'server': ctx.message.server,
'message': ctx.message,
}
env.update(globals())
body = cleanup_code(content=body)
stdout = io.StringIO()
to_compile = 'async def func():\n%s' % textwrap.indent(body, ' ')
try:
exec(to_compile, env)
except SyntaxError as e:
return await bot.say(get_syntax_error(e))
func = env['func']
try:
with redirect_stdout(stdout):
ret = await func()
except Exception as e:
value = stdout.getvalue()
x = await bot.say('```py\n{}{}\n```'.format(value,
traceback.format_exc()))
try:
await bot.add_reaction(x, '\U0001f534')
except:
pass
else:
value = stdout.getvalue()
if TOKEN in value:
value = value.replace(TOKEN, "[EXPUNGED]")
if ret is None:
if value:
try:
x = await bot.say('```py\n%s\n```' % value)
except:
x = await bot.say('```py\n\'Result was too long.\'```')
try:
await bot.add_reaction(x, '\U0001f535')
except:
pass
else:
try:
await bot.add_reaction(ctx.message, '\U0001f535')
except:
pass
else:
try:
x = await bot.say('```py\n%s%s\n```' % (value, ret))
except:
x = await bot.say('```py\n\'Result was too long.\'```')
try:
await bot.add_reaction(x, '\U0001f535')
except:
pass
def AddScripts(self, codeArg, outputArg, mixerLevelArg):
listCheckLineWhitoutSpace = []
listCheckLine = []
countBackSlash = 0
countScriptsAdded = 0
countLineAdded = 0
countLine = 0
checkLine = 0
checkQuotePassing = 0
checkCharPassing = 0
countRecursFiles = 0
checkParenthesesCharPassing = 0
checkBracketsCharPassing = 0
checkBracesCharPassing = 0
if codeArg == "python":
detectFile = "py"
blockDirs = r"__pycache__"
recursFiles = [f for f in glob.glob("{0}{1}**{1}*.{2}".format(outputArg, self.utils.Platform(), detectFile), recursive=True)]
# -- Count the number of lines that will be checked before filling -- #
for file in recursFiles:
if re.match(blockDirs, file):
continue
else:
with open(file , "r") as readFile:
readF = readFile.readlines()
for eachLine in readF:
if not eachLine:
continue
countLine += 1
for number in recursFiles:
countRecursFiles += 1
print("\n[+] Running adding of random scripts in {0} file(s)...\n".format(countRecursFiles))
# -- Padding scripts added -- #
with tqdm(total=countRecursFiles) as pbar:
for file in recursFiles:
pbar.update(1)
if re.match(blockDirs, file):
continue
else:
with fileinput.input(file, inplace=True) as inputFile:
for eachLine in inputFile:
print(eachLine)
if eachLine == "\n" or "coding" in eachLine:
continue
else:
if codeArg == "python":
listCheckLine = [] # Initialize var
spaces = len(eachLine) - len(eachLine.lstrip()) # Check line indent
detectStringVar = r".*\w+\s*\={1}\s*r+[\"|\']{1}"
noAddScript = r"^\@|\s+\@|\s+return|\s*def\s+.+\s*\:{1}|^class\s+.+\s*\:{1}|.*[\[|\(|\{|\,|\\]$|\s+[\)|\]|\}]$"
addIndentScript = r".*\:{1}\s"
quoteIntoVariable = r".*\={1}\s*\w*\.?\w*[\(|\.]{1}[\"|\']{3}|.*\={1}\s*[\"|\']{3}" # """ and ''' before an variables
quoteOfCommentariesMultipleLines = r"^\s*[\"|\']{3}$" # """ and ''' without before variables and if commentaries is over multiple lines
quoteOfEndCommentariesMultipleLines = r"^\s*[\"|\']{3}\)?\.?" # """ and ''' without before variables, if commentaries is over multiple lines and he finish by .format() funtion
for i in eachLine:
listCheckLine.append(i)
# -- Check line below after '\' backslash char -- #
if re.match(r".+\\$", eachLine):
countBackSlash += 1
if countBackSlash > 0:
if re.match(r".+\\$", eachLine) is None:
countBackSlash = 0
continue
# -- Check if end char in line is " or ' -- #
if re.match(r"\"|\'", listCheckLine[-2]):
try:
if re.match(r"\'|\"", listCheckLine[-3]) and re.match(r"\'|\"", listCheckLine[-4]):
pass
else:
if re.match(detectStringVar, eachLine):
pass
else:
continue
except IndexError:
continue
# -- Check code into """ or ''' -- #
if re.match(quoteIntoVariable, eachLine):
checkQuotePassing += 1
continue
elif re.match(quoteOfCommentariesMultipleLines, eachLine) or re.match(quoteOfEndCommentariesMultipleLines, eachLine):
checkQuotePassing += 1
if checkQuotePassing == 2:
checkQuotePassing = 0
continue
if checkQuotePassing == 1:
continue
elif checkQuotePassing == 2:
checkQuotePassing = 0
continue
else:
checkQuotePassing = 0
# -- Add scripts -- #
if re.match(noAddScript, eachLine) is not None:
continue
elif re.match(addIndentScript, eachLine) is not None:
if spaces == 0:
print(textwrap.indent(Padding.ScriptsGenerator(self, codeArg, mixerLevelArg)," "))
countScriptsAdded += 1
elif spaces == 4:
print(textwrap.indent(Padding.ScriptsGenerator(self, codeArg, mixerLevelArg), " "))
countScriptsAdded += 1
elif spaces == 8:
print(textwrap.indent(Padding.ScriptsGenerator(self, codeArg, mixerLevelArg), " "))
countScriptsAdded += 1
elif spaces == 12:
print(textwrap.indent(Padding.ScriptsGenerator(self, codeArg, mixerLevelArg), " "))
countScriptsAdded += 1
elif spaces == 16:
print(textwrap.indent(Padding.ScriptsGenerator(self, codeArg, mixerLevelArg), " "))
countScriptsAdded += 1
elif spaces == 20:
print(textwrap.indent(Padding.ScriptsGenerator(self, codeArg, mixerLevelArg), " "))
countScriptsAdded += 1
elif spaces == 24:
print(textwrap.indent(Padding.ScriptsGenerator(self, codeArg, mixerLevelArg), " "))
countScriptsAdded += 1
elif spaces == 28:
print(textwrap.indent(Padding.ScriptsGenerator(self, codeArg, mixerLevelArg), " "))
countScriptsAdded += 1
elif spaces == 32:
print(textwrap.indent(Padding.ScriptsGenerator(self, codeArg, mixerLevelArg), " "))
countScriptsAdded += 1
elif spaces == 36:
print(textwrap.indent(Padding.ScriptsGenerator(self, codeArg, mixerLevelArg), " "))
countScriptsAdded += 1
elif spaces == 40:
print(textwrap.indent(Padding.ScriptsGenerator(self, codeArg, mixerLevelArg), " "))
countScriptsAdded += 1
elif spaces == 44:
print(textwrap.indent(Padding.ScriptsGenerator(self, codeArg, mixerLevelArg), " "))
countScriptsAdded += 1
elif spaces == 48:
print(textwrap.indent(Padding.ScriptsGenerator(self, codeArg, mixerLevelArg), " "))
countScriptsAdded += 1
elif spaces == 52:
print(textwrap.indent(Padding.ScriptsGenerator(self, codeArg, mixerLevelArg), " "))
countScriptsAdded += 1
elif spaces == 56:
print(textwrap.indent(Padding.ScriptsGenerator(self, codeArg, mixerLevelArg), " "))
countScriptsAdded += 1
elif spaces == 60:
print(textwrap.indent(Padding.ScriptsGenerator(self, codeArg, mixerLevelArg), " "))
countScriptsAdded += 1
else:
continue
else:
if spaces == 0:
print(textwrap.indent(Padding.ScriptsGenerator(self, codeArg, mixerLevelArg),""))
countScriptsAdded += 1
elif spaces == 4:
print(textwrap.indent(Padding.ScriptsGenerator(self, codeArg, mixerLevelArg), " "))
countScriptsAdded += 1
elif spaces == 8:
print(textwrap.indent(Padding.ScriptsGenerator(self, codeArg, mixerLevelArg), " "))
countScriptsAdded += 1
elif spaces == 12:
print(textwrap.indent(Padding.ScriptsGenerator(self, codeArg, mixerLevelArg), " "))
countScriptsAdded += 1
elif spaces == 16:
print(textwrap.indent(Padding.ScriptsGenerator(self, codeArg, mixerLevelArg), " "))
countScriptsAdded += 1
elif spaces == 20:
print(textwrap.indent(Padding.ScriptsGenerator(self, codeArg, mixerLevelArg), " "))
countScriptsAdded += 1
elif spaces == 24:
print(textwrap.indent(Padding.ScriptsGenerator(self, codeArg, mixerLevelArg), " "))
countScriptsAdded += 1
elif spaces == 28:
print(textwrap.indent(Padding.ScriptsGenerator(self, codeArg, mixerLevelArg), " "))
countScriptsAdded += 1
elif spaces == 32:
print(textwrap.indent(Padding.ScriptsGenerator(self, codeArg, mixerLevelArg), " "))
countScriptsAdded += 1
elif spaces == 36:
print(textwrap.indent(Padding.ScriptsGenerator(self, codeArg, mixerLevelArg), " "))
countScriptsAdded += 1
elif spaces == 40:
print(textwrap.indent(Padding.ScriptsGenerator(self, codeArg, mixerLevelArg), " "))
countScriptsAdded += 1
elif spaces == 44:
print(textwrap.indent(Padding.ScriptsGenerator(self, codeArg, mixerLevelArg), " "))
countScriptsAdded += 1
elif spaces == 48:
print(textwrap.indent(Padding.ScriptsGenerator(self, codeArg, mixerLevelArg), " "))
countScriptsAdded += 1
elif spaces == 52:
print(textwrap.indent(Padding.ScriptsGenerator(self, codeArg, mixerLevelArg), " "))
countScriptsAdded += 1
elif spaces == 56:
print(textwrap.indent(Padding.ScriptsGenerator(self, codeArg, mixerLevelArg), " "))
countScriptsAdded += 1
elif spaces == 60:
print(textwrap.indent(Padding.ScriptsGenerator(self, codeArg, mixerLevelArg), " "))
countScriptsAdded += 1
else:
continue
# -- Check padding has added in output script -- #
for file in recursFiles:
if re.match(blockDirs, file):
continue
else:
with open(file , "r") as readFile:
readF = readFile.readlines()
for eachLine in readF:
if not eachLine:
continue
checkLine += 1
countLineAdded = checkLine - countLine
if (self.remove.LineBreaks(codeArg, outputArg) == 0):
if checkLine > countLine:
print("\n-> {0} scripts added in {1} file(s)\n".format(countScriptsAdded, countRecursFiles))
print("-> {0} lines added in {1} file(s)\n".format(countLineAdded, countRecursFiles))
return EXIT_SUCCESS
else:
return EXIT_FAILURE
else:
return EXIT_FAILURE
# Patch the socket module
socks.setdefaultproxy(
socks.PROXY_TYPE_SOCKS5, proxy_host, int(proxy_port), rdns=True
) # rdns is by default on true. Never use rnds=False with TOR, otherwise you are screwed!
socks.wrap_module(socket)
socket.create_connection = create_connection
if args.deep_scrape:
results = deep_scrape(args.query)
else:
results = scrape(args.query, args.num_results_per_page, args.num_pages)
for result in results:
logger.info(
'{} links found! The search with the keyword "{}" yielded the result:{}'
.format(len(result['results']), result['search_keyword'],
result['num_results_for_kw']))
if args.view:
import webbrowser
webbrowser.open(result['cache_file'])
for link_title, link_snippet, link_url in result['results']:
print('Link: {}'.format(urllib.parse.unquote(link_url.geturl())))
if args.verbosity > 1:
import textwrap
print('Title: \n{}'.format(
textwrap.indent('\n'.join(textwrap.wrap(link_title, 50)),
'\t')))
print('Description: \n{}\n'.format(
textwrap.indent('\n'.join(textwrap.wrap(link_snippet, 70)),
'\t')))
print('*' * 70)
def module_docstrings2rst(component_name, module_path, module_name,
sphinx_directory, swig_working_dir):
"""Import a module and create 'rst' (autodoc)
file for each documented (docstrings) object.
Parameters
----------
component_name : string
name of the current component (e.g. kernel)
module_path : string
current module path, relative to swig working dir
(usually wrap/siconos), e.g. mechanics/collision for module bodies.
module_name : string
name of the module (e.g. sensor)
sphinx_directory : string
directory (absolute) where rst files will
be written.
swig_working_dir : string
directory (absolute) where python modules
are generated using swig
Notes:
* module_path is required for module like sensor
located in siconos/control directory,
to build module name like siconos.control.sensor
* Usually : sphinx_directory
= binary_dir/docs/sphinx/reference/python/module_name
* Results :
* pyclass*.rst or pyfunction*.rst in sphinx_directory
* pyclass.rst to collect (docstree toc) all pyclass*
* pyfunctions.rst to collect all pyfunc *
"""
# Test case with submodules (e.g. sensor in control)
if module_path == '.' or module_path == '':
module_name = 'siconos.' + module_name
else:
module_path = module_path.replace(r'/', r'.')
module_name = 'siconos.' + module_path + '.' + module_name
comp = importlib.import_module(module_name)
sphinx_directory = sphinx_directory + module_name.replace(r'.', '_')
if not os.path.exists(sphinx_directory):
os.makedirs(sphinx_directory)
features_filename = comp.__file__.split(
component_name)[0] + component_name + '.pickle'
with open(features_filename, 'rb') as f:
features = pickle.load(f)
# We have to remove pyfiles each time the function is call
# because of 'a' (append) in writing process
pyfiles = glob.glob(os.path.join(sphinx_directory, '*pyfile.rst'))
pyfiles += glob.glob(os.path.join(sphinx_directory, '*pyclass.rst'))
for file in pyfiles:
os.remove(file)
class_files = []
pyfunc_files = []
objlist = [obj for obj in dir(comp) if not obj.startswith('_')]
for obj in objlist:
current = getattr(comp, obj)
needs_doc = hasattr(current, '__doc__') and current.__doc__ is not None
if needs_doc and len(current.__doc__.strip()) > 0:
gen, kind, name = create_autodoc(current, module_name)
if kind == 'pyclass': # one rst file per class
outputname = name + '_' + kind + '.rst'
outputname = os.path.join(sphinx_directory, outputname)
class_files.append(outputname)
elif kind == 'pyfunction': # one rst file for all functions from a given header
if name in features:
featname = features[name]
elif name.replace('_', '::', 1) in features:
# - when two classes wrapped with swig have
# the same method (same name),
# swig create class1_methodname and class2_methodname.
# while features name is class1::methodname.
# - the same kind of thing happens for static class methods.
# We have to take these into account ... and the fact
# that some methods names may contain '_'
# (maxreplace=1 in replace below)
featname = features[name.replace('_', '::', 1)]
elif name.split('::')[-1] in features:
# another way for swig to deal with
# namespaces ...
featname = features[name.split('::')[-1]]
else:
keys = list(features.keys())
for k in keys:
if k.count(name) > -1:
featname = features[k]
break
else:
raise Exception('Unknown feature name : ', name)
#featname = name
outputname = os.path.basename(featname).split('.')[0]
outputname = outputname + '_pyfile.rst'
outputname = os.path.join(sphinx_directory, outputname)
pyfunc_files.append(outputname)
else:
# pydata, processed later.
continue
with open(outputname, 'a+') as out:
out.write(gen)
out.write('\n')
# -- Create rst files to collect list of classes and files (i.e. files just created above) --
#class_files.sort()
pyfunc_files = list(set(pyfunc_files))
#pyfunc_files.sort()
# Insert title (required to be taken into account in toctree ...)
for fname in pyfunc_files:
shortname = os.path.basename(fname).split('.')[0]
label = '.. _' + shortname + ':\n\n'
title = shortname.split('_pyfile')[0] + ' (functions)'
lenname = len(title)
title = label + title + '\n' + lenname * '-' + '\n\n'
with open(fname, 'r+') as f:
lines = f.read()
f.seek(0, 0)
f.write(title + lines)
all_index_filename = os.path.join(swig_working_dir,
component_name + '_index.pickle')
with open(all_index_filename, 'rb') as f:
all_index = pickle.load(f)
allfiles = class_files + pyfunc_files
allfiles.sort()
submodule_name = module_name.split('.')[-1]
outputname = os.path.join(sphinx_directory, 'autodoc_all.rst')
title = module_name + '\n'
title += len(title) * '=' + '\n\n'
indent = 4 * ' '
basename = '/reference/python/' + module_name.replace(r'.', '_')
# A few lines to illustrate module usage
header = '**Usage example** :\n\n.. code-block:: python\n\n'
importname = 's' + module_name.split('.')[-1][0]
code = 'import ' + module_name + ' as ' + importname
code += '\n \nhelp(' + importname + '.SomeClass)\n\n'
header += textwrap.indent(code, ' ')
header += '**Classes and functions**\n\n'
with open(outputname, 'wt') as out:
out.write(title)
out.write(header)
#out.write('.. toctree::\n :maxdepth: 2\n\n')
gen = '* :doc:`Enums and constants <' + os.path.join(
basename, 'autodoc_pydata') + '>`\n'
for f in allfiles:
name = os.path.basename(f).split('.')[0]
text = ''
if name.count('_pyclass') > 0:
realname = name.split('_pyclass')[0]
shorttitle = realname + ' (class) '
#text = '* :doc:`' + shorttitle + '<' + basename + name + '>` : '
text = '* :py:class:`' + module_name + '.' + realname + '` : '
try:
text += all_index[realname] + '\n'
except:
text += ' \n'
elif name.count('_pyfile') > 0:
realname = name.split('_pyfile')[0]
shorttitle = realname + ' (functions) '
text = '* :doc:`' + shorttitle + '<' + os.path.join(
basename, name) + '>` : '
if realname + '.h' in all_index:
text += all_index[realname + '.h'] + ' \n'
elif realname + '.hpp' in all_index:
text += all_index[realname + '.hpp'] + ' \n'
else:
text += ' \n'
else:
shorttitle = ''
gen += text
# out.write(textwrap.indent(gen, indent))
out.write(gen + '\n')
# It might be necessary to parse some latex from doxygen and convert it to sphinx ...
latex_dir = swig_working_dir + 'tmp_' + component_name
replace_latex(outputname, latex_dir)
# Process enums
# Get saved enums for the current module
outputname = os.path.join(sphinx_directory, 'autodoc_pydata.rst')
title = module_name + ' constants (Python API)\n'
title += len(title) * '-' + '\n\n'
title += 'All the predefined global constants in ' + module_name
title += '(generated from C++ enum, global variables, ...) \n\n'
enumskeys = [k for k in features if k.find('pydata') > -1]
enums = [features[key] for key in enumskeys]
header = '**Usage** :\n\n.. code-block:: python\n\n'
importname = 's' + module_name.split('.')[-1][0]
code = 'import ' + module_name + ' as ' + importname
code += '\n \nprint(' + importname + '.CONSTNAME)\n\n'
header += textwrap.indent(code, ' ')
title += header
title += '\n-----\n\n**List and descriptions of available constants** :\n\n'
with open(outputname, 'wt') as out:
out.write(title)
for key in enumskeys:
enums = features[key]
for ename in enums:
# Document only data available in python API
if hasattr(comp, ename):
# and only data with a description
if len(enums[ename][1].strip()) > 0:
gen = ''
gen += '.. _pydata_' + ename + ':\n\n'
gen += '.. py:data:: ' + ename + '\n\n'
if len(enums[ename][0]) > 0:
# Add initializer value if set
gen += ' {0} ({1})\n\n'.format(
enums[ename][1].strip(), enums[ename][0])
else:
gen += ' {0} \n\n'.format(
enums[ename][1].strip())
out.write(gen)
# It might be necessary to parse some latex from doxygen and convert it to sphinx ...
replace_latex(outputname, latex_dir)
module_name, func_name = func.split(":", 1)
module = import_module(module_name)
func = getattr(module, func_name)
return func
def parse_static_item(static_item):
assert "=" in static_item, "{!r} should have format 'path=directory'"
return static_item.split("=", 1)
def main():
sys.path.insert(0, os.getcwd())
# Parse the args.
kwargs = vars(parser.parse_args(sys.argv[1:]))
application = import_func(kwargs.pop("application"))
static = list(map(parse_static_item, kwargs.pop("static")))
# Set up logging.
verbosity = (kwargs.pop("verbose") - kwargs.pop("quiet")) * 10
logging.basicConfig(level=max(logging.ERROR - verbosity, logging.DEBUG),
format="%(message)s")
logging.getLogger("aiohttp").setLevel(
max(logging.INFO - verbosity, logging.DEBUG))
logger.setLevel(max(logging.INFO - verbosity, logging.DEBUG))
# Serve!
serve(application, static=static, **kwargs)
__doc__ = __doc__.format(help=textwrap.indent(parser.format_help(), " "),
**HELP)
def replace_latex(pyfile, latex_dir):
"""Post processing of latex forms in docstrings.
Parameters
----------
pyfile : string
name (full path) of the python file to process
latex_dir : string
directory which contains pickle files with latex forms
(result of do_latex or do_verbatim call in sicodoxy2swig)
Usually : pyfile = some_component.py (e.g. numerics.py)
and latex_dir = wrap/siconos/tmp_component_name.
"""
# Parse latex_dir and get all pickle files
formfiles = glob.glob(os.path.join(latex_dir, 'latex_*.pickle'))
# Runner : use sed rather than python replace. But less portable (?)
#runner = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'replace_latex.sh')
# temp file for outputs.
target = pyfile + '.copy'
shutil.copyfile(pyfile, target)
# Read input (.py)
with open(pyfile, "r") as f:
source_lines = f.readlines()
rst = []
# Parse and replace :
# for each formula found in each pickle file,
# replace FORMULA_Id with the proper string
# in temp list.
for fname in formfiles:
with open(fname, 'rb') as f:
latex_dict = pickle.load(f)
for form in latex_dict:
idf = 'FORMULA' + str(form) + '_'
# we must \\dot in \rst doxygen
# else there is a confusion with dot from graphviz.
formula = latex_dict[form]["latex"].replace(r'\\', '\\\\')
# escape \
formula = latex_dict[form]["latex"].replace('\\', '\\\\')
formula_type = latex_dict[form]["label"] # inline or not
#formula = ''.join(formula)
for line in source_lines:
if formula_type == 'inline':
rst.append(line.replace(idf, formula))
else:
indent = len(line) - len(line.lstrip())
rst.append(
line.replace(
idf, textwrap.indent(formula, indent * ' ')))
source_lines = list(rst)
rst = []
#cmd = [runner, idf, formula, source, target]
#print(cmd)
#subprocess.call(cmd)
#shutil.copyfile(target, source)
# Replace .py with new results.
with open(target, 'w') as f:
for line in source_lines:
f.write(line)
shutil.move(target, pyfile)
def as_text(self) -> str:
return '\n'.join('* DistributionPoint:\n%s' % textwrap.indent(dp.as_text(), ' ')
for dp in self.value)
def query_iter(self,
ids=None,
patterns=None,
children=False,
parents=False,
objects_per_report=0):
"""
Match and fetch objects from the database, in object number-limited
chunks.
Args:
ids: A dictionary of object list names, and lists
of IDs of objects to match. None means empty
dictionary.
patterns: A dictionary of object list names, and lists
of LIKE patterns, for IDs of objects to match.
None means empty dictionary.
children: True if children of matched objects should be
matched as well.
parents: True if parents of matched objects should be
matched as well.
objects_per_report: An integer number of objects per each returned
report data, or zero for no limit.
Returns:
An iterator returning report JSON data adhering to the latest I/O
schema version, each containing at most the specified number of
objects.
Raises:
`IncompatibleSchema` if the dataset schema is incompatible with
the latest I/O schema.
"""
# Calm down, we'll get to it,
# pylint: disable=too-many-locals,too-many-statements
assert ids is None or isinstance(ids, dict)
if ids is None:
ids = dict()
assert all(
isinstance(k, str) and isinstance(v, list) and all(
isinstance(e, str) for e in v) for k, v in ids.items())
assert patterns is None or isinstance(patterns, dict)
if patterns is None:
patterns = dict()
assert all(
isinstance(k, str) and isinstance(v, list) and all(
isinstance(e, str) for e in v) for k, v in patterns.items())
assert isinstance(objects_per_report, int)
assert objects_per_report >= 0
major, minor = self.get_schema_version()
if major != io.schema.LATEST.major:
raise IncompatibleSchema(major, minor)
# A dictionary of object list names and tuples containing a SELECT
# statement and the list of its parameters, returning IDs of the
# objects to fetch.
obj_list_queries = {
# JOIN selecting objects with IDs matching the patterns
obj_list_name: [
f"SELECT id FROM UNNEST(?) AS id\n" \
f"UNION DISTINCT\n" \
f"SELECT {obj_list_name}.id AS id " \
f"FROM {obj_list_name} " \
f"INNER JOIN UNNEST(?) AS id_pattern " \
f"ON {obj_list_name}.id LIKE id_pattern\n",
[
bigquery.ArrayQueryParameter(
None, "STRING", ids.get(obj_list_name, [])
),
bigquery.ArrayQueryParameter(
None, "STRING", patterns.get(obj_list_name, [])
)
]
]
for obj_list_name in io.schema.LATEST.tree if obj_list_name
}
# Add referenced parents if requested
if parents:
def add_parents(obj_list_name):
"""Add parent IDs to query results"""
obj_name = obj_list_name[:-1]
query = obj_list_queries[obj_list_name]
for child_list_name in io.schema.LATEST.tree[obj_list_name]:
add_parents(child_list_name)
child_query = obj_list_queries[child_list_name]
query[0] += \
f"UNION DISTINCT\n" \
f"SELECT {child_list_name}.{obj_name}_id AS id " \
f"FROM {child_list_name} " + \
"INNER JOIN (\n" + \
textwrap.indent(child_query[0], " " * 4) + \
") USING(id)\n"
query[1] += child_query[1]
for obj_list_name in io.schema.LATEST.tree[""]:
add_parents(obj_list_name)
# Add referenced children if requested
if children:
def add_children(obj_list_name):
"""Add child IDs to query results"""
obj_name = obj_list_name[:-1]
query = obj_list_queries[obj_list_name]
for child_list_name in io.schema.LATEST.tree[obj_list_name]:
child_query = obj_list_queries[child_list_name]
child_query[0] += \
f"UNION DISTINCT\n" \
f"SELECT {child_list_name}.id AS id " \
f"FROM {child_list_name} " + \
"INNER JOIN (\n" + \
textwrap.indent(query[0], " " * 4) + \
f") AS {obj_list_name} ON " \
f"{child_list_name}.{obj_name}_id = " \
f"{obj_list_name}.id\n"
child_query[1] += query[1]
add_children(child_list_name)
for obj_list_name in io.schema.LATEST.tree[""]:
add_children(obj_list_name)
# Fetch the data
obj_num = 0
data = io.new()
for obj_list_name, query in obj_list_queries.items():
query_parameters = query[1]
query_string = \
f"SELECT * FROM {obj_list_name} INNER JOIN (\n" + \
textwrap.indent(query[0], " " * 4) + \
") USING(id)\n"
LOGGER.debug("Query string: %s", query_string)
LOGGER.debug("Query params: %s", query_parameters)
job_config = bigquery.job.QueryJobConfig(
query_parameters=query_parameters,
default_dataset=self.dataset_ref)
query_job = self.client.query(query_string, job_config=job_config)
obj_list = None
for row in query_job:
if obj_list is None:
obj_list = []
data[obj_list_name] = obj_list
obj_list.append(Client._unpack_node(dict(row.items())))
obj_num += 1
if objects_per_report and obj_num >= objects_per_report:
assert io.schema.is_valid_latest(data)
yield data
obj_num = 0
data = io.new()
obj_list = None
if obj_num:
assert io.schema.is_valid_latest(data)
yield data
c.KubeSpawner.init_containers.append(ip_block_container)
if get_config('debug.enabled', False):
c.JupyterHub.log_level = 'DEBUG'
c.Spawner.debug = True
extra_config = get_config('hub.extraConfig', {})
if isinstance(extra_config, str):
from textwrap import indent, dedent
msg = dedent("""
hub.extraConfig should be a dict of strings,
but found a single string instead.
extraConfig as a single string is deprecated
as of the jupyterhub chart version 0.6.
The keys can be anything identifying the
block of extra configuration.
Try this instead:
hub:
extraConfig:
myConfig: |
{}
This configuration will still be loaded,
but you are encouraged to adopt the nested form
which enables easier merging of multiple extra configurations.
""")
print(msg.format(indent(extra_config, ' ' * 10).lstrip()), file=sys.stderr)
extra_config = {'deprecated string': extra_config}
for key, config_py in sorted(extra_config.items()):
print("Loading extra config: %s" % key)
exec(config_py)
def _execute_nomad_job(job, previews, params, movie_file_path):
import nomad
import zlib
import json
preview_ids = [
preview["id"] for preview in previews if preview["extension"] == "mp4"
]
input_bytes = zlib.compress(bytes(json.dumps(preview_ids), "utf-8"))
input_string = base64.b64encode(input_bytes).decode("ascii")
bucket_prefix = config.FS_BUCKET_PREFIX
params = {
"version": "1",
"bucket_prefix": bucket_prefix,
"output_filename": Path(movie_file_path).name,
"output_key": file_store.make_key("playlists", job["id"]),
"input": input_string,
"width": params.width,
"height": params.height,
"fps": params.fps,
"FS_BACKEND": config.FS_BACKEND,
}
# Add object storage information
if config.FS_BACKEND == "s3":
params.update({
"S3_ENDPOINT": config.FS_S3_ENDPOINT,
"AWS_DEFAULT_REGION": config.FS_S3_REGION,
"AWS_ACCESS_KEY_ID": config.FS_S3_ACCESS_KEY,
"AWS_SECRET_ACCESS_KEY": config.FS_S3_SECRET_KEY,
})
elif config.FS_BACKEND == "swift":
params.update({
"OS_USERNAME": config.FS_SWIFT_USER,
"OS_PASSWORD": config.FS_SWIFT_KEY,
"OS_AUTH_URL": config.FS_SWIFT_AUTHURL,
"OS_TENANT_NAME": config.FS_SWIFT_TENANT_NAME,
"OS_REGION_NAME": config.FS_SWIFT_REGION_NAME,
})
# don't use 'app.config' because the webapp doesn't use this variable,
# only the rq worker does.
nomad_job = os.getenv("JOB_QUEUE_NOMAD_PLAYLIST_JOB", "zou-playlist")
nomad_host = os.getenv("JOB_QUEUE_NOMAD_HOST", "zou-nomad-01.zou")
data = json.dumps(params).encode("utf-8")
payload = base64.b64encode(data).decode("utf-8")
ncli = nomad.Nomad(host=nomad_host, timeout=5)
response = ncli.job.dispatch_job(nomad_job, payload=payload)
nomad_jobid = response["DispatchedJobID"]
while True:
summary = ncli.job.get_summary(nomad_jobid)
task_group = list(summary["Summary"])[0]
status = summary["Summary"][task_group]
if status["Failed"] != 0 or status["Lost"] != 0:
logger.debug("Nomad job %r failed: %r", nomad_jobid, status)
out, err = _get_nomad_job_logs(ncli, nomad_jobid)
out = textwrap.indent(out, "\t")
err = textwrap.indent(err, "\t")
raise Exception("Job %s is 'Failed' or 'Lost':\nStatus: "
"%s\nerr:\n%s\nout:\n%s" %
(nomad_jobid, status, err, out))
if status["Complete"] == 1:
logger.debug("Nomad job %r: complete", nomad_jobid)
break
# there isn't a timeout here but python rq jobs have a timeout. Nomad
# jobs have a timeout too.
time.sleep(1)
# fetch movie from object storage
with open(movie_file_path, "wb") as movie_file:
for chunk in file_store.open_movie("playlists", job["id"]):
movie_file.write(chunk)
dedented_text = textwrap.dedent(sample_text)
print("Dedented:")
print(dedented_text)
# 控制段落宽度
dedented_text = textwrap.dedent(sample_text).strip()
for width in [45, 60]:
print("{} Columns:\n".format(width))
print(textwrap.fill(dedented_text, width=width)) # 注意fill()有一个控制width的参数
print()
# Indenting Blocks
dedented_text = textwrap.dedent(sample_text)
wrapped = textwrap.fill(dedented_text, width=50)
wrapped += "\n\nSecond paragraph after a blank line"
final = textwrap.indent(wrapped, "> ") # indent这个method允许对目标使用一个字符作为前置,每行都用
print("Quoted block:\n")
print(final)
print()
# 对于一行中含有偶数字符的前方加"EVEN "
def should_indent(line):
print("Indent {!r}?".format(line))
return len(line.strip()) % 2 == 0
dedented_text = textwrap.dedent(sample_text)
wrapped = textwrap.fill(dedented_text, width=50)
final = textwrap.indent(wrapped, "EVEN ", predicate=should_indent)
def indent4(a_string):
return textwrap.indent(a_string, ' '*4)
result = run_script(script_name=script_key, config_pathname=config_file)
# Inform the user when the script is finished, mostly for
# completeness.
core.error.ifas_info("FINISH! The script `{script}` using the "
"configuration file `{config}` has been completed."
.format(script=script_key, config=config_file))
# All done.
return result
if (__name__ == '__main__'):
# Creating the argpaser. The textwrap is for formatting.
parser = argparse.ArgumentParser(
description=textwrap.indent(textwrap.dedent(
'''
Execute an IfA-Smeargle script function. Change the
script by name and the configuration parameters by the
configuration file."
'''), prefix=' '),
epilog=textwrap.indent(textwrap.dedent(
'''
The script to show all available scripts is:
python -m ifa_smeargle script_special_list_scripts none
'''), prefix=' '),
formatter_class=argparse.RawDescriptionHelpFormatter)
# Adding the three required arguments for all pipelines. These
# are always constant requirements and thus are positional.
parser.add_argument("script_key",
help=("the name of the script function to be run"),
type=str)
parser.add_argument("configuration_path",
"""
# TODO: remove this warning once Tui is delta-driven
# https://github.com/cylc/cylc-flow/issues/3527
from textwrap import indent
from urwid import html_fragment
from cylc.flow.option_parsers import CylcOptionParser as COP
from cylc.flow.terminal import cli_function
from cylc.flow.tui import (TUI)
from cylc.flow.tui.app import (TuiApp, TREE_EXPAND_DEPTH
# ^ a nasty solution
)
__doc__ += indent(TUI, ' ')
def get_option_parser():
parser = COP(
__doc__,
argdoc=[('REG', 'Workflow name')],
# auto_add=False, NOTE: at present auto_add can not be turned off
color=False)
parser.add_option(
'--display',
help=('Specify the display technology to use.'
' "raw" for interactive in-terminal display.'
' "html" for non-interactive html output.'),
action='store',
def debug_print_env(path):
import textwrap
logger.info('PATH:\n{}'.format(textwrap.indent(path.replace(os.pathsep, '\n'), ' ')))
