def show_loc(proc):
print('Lines of code copied from each module:')
out = []
for mod, val in sorted(proc.loaded.items()):
out.append([(mod + ':'),
str(sum(val)) + ' loc', 'in',
str(len(val)) + ' functions'])
auto_cols(out, space=2)
def get_network_usage(interval=1, samples=4, verbose=0):
'''Return total network usage in Bytes / second, adds up rxkB/s and txkB/s columns from sar
Requires: sudo apt install sysstat'''
out = quickrun('sar', '-n', 'DEV', interval, samples, verbose=verbose)
if verbose:
auto_cols(map(str.split, out[-3:]))
out = [line for line in out if line.startswith('Average:')]
out = flatten([line.split()[4:6] for line in out[1:]])
return int(sum(map(float, out)) * 1024)
def all_disk_usage(wait=2, reps=4, verbose=0, ignore_links=True):
'''Return total i/o for all devices in Bytes / second
ignore_links will ignore loop and dm-? devs for total'''
ready = False
total = 0
table = dict()
rep = -1
for line in quickrun('nice',
'iostat',
'-d',
wait,
reps + 1,
verbose=verbose):
if verbose >= 2:
print(rep, line)
if not line:
continue
if line.startswith('Device'):
ready = True
rep += 1
continue
if ready and rep > 0:
# Skip the first rep because it contains bad data
line = line.split()
dev = line[0]
usage = sum(map(float, line[2:4]))
table.setdefault(dev, []).append(usage)
if ignore_links and (dev.startswith('dm-')
or dev.startswith('loop')):
continue
total += usage
if verbose:
out = [['Device'] + ['Rep ' + str(rep + 1)
for rep in range(reps)] + ['', 'Average MB/s']]
for dev in sorted(table.keys()):
out.append([dev] + list(map(int, table[dev])))
out[-1] += ['=', int(avg(table[dev]))]
out = [out[0]] + list(sorted_array(out[1:], reverse=True))
auto_cols(out, manual={-3: 2, -2: 2})
return int(total / reps * 1024)
def main():
args = get_args()
mymod = universe.load_mod(args.mymod)
modname = universe.get_modname(mymod)
members = dict(inspect.getmembers(mymod))
output_name = args.output_name.rstrip('.py')
if output_name == 'same':
output_name = modname
filenames = args.filenames
if args.onefile:
filename = filenames[0]
output_filename = os.path.join(args.output_dir,
os.path.basename(filename))
else:
output_filename = os.path.join(args.output_dir, output_name + '.py')
if samepath(output_filename, *filenames):
error("Cannot overwrite self!")
mkdir('/tmp/Star_Wrangler')
print(modname, 'functions:')
auto_cols([(name, str(func).replace('\n', ' '))
for name, func in sorted(members.items())],
crop={1: 200})
print("\n")
# Generate dict of required functions and their code
filename = 'output' # Default filename
functions = odict() # Dict of function names to code
file_functions = dict() # Dict filenames to function dicts
file_imports = dict() # Dict of filenames to import lists
proc = Processor(mymod,
max_level=args.max_level,
follow=not args.nofollow,
ignore=args.ignore)
if args.functions:
functions = proc.process_words(args.functions)
else:
for filename in filenames:
line_nums = set()
imports = []
with open(filename) as f:
source = f.read()
# Go through file source and remove "import modname"
terms = args.modnames if args.modnames else [
modname,
]
for num, line in universe.scrape_imports(source):
# if modname in line:
if any(name in line for name in terms):
print(line)
line_nums.add(num)
imports.append(re.sub('.*import ', '', line))
if not imports:
warn("Could not find any common imports in",
filename,
"for module name:",
modname,
delay=0)
else:
file_imports[filename] = imports
imports = [re.sub(' as .*$', '', word) for word in imports]
sub = proc.get_code_words(filename, imports)
file_functions[filename] = sub
for func in sub:
if func not in functions:
functions[func] = sub[func]
if not functions:
print("No functions discovered")
sys.exit(0)
print('\n' * 5)
print("Done. Outputting to file:", output_filename)
print('#' * 80, '\n')
output = []
def owl(*args):
"Output write lines"
output.append(' '.join(args))
# Header
if not args.onefile:
owl("#!/usr/bin/python3")
owl(shared.HEADER.strip())
if args.onefile:
owl(shared.HEADER.replace('file', 'section').strip())
# Write import lines to top of the file
owl('')
func_names = functions.keys()
for line in sorted(proc.imports.values(), key=len):
words = re.sub('.* import ', '', line).split()
if not any([word in func_names for word in words]):
owl(line)
else:
print("Skipping locally referenced import line:", line)
if proc.imports:
owl("\n")
# Functions
for code in reversed(functions.values()):
owl('\n'.join(code))
owl('\n')
#Aliases
for line in set(proc.aliases.values()):
owl(line)
# Put it all together and output
if args.onefile:
ie = max(line_nums)
source = source.splitlines()
for num in line_nums:
source.pop(num)
output = source[:ie] + ['#' * 80] + output + ['#' * 80, '', ''
] + source[ie:]
output.append("\n'''\n" + shared.FOOTER.strip())
output.append(time.strftime('%Y-%m-%d', time.localtime()))
output.append("'''")
with open(output_filename, 'w') as out:
for line in output:
out.write(line + '\n')
# List imports for each file for copy paste
# https://www.python.org/dev/peps/pep-0008/#imports
print('\n')
for filename, words in file_imports.items():
print(filename, "functions to be imported:", '\n')
for line in indenter(', '.join(words),
header='from ' + output_name + ' import ',
wrap=80):
print(line.rstrip(','))
print('\n')
if len(proc.loaded) > 1:
print('\n')
show_loc(proc)
# Finished
print('\n')
print(rfs(os.path.getsize(output_filename)), 'of code saved to',
output_filename)
qrun('chmod', '+x', output_filename)
print("\nCopy to script directory with:")
print(
'cp', output_filename,
os.path.join(os.path.dirname(filename),
os.path.basename(output_filename)))
def print_help(self, show_type=True, wrap=-4, tab=' '):
'''Print a custom help message using only ArgMaster args
show_type = append the variable type expected after each optional argument.
--arg <int> <int> will expect 2 integers after the arg
wrap = word wrap instead of using full terminal. 0 = Terminal width
sort = sort alphabetically. Positional variables are never sorted.
To sort individual groups, add a special key: group.sortme = True
Warning: If your variable is not in a group, it will not be shown!'''
if self.description:
print('\n' + self.description)
if self.usage:
name = os.path.basename(sys.argv[0])
print('\n' + "Usage:", name, self.usage)
final = []
width = 0 # Max width of the variables column
for group in self.groups:
out = []
for args in group['args']:
msg = args['msg']
if msg == SUPPRESS:
continue
alias = args['alias']
if show_type:
if args['typ'] and args['typ'] != bool:
if args['typ'] == list:
typ = '...'
else:
typ = '<' + str(args['typ']).replace(
'class ', '')[2:-2] + '>'
alias += ' ' + typ
if len(alias) > width:
width = len(alias)
out.append([alias, msg])
if group['sortme'] is not None:
sortme = group['sortme']
else:
sortme = self.sortme
if sortme:
# Sort the group while mainting the order of positional arguments at the top
positionals = [
out.pop(line) for line in range(len(out) - 1, -1, -1)
if out[line][0].startswith('<')
]
out.sort()
out = list(reversed(positionals)) + out
final.append(out)
for index, out in enumerate(final):
group = self.groups[index]
if out:
print(self.newline, end='')
for line, _ in enumerate(out):
out[line][0] = tab + out[line][0].ljust(width)
if 'title' in group:
print(group['title']) # .rstrip(':') + ':')
if 'description' in group:
auto_cols([[tab + group['description']]], wrap=wrap)
auto_cols(out, wrap=wrap)
print()
def help_parser(parser, show_type=True, sortme=True, wrap=100, tab=' '):
"Standalone version of help that only needs an argparse parser"
'''Print a custom help message from the ArgumentParser class
show_type = append the variable type expected after each optional argument.
--arg <int> <int> will expect 2 integers after the arg
wrap = word wrap instead of using full terminal. 0 = disable
sort = sort alphabetically. Positional variables are never sorted.
To sort individual groups, add a special key: group['sortme'] = True
Warning: If your variable is not in a group, it will not be shown!
'''
def alias(action):
"Return string representing the variable for the leftmost column"
if action.option_strings:
text = ', '.join(action.option_strings)
else:
text = action.dest
if show_type:
if not action.option_strings:
text = '<' + text + '>'
if action.type and action.type != bool:
if action.type == list:
typ = '...'
else:
typ = '<' + str(action.type).replace('class ',
'')[2:-2] + '>'
if isinstance(action.nargs, int):
count = action.nargs
else:
count = 1
text += ' ' + ' '.join([typ] * count)
return text
final = []
width = 0
for group in parser._action_groups: # pylint: disable=W0212
out = []
for action in group._group_actions: # pylint: disable=W0212
if action.help and action.help == SUPPRESS:
continue
msg = list(action.help)
msg[0] = msg[0].title()
variable = alias(action)
out.append([variable, ''.join(msg)])
if len(variable) > width:
width = len(variable)
if sortme or group.__dict__.get('sortme', False):
# Sort the group while mainting the order of positional arguments at the top
positionals = [
out.pop(line) for line in range(len(out) - 1, -1, -1)
if out[line][0].startswith('<')
]
out.sort()
out = list(reversed(positionals)) + out
final.append(out)
for group in parser._action_groups: # pylint: disable=W0212
out = final.pop(0)
if out:
for line, _ in enumerate(out):
out[line][0] = tab + out[line][0].ljust(width)
print()
if group.title:
print(group.title + ':')
if group.description:
auto_cols([[tab + group.description]])
auto_cols(out, wrap=wrap)
def show_args(uargs):
"Show the arguments returned"
if not uargs:
return
auto_cols(sorted([[key, repr(val)] for key, val in (vars(uargs)).items()]))
print()
def main():
if len(sys.argv) < 3:
print(
"Usage: ./star_namer.py <module_file> <script_filenames...> --options..."
)
sys.exit(1)
args = [\
["exclude", "", list],
"Exclude any function found in these module names",
["local", '', bool],
"Don't list any functions outside of file",
["actual", '', bool, False],
"Print the actual module each function is found in",
]
positionals = [\
["module"],
"Module name to search through functions",
["scripts", '', list],
"Python scripts to scan through",
]
#Load the args:
args = easy_parse(args, positionals)
filenames = args.scripts
for name in filenames:
if not os.path.exists(name):
error(name, "does not exist")
mymod = universe.load_mod(args.module)
modname = universe.get_modname(mymod)
modvars = universe.get_members(args.module)
print("Found defined variables in module", modname + ':')
out = [['Name:', 'Module:', 'Function:']]
for name, func in modvars.items():
out.append([name, universe.get_modname(inspect.getmodule(func)), func])
auto_cols(out)
print("\n")
for filename in filenames:
functions = universe.scrape_wildcard(filename, modvars)
if len(filenames) > 1:
print('\n')
eprint(filename + ':', '\n', v=2)
if functions:
out = dict()
for name, func in functions.items():
mod = universe.get_modname(inspect.getmodule(func))
if mod in args.exclude:
continue
if args.local and mod != modname:
continue
mod = mod if args.actual else modname
out.setdefault(mod, []).append(name)
for mod, funcs in out.items():
header = 'from ' + mod + ' import '
for line in indenter(', '.join(funcs), header=header, wrap=80):
print(line.rstrip(','))
else:
print("<no functions found>")
