def scrape_wildcard(filename, modvars):
"Get variables imported from module in wild * import"
unused = []
cmd = [get_pylint(), filename
] + '--output-format=json --disable=W0312'.split()
for item in json_loader('\n'.join(qrun(cmd, hidewarning=True))):
if item['message-id'] == 'W0614':
line = item['message']
line = re.sub('^Unused import ', '', line)
line = re.sub(' from wildcard import$', '', line)
unused.append(line)
if not unused:
warn("Pylint did not detect any unused imports for",
filename,
'\n' * 2,
"Make sure there is a:",
'\n',
"from __modulename__ import *",
'\n',
"line in the file",
delay=0)
return {}
out = dict()
for name in set(modvars) - set(unused):
if not name.startswith('__'):
func = modvars[name]
if not isinstance(func, types.ModuleType):
out[name] = modvars[name]
return out
def get_undefined(source):
"Run souce code through pylint and get each undefined variable"
data = qrun(
get_pylint(),
'--from-stdin stdin --output-format=json --disable=W0312'.split(),
stdin=source,
hidewarning=True)
for item in json_loader('\n'.join(data)):
idc = item['message-id']
if idc == 'E0602': # undefined variable:
yield item
def get_pylint():
"Check pylint version and return file path"
if CACHED.pylint:
return CACHED.pylint
pylint = shared.PYLINT
if not os.path.exists(pylint):
warn("Pylint path does not exist:", pylint)
print("Please install: https://www.pylint.org/#install")
print("and then set the correct path in shared.py")
sys.exit(1)
version = qrun(pylint, '--version')
version = search_list('pylint', version,
getfirst=True).split()[-1].split('.')[:2]
if list(map(int, version)) < [2, 4]:
error("Pylint must be at least version 2.4")
CACHED.pylint = pylint
return pylint
def undefined(func):
"Run code through pylint and get all undefined variables"
cache = CACHED.undefined # Dict of functions to undefined words
if func in cache:
return cache[func]
code = '\n'.join(getsource(func))
data = qrun(
get_pylint(),
'--from-stdin stdin --output-format=json --disable=W0312'.split(),
stdin=code,
hidewarning=True)
words = set()
for item in json_loader('\n'.join(data)):
idc = item['message-id']
msg = item['message']
# print(msg)
if idc == 'E0602': # undefined variable:
word = re.sub('Undefined variable ', '', msg).strip("'")
words.add(word)
return words
def main():
args = get_args()
mymod = universe.load_mod(args.mymod)
modname = universe.get_modname(mymod)
members = dict(inspect.getmembers(mymod))
output_name = args.output_name.rstrip('.py')
if output_name == 'same':
output_name = modname
filenames = args.filenames
if args.onefile:
filename = filenames[0]
output_filename = os.path.join(args.output_dir,
os.path.basename(filename))
else:
output_filename = os.path.join(args.output_dir, output_name + '.py')
if samepath(output_filename, *filenames):
error("Cannot overwrite self!")
mkdir('/tmp/Star_Wrangler')
print(modname, 'functions:')
auto_cols([(name, str(func).replace('\n', ' '))
for name, func in sorted(members.items())],
crop={1: 200})
print("\n")
# Generate dict of required functions and their code
filename = 'output' # Default filename
functions = odict() # Dict of function names to code
file_functions = dict() # Dict filenames to function dicts
file_imports = dict() # Dict of filenames to import lists
proc = Processor(mymod,
max_level=args.max_level,
follow=not args.nofollow,
ignore=args.ignore)
if args.functions:
functions = proc.process_words(args.functions)
else:
for filename in filenames:
line_nums = set()
imports = []
with open(filename) as f:
source = f.read()
# Go through file source and remove "import modname"
terms = args.modnames if args.modnames else [
modname,
]
for num, line in universe.scrape_imports(source):
# if modname in line:
if any(name in line for name in terms):
print(line)
line_nums.add(num)
imports.append(re.sub('.*import ', '', line))
if not imports:
warn("Could not find any common imports in",
filename,
"for module name:",
modname,
delay=0)
else:
file_imports[filename] = imports
imports = [re.sub(' as .*$', '', word) for word in imports]
sub = proc.get_code_words(filename, imports)
file_functions[filename] = sub
for func in sub:
if func not in functions:
functions[func] = sub[func]
if not functions:
print("No functions discovered")
sys.exit(0)
print('\n' * 5)
print("Done. Outputting to file:", output_filename)
print('#' * 80, '\n')
output = []
def owl(*args):
"Output write lines"
output.append(' '.join(args))
# Header
if not args.onefile:
owl("#!/usr/bin/python3")
owl(shared.HEADER.strip())
if args.onefile:
owl(shared.HEADER.replace('file', 'section').strip())
# Write import lines to top of the file
owl('')
func_names = functions.keys()
for line in sorted(proc.imports.values(), key=len):
words = re.sub('.* import ', '', line).split()
if not any([word in func_names for word in words]):
owl(line)
else:
print("Skipping locally referenced import line:", line)
if proc.imports:
owl("\n")
# Functions
for code in reversed(functions.values()):
owl('\n'.join(code))
owl('\n')
#Aliases
for line in set(proc.aliases.values()):
owl(line)
# Put it all together and output
if args.onefile:
ie = max(line_nums)
source = source.splitlines()
for num in line_nums:
source.pop(num)
output = source[:ie] + ['#' * 80] + output + ['#' * 80, '', ''
] + source[ie:]
output.append("\n'''\n" + shared.FOOTER.strip())
output.append(time.strftime('%Y-%m-%d', time.localtime()))
output.append("'''")
with open(output_filename, 'w') as out:
for line in output:
out.write(line + '\n')
# List imports for each file for copy paste
# https://www.python.org/dev/peps/pep-0008/#imports
print('\n')
for filename, words in file_imports.items():
print(filename, "functions to be imported:", '\n')
for line in indenter(', '.join(words),
header='from ' + output_name + ' import ',
wrap=80):
print(line.rstrip(','))
print('\n')
if len(proc.loaded) > 1:
print('\n')
show_loc(proc)
# Finished
print('\n')
print(rfs(os.path.getsize(output_filename)), 'of code saved to',
output_filename)
qrun('chmod', '+x', output_filename)
print("\nCopy to script directory with:")
print(
'cp', output_filename,
os.path.join(os.path.dirname(filename),
os.path.basename(output_filename)))