def longest_path(parameters, exclusions):
"""
Traces all subdirectories of provided commandline paths
using MaxWidth object
Args:
:parameters (list): list of all sys.argv parameters supplied with --sum
:exclusions (ExcludedTypes object): types to exclude
Returns:
width (integer), number of characters in longest path
"""
mp = MaxWidth() # max path object
abspath = absolute_paths(parameters)
container = []
max_width = 0
for i in parameters:
try:
paths = sp_linecount(i, abspath, exclusions.types)
width = mp.calc_maxpath(paths)
max_width = width if (width > max_width) else max_width
container.extend(paths)
except TypeError:
stdout_message(message='Provided path appears to be invalid',
prefix='WARN')
sys.exit(exit_codes['EX_OSFILE']['Code'])
return max_width, container
def display_exclusions(expath, exdirpath, offset_spaces=default_width):
"""
Show list of all file type extensions which are excluded
from line total calculations
"""
tab = '\t'.expandtabs(15)
offset = '\t'.expandtabs(offset_spaces)
# numbering
div = cm.bpl + ')' + rst
adj = 10
try:
ex = ProcessExclusions(expath)
exclusions = ex.exclusions
stdout_message(message='File types excluded from line counts:', indent=offset_spaces + adj)
for index, ext in enumerate(exclusions):
print('{}{}{:>3}{}'.format(offset, tab, index + 1, div + ' ' + ext))
sys.stdout.write('\n')
return True
except OSError as e:
fx = inspect.stack()[0][3]
stdout_message(message=f'{fx}: Error: {e}. ', prefix='WARN')
return False
def multiprocessing_main(valid_paths, max_width, _threshold, wspace,
exclusions, debug):
"""
Execute Operations using concurrency (multi-process) model
Args:
:valid_paths (list): list of filesystem paths (str) filtered for binary,
other uncountable objects
:max_width (int): width of output pattern, sans line count total column
:_threshold (int): high line count threshold value (highlighted objects)
:wspace (bool): when True, omit whitespace lines from count (DEFAULT: False)
:exclusions (ex object): instance of ExcludedTypes
:debug (boot): debug flag
"""
def debug_messages(flag, paths):
if flag:
stdout_message('Objects contained in container directories:',
prefix='DEBUG')
for i in paths:
print(i)
def queue_generator(q, p):
"""Generator which offloads a queue before full"""
while p.is_alive():
p.join(timeout=1)
while not q.empty():
yield q.get(block=False)
global q
q = multiprocessing.Queue()
processes, results = [], []
debug_messages(debug, valid_paths)
# maximum cores is 4 due to i/o contention single drive systems
cores = 4 if cpu_cores() >= 4 else cpu_cores()
equal_lists = [sorted(x) for x in split_list(valid_paths, cores)]
for i in equal_lists:
t = multiprocessing.Process(target=mp_linecount,
args=(i, exclusions.types, wspace))
processes.append(t)
t.start()
results.extend([x for x in queue_generator(q, t)])
if debug:
print('Completed: list {}'.format(get_varname(i))) # show progress
print_results(results, _threshold, max_width)
if debug:
export_json_object(results, logging=False)
stdout_message(message='Num of objects: {}'.format(len(results)))
return 0
def _configure_add(expath, exdirpath, startpt):
"""
Add exclusions and update runtime constants
Returns:
Success | Failure, TYPE: bool
"""
tab4 = '\t'.expandtabs(4)
loop = True
adj = 4
vert_adj = 2
try:
with open(expath) as f1:
exclusions = [x.strip() for x in f1.readlines()]
while loop:
width = _init_screen(starting_row=startpt + vert_adj)
pattern_width = section_header('add', width, tabspaces=16)
offset_chars = int((width / 2) - (pattern_width / 2)) + adj
offset = '\t'.expandtabs(offset_chars)
display_exclusions(expath, exdirpath, offset_chars)
# query user input for new exclusions
msg = 'Enter file extension types separated by commas [done]: '
offset_msg = '\t'.expandtabs(
int((width / 2) - (pattern_width / 2) + adj * 2))
response = input(f'{offset_msg}{msg}')
if not response:
loop = False
sys.stdout.write('\n')
return True
else:
add_list = response.split(',')
# add new extensions to existing
exclusions.extend(
[x if x.startswith('.') else '.' + x for x in add_list])
# write out new exclusions config file
with open(expath, 'w') as f2:
f2.writelines([x + '\n' for x in exclusions])
except OSError:
stdout_message(
message='Unable to modify local config file located at {}'.format(
expath),
prefix='WARN')
return False
def main_menupage(expath, exdirpath):
"""
Displays main configuration menu jump page and options
"""
def menu():
border = bbl
icolor = bbl
bar = border + '''
________________________________________________________________________________
'''
#bar = '\n' + ('_' * 80) + '\n'
pattern_width = len(bar)
width, srow = _init_screen()
offset = '\t'.expandtabs(int((width / 2) - (pattern_width / 2)))
_menu = (border + bar + rst + '''\n\n
''' + bdwt + PACKAGE + rst + ''' configuration main menu:\n\n
''' + icolor + 'a' + rst +
''') Add file type to exclusion list\n\n
''' + icolor + 'b' + rst +
''') Remove file type from exclusion list\n\n
''' + icolor + 'c' + rst +
''') Set high line count threshold (''' + acct +
'highlight' + rst + ''' file objects)\n\n
''' + icolor + 'd' + rst + ''') quit\n
''' + border + bar + rst)
for line in _menu.split('\n'):
print('{}{}'.format(offset, line))
return offset, srow
loop = True
tab8 = '\t'.expandtabs(8)
while loop:
offset, verticalstart = menu()
answer = input('\n{}{}Choose operation [quit]: '.format(offset,
tab8)).lower()
sys.stdout.write('\n')
if not answer or answer == 'd':
return True
elif answer in ['a', 'b', 'c']:
condition_map(answer, expath, exdirpath, verticalstart)
else:
stdout_message(message='You must provide a letter a, b, c, or d',
indent=16,
prefix='INFO')
sys.stdout.write('\n')
def _configure_rewrite(expath, newlist):
"""
Rewrite existing exclusion list on local filesystem with
modified contents from _configure operation
Return:
Succsss || Failure, TYPE: bool
"""
try:
# write new exclusion list to local disk
with open(expath, 'w') as f1:
list(filter(lambda x: f1.write(x.strip() + '\n'), newlist))
except OSError as e:
fx = inspect.stack()[0][3]
stdout_message(
f'{fx}: Problem writing new file type exclusion list: {expath}: {e}',
prefix='WARN')
return False
return True
def precheck(user_exfiles, user_exdirs, debug):
"""
Runtime Dependency Check
"""
_os_configdir = os.path.join(modules_location(), 'config')
_os_ex_fname = os.path.join(_os_configdir,
local_config['EXCLUSIONS']['EX_FILENAME'])
_os_dir_fname = os.path.join(_os_configdir,
local_config['EXCLUSIONS']['EX_DIR_FILENAME'])
_config_dir = local_config['CONFIG']['CONFIG_DIR']
if debug:
stdout_message(
f'_os_configdir: {_os_configdir}: system py modules location',
'DBUG')
stdout_message(
f'_os_ex_fname: {_os_ex_fname}: system exclusions.list path',
'DBUG')
stdout_message(
f'_os_dir_fname: {_os_dir_fname}: system directories.list file path',
'DBUG')
stdout_message(
f'_configdir: {_config_dir}: user home config file location',
'DBUG')
try:
# check if exists; copy
if not os.path.exists(_config_dir):
os.makedirs(_config_dir)
# cp system config file to user if user config files absent
if os.path.exists(_os_ex_fname) and os.path.exists(_os_dir_fname):
if not os.path.exists(user_exfiles):
copyfile(_os_ex_fname, user_exfiles)
if not os.path.exists(user_exdirs):
copyfile(_os_dir_fname, user_exdirs)
except OSError:
fx = inspect.stack()[0][3]
logger.exception(
'{}: Problem installing user config files. Exit'.format(fx))
return False
return True
def debug_messages(flag, paths):
if flag:
stdout_message('Objects contained in container directories:',
prefix='DEBUG')
for i in paths:
print(i)
def init_cli():
ex_files = local_config['EXCLUSIONS']['EX_EXT_PATH']
ex_dirs = local_config['EXCLUSIONS']['EX_DIR_PATH']
# process commandline args
parser = argparse.ArgumentParser(add_help=False)
try:
args, unknown = options(parser)
except Exception as e:
help_menu()
stdout_message(str(e), 'ERROR')
sys.exit(exit_codes['E_BADARG']['Code'])
# validate configuration files
if precheck(ex_files, ex_dirs, args.debug):
_ct_threshold = set_hicount_threshold(
) or local_config['OUTPUT']['COUNT_HI_THRESHOLD']
if len(sys.argv) == 1 or args.help:
help_menu()
sys.exit(exit_codes['EX_OK']['Code'])
elif args.version:
package_version()
elif args.exclusions:
display_exclusions(ex_files, ex_dirs)
elif args.configure:
main_menupage(ex_files, ex_dirs)
elif len(sys.argv) == 2 and (sys.argv[1] != '.'):
help_menu()
sys.exit(exit_codes['EX_OK']['Code'])
elif args.sum:
ex = ExcludedTypes(ex_path=str(Path.home()) +
'/.config/xlines/exclusions.list')
container = create_container(args.sum)
abspath = absolute_paths(container)
if args.debug:
stdout_message(f'xlines command line option parameter detail',
prefix='DEBUG')
print('\targs.sum: {}'.format(args.sum))
print('\n\tsys.argv contents:\n')
for i in sys.argv:
print('\t\to {}'.format(i))
print(f'\n\tcontainer is:\t{container}')
print(f'\n\tobject "unknown" is:\t{unknown}')
print('\tabspath bool is {}\n'.format(abspath))
print('\tmultiprocess bool is {}\n'.format(args.multiprocess))
if args.multiprocess:
# --- run with concurrency --
width, paths = longest_path(container, ex)
paths = remove_excluded(args.exclude, paths)
multiprocessing_main(paths, width, _ct_threshold, args.whitespace,
ex, args.debug)
elif not args.multiprocess:
io_fail = []
tcount, tobjects = 0, 0
width, paths = longest_path(container, ex)
paths = remove_excluded(args.exclude, paths)
print_header(width)
count_width = local_config['OUTPUT']['COUNT_COLUMN_WIDTH']
for path in paths:
try:
inc = linecount(path, args.whitespace)
highlight = acct if inc > _ct_threshold else cm.aqu
tcount += inc # total line count
tobjects += 1 # increment total number of objects
# truncation
lpath, fname = os.path.split(path)
if (len(path) + BUFFER * 2) > width:
cutoff = (len(path) + BUFFER * 2) - width
else:
cutoff = 0
tab = '\t'.expandtabs(width - len(lpath) - len(fname) -
count_width + BUFFER)
# with color codes added
if cutoff == 0:
lpath = text + lpath + rst
else:
lpath = text + os.path.split(
path)[0][:len(lpath) - cutoff -
BUFFER] + rst + arrow
tab = '\t'.expandtabs(width - len(lpath) - len(fname) +
count_width + BUFFER + cut_corr)
tab4 = '\t'.expandtabs(4)
fname = highlight + fname + rst
# incremental count formatting
ct_format = acct if inc > _ct_threshold else bwt
# format tabular line totals with commas
output_str = f'{tab4}{lpath}{div}{fname}{tab}{ct_format}{"{:,}".format(inc):>10}{rst}'
print(output_str)
if args.debug:
print(tab4 * 2 + 'lpath is {}'.format(lpath))
print(tab4 * 2 + 'fname is {}\n'.format(fname))
except Exception:
io_fail.append(path)
continue
print_footer(tcount, tobjects, width)
if args.debug:
tab4 = '\t'.expandtabs(4)
stdout_message(
f'cli screen columns variable, width: {cm.bdwt}{width}{cm.rst}',
prefix='DBUG')
print('\n' + tab4 + 'Skipped file objects:\n' + tab4 +
('-' * (width + count_width)))
if io_fail:
for file in io_fail:
print(
'\t{}'.format(file)
) # Write this out to a file in /tmp for later viewing
else:
print('\tNone')
sys.stdout.write('\n')
sys.exit(exit_codes['EX_OK']['Code'])
else:
stdout_message('Dependency check fail %s' % json.dumps(args, indent=4),
prefix='AUTH',
severity='WARNING')
sys.exit(exit_codes['E_DEPENDENCY']['Code'])
failure = """ : Check of runtime parameters failed for unknown reason.
Please ensure you have both read and write access to local filesystem. """
logger.warning(failure +
'Exit. Code: %s' % sys.exit(exit_codes['E_MISC']['Code']))
print(failure)
def precheck(user_exfiles, user_exdirs, debug):
"""
Runtime Dependency Checks: postinstall artifacts, environment
"""
def set_environment():
lang = 'undefined'
if os.getenv('LANG') is None:
lang = '{}export LANG=en_US.UTF-8{}'.format(yl, rst)
elif 'UTF-8' not in os.getenv('LANG'):
lang = '{}export LANG=$LANG.UTF-8{}'.format(yl, rst)
return lang
# local user configuration: excluded file types
_os_configdir = os.path.join(modules_location(), 'config')
_ex_fname = local_config['EXCLUSIONS']['EX_FILENAME']
_os_ex_fname = os.path.join(_os_configdir, _ex_fname)
# local user configuration: excluded directories
_dir_fname = local_config['EXCLUSIONS']['EX_DIR_FILENAME']
_os_dir_fname = os.path.join(_os_configdir, _dir_fname)
_config_dir = local_config['CONFIG']['CONFIG_DIR']
_language = set_environment()
_environment_setup = 'fail' if 'UTF-8' in _language else 'success'
if debug:
tab = '\t'.expandtabs(16)
stdout_message(
f'_os_configdir: {_os_configdir}: system py modules location',
'DBUG')
stdout_message(
f'_os_ex_fname: {_os_ex_fname}: system exclusions.list path',
'DBUG')
stdout_message(
f'_os_dir_fname: {_os_dir_fname}: system directories.list file path',
'DBUG')
stdout_message(
f'_configdir: {_config_dir}: user home config file location',
'DBUG')
stdout_message(
f'Environment setup status: {_environment_setup.upper()}')
if _environment_setup.upper() == 'FAIL':
_env = _environment_setup.upper()
msg = f'Environment setting is {_env}. Add the following code in your .bashrc file'
stdout_message('{}: {}'.format(msg, _language))
try:
# check if exists; copy
if not os.path.exists(_config_dir):
os.makedirs(_config_dir)
# cp system config file to user if user config files absent
if os.path.exists(_os_ex_fname) and os.path.exists(_os_dir_fname):
if not os.path.exists(user_exfiles):
copyfile(_os_ex_fname, user_exfiles)
if not os.path.exists(user_exdirs):
copyfile(_os_dir_fname, user_exdirs)
# debian-style installation paths
elif os.path.exists(
os.path.join('usr/local/lib/xlines/config', _ex_fname)):
if not os.path.exists(user_exfiles):
copyfile(
os.path.join('usr/local/lib/xlines/config', _ex_fname),
user_exfiles)
if not os.path.exists(user_exdirs):
copyfile(
os.path.join('usr/local/lib/xlines/config', _dir_fname),
user_exdirs)
except OSError:
fx = inspect.stack()[0][3]
logger.exception(
'{}: Problem installing user config files. Exit'.format(fx))
return False
return True
def _configure_hicount(expath, exdirpath, startpt):
"""
User update high line count threshold persisted on local filesystem
Returns:
Success || Failure, TYPE: bool
"""
def _exit(loop_break):
leave = input('Exit? [quit]')
if not leave or 'q' in leave:
loop_break = False
return loop_break
tab4 = '\t'.expandtabs(4)
tab13 = '\t'.expandtabs(13)
loop = True
adj = 12
vert_adj = 2
local_linecount_file = local_config['CONFIG']['HI_THRESHOLD_FILEPATH']
try:
width = _init_screen(starting_row=startpt + vert_adj)
pattern_width = section_header('threshold', width, tabspaces=14)
offset_chars = int((width / 2) - (pattern_width / 2))
offset = '\t'.expandtabs(offset_chars)
while loop:
if os.path.exists(local_linecount_file):
with open(local_linecount_file) as f1:
threshold = int(f1.read().strip())
stdout_message(
message='Current high line count threshold: {}{}{}'.format(bdwt, threshold, rst),
indent=offset_chars + adj
)
answer = input(f'{tab4}{offset}{tab13}Enter high line count threshold [{threshold}]: ')
try:
if not answer:
stdout_message(
f'High line count threshold remains {threshold}',
prefix='INFO',
indent=offset_chars + adj
)
loop = False
return mainmenu_return(offset)
elif 'q' in answer:
loop = False
return mainmenu_return(offset)
elif type(int(answer)) is int:
# rewrite threshold file on local filesystem
with open(local_linecount_file, 'w') as f1:
f1.write(str(answer) + '\n')
stdout_message(
message='high line count threshold set to {}'.format(answer),
prefix='ok',
indent=offset_chars + adj
)
loop = False
return mainmenu_return(offset)
else:
stdout_message(
message='You must enter an integer number',
prefix='INFO',
indent=offset_chars + adj
)
except ValueError:
pass
except OSError:
fx = inspect.stack()[0][3]
logger.exception(f'{fx}: Problem reading local hicount threshold file. Abort')
return False
return True
def _configure_remove(expath, exdirpath, startpt):
"""
Remove file type extension from exclusion list
Return:
Succsss || Failure, TYPE: bool
"""
delay_seconds = 2
tabspaces = 4
tab4 = '\t'.expandtabs(tabspaces)
loop = True
adj = tabspaces * 2
vert_adj = 2
try:
# open current file type exclusions
with open(expath) as f1:
f2 = [x.strip() for x in f1.readlines()]
while loop:
width = _init_screen(starting_row=startpt + vert_adj)
pattern_width = section_header('delete', width, tabspaces=14)
offset_chars = int((width / 2) - (pattern_width / 2)) + tabspaces
offset = '\t'.expandtabs(offset_chars)
display_exclusions(expath, exdirpath, offset_chars)
answer = input(offset + tab4 + 'Pick the number of a file type to remove [done]: ')
try:
if not answer:
loop = False
sys.stdout.write('\n')
return True
elif int(answer) in range(1, len(f2) + 1):
# correct for f2 list index
answer = int(answer) - 1
# remove entry selected by user
deprecated = f2[answer]
f2.pop(int(answer))
if not _configure_rewrite(expath, f2):
return False
# Acknowledge removal
if str(answer) in f2:
stdout_message(
message='Failure to remove {} - reason unknown'.format(f2[answer]),
indent=offset_chars + adj,
prefix='FAIL'
)
sleep(delay_seconds)
else:
max_index = len(f2)
stdout_message(
message=f'You must pick a number between 1 and {max_index}',
prefix='WARN',
indent=offset_chars + adj
)
sleep(delay_seconds)
except ValueError:
continue
except OSError:
stdout_message(
message='Unable to modify local config file located at {}'.format(expath),
prefix='WARN')
return False