def __delete(self, *, table):
while True:
records = self.__record(table=table)
self.__display_links_for_edit(table=table, records=records)
if not records:
Display.info(what='No record available to delete for {0} '.format(self.name), info=' [QUITING]')
break
Display.info('Type Comma or space separated row no to delete ', info=' [ q/Q to Quit]')
confirmation = input()
if confirmation in ['q', 'Q']:
Display.info(what='You chose to Quit ', info=' [ Leaving]')
break
else:
# extracting all row no from confirmation
rows = []
for element in confirmation.split():
rows.extend(element.split(','))
# Now deleting one row at a time
for row in rows:
if row.isdigit() and int(row) < len(records):
# add row deletion logic
row = int(row)
self.delete_row(table=config.TABLES['links'], records=records, row=row)
Display.info(what='Row "{0}"" Deletion '.format(row), info=' [ SUCCESS ]')
print()
else:
Display.warning(what='Row "{0}" deletion : Not Digit or beyond the record length '.format(row), info=' [ FAILED ]')
def delete(self, *pargs, **kwargs):
# Taking Conformation from the user interactively if -y flag was not provided
try:
args = kwargs['args']
except KeyError as keyerror:
msg = 'Keyword argument {0} in class: {1}, function: {2}'.format(
keyerror, self.__class__.__name__, self.delete.__name__)
print(Display.fail(msg, info='[ MISSING ]'))
return
else:
if not args.yes:
while True:
confirmation = input('Confirm to delete database: y/n?\n')
if confirmation in ['n', 'N']:
return
elif confirmation in ['y', 'Y']:
break
# closing the connection
self.connection.close()
try:
os.remove(config.DB_NAME)
except FileNotFoundError:
print(Display.fail(config.DB_NAME, info='[ DOES NOT EXIST ]'))
else:
Display.success(what='Deleting DB: {0} '.format(config.DB_NAME),
info=' [ SUCCESS ]')
def edit(self):
self.__opt_flags_check(['links', 'shells'])
self.__opt_flags_check(['add', 'delete', 'delete_all'])
if self.args.add:
# Adding
if self.args.links:
self.__add(table=config.TABLES['links'], commands=config.SUPPORTED_BROWSERS)
else:
pass
elif self.args.delete:
# deleting
if self.args.links:
self.__delete(table=config.TABLES['links'])
else:
pass
elif self.args.delete_all:
# deleting all entry of link/shell apps for session name
if self.args.links:
Display.title(title='Deleting entries from table "{0}" for Session "{1}"'.format(config.TABLES['links'], self.name))
sql_string = 'delete from {table} where session = ?'.format(table=config.TABLES['links'])
self.db.update(table=config.TABLES['links'], sql_string=sql_string, values=(self.name, ), commit=True)
else:
pass
Display.success('Deleting All ', info=' [ SUCCESS ]')
def __generate_performance_data(self):
hostname = os.popen('echo $HOSTNAME').read()
for cluster in settings.CLUSTERS:
if cluster.lower() in hostname:
self.cluster = cluster
break
else:
self.cluster = 'UNKNOWN-CLUSTER'
Display.title(title='Analyzing for {0}'.format(self.cluster))
for dir_tag in self.args.dir_tag:
dir_abspath, tag = self.__extract_dir_tag(dir_tag=dir_tag)
log_files = []
if self.args.recursive:
for (dirname, dirshere, fileshere) in os.walk(dir_abspath):
log_files.extend(
glob.glob(
os.path.join(
dirname,
settings.DEFAULT_ANALYZE_FILE_SUFFIX)))
else:
log_files = [
log_file for log_file in glob.glob(
os.path.join(dir_abspath,
settings.DEFAULT_ANALYZE_FILE_SUFFIX))
]
for log_file in log_files:
self.__add_performance_data(log_file=log_file, tag=tag)
def __add(self, *, table, commands):
Display.title(title='Adding to table "{0}" for Session "{1}"'.format(table, self.name))
while True:
Display.info('INPUT FORMAT: [ "{0}" args1 args2, "{0}" args1 args2 ... ] '.format('|'.join(commands.keys())), info=' [ q/Q to delete ]')
user_input = input().split(',')
if user_input[0].strip() in ['q', 'Q']:
Display.info(what='You have chosen to Quit ', info=' [ QUITING ]')
break
else:
sql_string = 'insert into {0} values(?,?,?)'.format(table)
for entry in user_input:
command_for, *args = [x.strip() for x in entry.split()]
if not command_for in commands.keys():
Display.info(what='Wrong input : {0}, available are '.format(command_for),
info=' [ {0} ]'.format('|'.join(commands.keys())))
else:
for arg in args:
self.db.update(
table=table,
sql_string=sql_string,
values=(self.name, commands[command_for], arg),
commit=True,
create_if_required=False
)
Display.success('Addition of {0} : {1} '.format(command_for, arg), info=' [ SUCCESS ]')
def submit(cls, *, directory, job):
current_working_directory = os.getcwd()
os.chdir(directory)
os.system('{0} {1}'.format(cls.sbatch, job))
os.chdir(current_working_directory)
Display.info(what='{0} '.format(os.path.join(directory, job)),
info=' [ Submitted ]',
fill='-')
def query(self, *, table, query_string, values=None):
if self.__table_exists(table=table):
if values:
self.cursor.execute(query_string, values)
else:
self.cursor.execute(query_string)
else:
Display.fail(what='Query on table "{0}" '.format(table),
info='[ NOT EXIST ]')
def open(self):
self.__opt_flags_check(['links', 'shells'])
if self.args.links:
query_string = 'select * from {0} where session = ?'.format(config.TABLES['links'])
self.db.query(table=config.TABLES['links'], query_string=query_string, values=[self.name])
for (name, command, args) in self.db.cursor.fetchall():
call(command + args, shell=True)
else:
Display.info(what='Opening all saved shell process ', info=' [ NOT IMPLEMENTED YET ]')
def tables(self, *args, **kwargs):
Display.title(title='List of Available Tables')
all_tables = self.get_tables()
data = []
for table in all_tables:
self.cursor.execute("PRAGMA table_info('{0}')".format(table[0]))
columns = self.cursor.fetchall()
data.append((table[0], [col[1] for col in columns]))
Display.dataframe(headers=['Table Name', 'Columns'], rows=data)
def __table_exists(self, *, table, trace=False):
all_tables = self.get_tables()
if all_tables and table in all_tables[0]:
if trace:
Display.info(what='Table {0} in database. '.format(table),
info='[ EXIST ]')
return True
else:
return False
def create_table(self, *, table):
if table in config.DB_TABLES and not self.__table_exists(table=table,
trace=True):
self.cursor.execute(config.DB_TABLES[table])
Display.success(what='table {} added to database '.format(table),
info='[ SUCCESS ]')
else:
raise Exception(
'No table creation recipe is avaibale in {0} for {1}'.format(
config.__name__, table))
def __load_template(self):
Display.title(title='Loading Job Template for {0}'.format(
self.__class__.__name__))
try:
file = open(
os.path.join(settings.TEMPLATE_DIRECTORY,
self.__class__.__name__.lower()), 'r')
except FileNotFoundError:
raise FileNotFoundError(
'Template not available for Cluster: {0}.'.format(
self.__class__.__name__))
else:
Display.info(what='Template ', info=' [ Found ]', fill='-')
self.template = file.read()
def __inspect(self):
hostname = os.popen('echo $HOSTNAME').read()
# Testing ...
# hostname = 'tegner'
for cluster in settings.CLUSTERS:
if cluster.lower() in hostname:
Display.title(
title='Inspecting Submitted Jobs on {0}'.format(cluster)
)
# setting the cluster
self.cluster = cluster
# call inspect method from cluster to inspect the submitted jobs
getattr(clusters, self.cluster).inspect()
break
def set_simg(self, target, simg):
# simg will be empty string if not provided by the user
if simg:
try:
self.replace_by_arg(target,
self.params[target].format(simg))
except KeyError:
Display.error(
what='Singularity Image for Cluster {0} '.format(
self.__class__.__name__),
info=' [ Not Supported ]',
fill='-')
self.replace_by_arg(target, '')
else:
Display.error(what='Singularity Image for Cluster {0} '.format(
self.__class__.__name__),
info=' [ not provided ]',
fill='-')
self.replace_by_arg(target, '')
def drop(self, *, args):
if args.table:
if self.__table_exists(table=args.table, trace=False):
if not args.yes:
Display.warning(what='Dropping table {0} '.format(
args.table),
info='[ Confirm y/Y? ]')
confirmation = input()
if not confirmation in ['y', 'Y']:
return
self.cursor.execute('drop table {0}'.format(args.table))
else:
Display.warning(what='Dropping table {0} '.format(args.table),
info='[ NOT EXIST ]')
elif args.all:
tables = self.get_tables()
for tables in tables:
self.cursor.execute('drop table {0}'.format(tables[0]))
def __submit(self):
hostname = os.popen('echo $HOSTNAME').read()
for cluster in settings.CLUSTERS:
if cluster.lower() in hostname:
Display.title(title='Submitting Job to {0}'.format(cluster))
# setting the cluster
self.cluster = cluster
for directory in self.args.directories:
dir_abspath = os.path.abspath(directory)
if self.args.recursive:
Display.info(what='{0} '.format(
os.path.abspath(directory)),
info=' [ Submitting Recursively ]',
fill='-')
self.__recursive_submit(directory=dir_abspath)
else:
Display.info(what='{0} '.format(dir_abspath),
info=' [ No Recursive Submitting]',
fill='-')
contents = os.listdir(dir_abspath)
for content in contents:
if content.endswith(self.args.suffix):
getattr(clusters, self.cluster).submit(
directory=dir_abspath, job=content)
break
def __save_links(self):
Display.title(title='Saving links to : {0}'.format(self.name))
cmd = "/usr/bin/osascript -e 'tell application \"{0}\"' -e 'get URL of every tab of every window' -e 'end tell'"
links = {}
total_links = 0
for browser in config.SUPPORTED_BROWSERS:
pipe = Popen(cmd.format(browser), stdout=PIPE, shell=True)
links[browser] = pipe.communicate()[0].strip().split(b',')
total_links += len(links[browser])
# saving all opened tabs along with session name, open command
sql_string = 'insert into {0} values(?,?,?)'.format(config.TABLES['links'])
# variable related to diplaying progress bar
progress_bar_step = 0
percentage_step_length = 100 // total_links
from time import sleep
for browser in config.SUPPORTED_BROWSERS:
browser_links = links.get(browser, None)
if browser_links:
for link in browser_links:
if link:
# Checking whether there is any record for the session, command and args
query_string = 'select * from {0} where session=? and command=? and links=?'.format(config.TABLES['links'])
self.db.query(table=config.TABLES['links'], query_string=query_string, values=(self.name, config.SUPPORTED_BROWSERS[browser], link.strip().decode()))
query_length = len(self.db.cursor.fetchall())
if query_length == 1:
pass
elif query_length > 1:
raise AssertionError('Multiple same entry for the Session: "{0}"'.format(self.name))
else:
# save unique entry
self.db.update(
table=config.TABLES['links'],
sql_string=sql_string,
values=(self.name, config.SUPPORTED_BROWSERS[browser], link.strip().decode()),
commit=True,
create_if_required=True
)
progress_bar_step += 1
Display.progress_bar(
total=total_links,
step=progress_bar_step,
completed=str(percentage_step_length * progress_bar_step) + '%'
)
# For testing
# sleep(.2)
Display.progress_bar(total=total_links, step=progress_bar_step + 1, completed='100%')
print()
def __update_template(self):
Display.title(title='Updating Template for {0}'.format(
self.__class__.__name__))
templte_backup = self.template[:]
for nnodes in range(self.args.min_nodes, self.args.max_nodes + 1,
1):
for ntasks in range(self.args.min_ntasks_per_node,
self.args.max_ntasks_per_node + 1, 1):
self.computed_params['$nodes$'] = str(nnodes)
self.computed_params['$ntasks-per-node$'] = str(ntasks)
self.computed_params['$MPI_NP$'] = str(nnodes * ntasks)
for (target, action) in self.targets.items():
try:
method = object.__getattribute__(self, action)
except AttributeError:
print(
'Class {0} does not have handler method named : {1}'
.format(self.__class__.__name__, action))
else:
try:
Display.info(
what='{0} Replaced by '.format(target),
info=' [ {0} ]'.format(
getattr(self.args, target[1:-1])),
fill='-')
method(target, getattr(self.args,
target[1:-1]))
except AttributeError:
Display.info(
what='{0} Replaced by '.format(target),
info=' [ {0} ]'.format(
self.computed_params[target]),
fill='-')
# print('Target {0} will be replaced with value computed from the user input'.format(target))
method(target, self.computed_params[target])
# write the script to the external file
self.__store_run_script(
nodes=self.computed_params['$nodes$'],
np=self.computed_params['$MPI_NP$'])
# reload template for the next run
self.template = templte_backup[:]
def __housekeeping(self):
Display.title(title='Setting Up Working Directory for {0}'.format(
self.__class__.__name__))
try:
workdir = os.path.abspath(self.args.workdir)
except Exception:
raise Exception(
'No default or user provided Working directory was provided...'
)
else:
if workdir == settings.HOME_DIRECTORY:
Display.warning(
what="Using tool's directory as working directory ",
info=' [ Warning ]')
confirmation = input("Type 'y/Y' to proceed? ")
if not confirmation in ['y', 'Y']:
Display.error(what='You chose not to proceed ',
info=' [ ABORTING ]')
os._exit(-1)
if workdir != settings.DEFAULT_WORKDIR:
os.chdir(workdir)
else:
if workdir != settings.DEFAULT_WORKDIR:
os.chdir(workdir)
Display.info(what='{0} '.format(workdir),
info=' [ Set as Working Dirrectory ]',
fill='-')
# Creating Directory for the benchmark
print('\n')
Display.title(title='Creating Benchmark Directory for {0}'.format(
self.__class__.__name__))
self.benchmark_directory = os.path.join(
os.getcwd(),
os.path.join(
self.args.name,
os.path.join(str(date.today()), self.__class__.__name__)))
if os.path.exists(self.benchmark_directory):
contents_here = os.listdir(self.benchmark_directory)
if contents_here:
Display.info(what='{0} '.format(self.benchmark_directory),
info=' [ Directory Not Empty ]',
fill='-')
confirmation = input(
"\nType 'o/O' to override, 'b/b' for backup contents and any other key to quit?\n"
.format(self.benchmark_directory))
if confirmation in ['o', 'O']:
# Remove every thing from the directory
Display.warning(what='Overriding {0} '.format(
self.benchmark_directory),
info=' [ Warning ]',
fill='-')
os.system('rm -rf {0}'.format(
os.path.join(self.benchmark_directory, '*')))
elif confirmation in ['b', 'B']:
Display.title(title='BACKING UP CONTENTS')
Display.info(what='{0} with suffix '.format(
os.path.join(self.benchmark_directory, '*')),
info=' [ "{0}" ]'.format(
settings.DEFAULT_FILE_BACKUP_SUFFIX),
fill='-')
# Storing current working directory to get back
cwd = os.getcwd()
os.chdir(self.benchmark_directory)
for content in sorted(contents_here, reverse=True):
os.rename(
content,
content + settings.DEFAULT_FILE_BACKUP_SUFFIX)
# going back to the working directory as it was
os.chdir(cwd)
else:
Display.error(what='You chose not to proceed ',
info=' [ ABORTING ]')
sys.exit()
else:
os.system('mkdir -p {0}'.format(self.benchmark_directory))
def inspect(cls):
pipe = os.popen('squeue -u {0}'.format(os.environ['USER']))
for line in pipe:
Display.line(line=line)
def __display_links_for_edit(self, table, records):
Display.title(title='Record for Session : {0}, Table: {1}'.format(self.name, table))
Display.dataframe(
headers=['Command', 'Arguments'],
rows=[(cmd, arg) for (sess, cmd, arg) in records]
)
def view(self, *pargs, **kwargs):
if self.args.name:
self.__opt_flags_check(['links', 'shells', 'all'])
if self.args.links:
headers = ['Command', 'Arguments']
Display.title(title='Record for Session : {0} , Table: {1}'.format(self.name, config.TABLES['links']))
records = self.__record(table=config.TABLES['links'])
Display.dataframe(
headers=headers,
rows=[(cmd, arg) for (sess, cmd, arg) in records]
)
elif self.args.shells:
Display.info(what='Viewing all saved shell process ', info=' [ NOT IMPLEMENTED YET ]')
else:
Display.info(what='Viewing all processes ', info=' [ NOT IMPLEMENTED YET ]')
elif self.args.sessions:
self.__opt_flags_check(['links', 'shells', 'all'])
if self.args.links:
tables = ['links']
elif self.args.shells:
Display.info(what='Viewing all session name under table : "shells" ', info=' [ NOT IMPLEMENTED YET ]')
return
else:
tables = config.TABLES.values()
sessions = self.__all_sessions(tables=tables)
Display.dataframe(
headers=['Table Name', 'List of Session'],
rows=sessions
)
else:
self.parser.error(message='Missing argument from [{0}, {1}]'.format('-n/name', '--sessions'))