def send_and_execute_file(self, commands, blocking): generated_uuid = str(uuid.uuid4()) path_to_sh_file = os.path.join(os.path.dirname(__file__), 'temp', generated_uuid + '.sh') path_to_remote_sh_file = os.path.join(self.project_path, generated_uuid + '.sh') with open(path_to_sh_file, 'w') as shell_file: shell_file.write("#!/bin/sh\n") for line in commands: shell_file.write("%s\n" % line) shell_file.write("rm -rf %s\n" % path_to_remote_sh_file) # Copy Shell file with jobs to execute with ShellHandler(self.host, self.username, self.password, self.port, self.tunnel_host, self.tunnel_username, self.tunnel_password, self.tunnel_port, self.use_tunnel, 10025, self.ssh_key_path) as handler: scp = SCPClient(handler.get_ssh_client().get_transport()) scp.put(path_to_sh_file, remote_path=b'%s' % str.encode(path_to_remote_sh_file)) # Delete local file subprocess.run(['rm', '-rf', path_to_sh_file]) # Make remote file executable self.execute_command('chmod +x %s' % path_to_remote_sh_file) # Execute. We need the variable order_needed as it distighushes between two separate possible execution methods logger.info("Execute command: %s" % path_to_remote_sh_file) if blocking: with ShellHandler(self.host, self.username, self.password, self.port, self.tunnel_host, self.tunnel_username, self.tunnel_password, self.tunnel_port, self.use_tunnel, 10026, self.ssh_key_path) as handler: out = handler.execute_file(path_to_remote_sh_file, True) logger.debug('Output: %s' % ' '.join(out)) return out else: thread = JobSubmissionThread( path_to_remote_sh_file, self.host, self.username, self.password, self.port, self.tunnel_host, self.tunnel_username, self.tunnel_password, self.tunnel_port, self.use_tunnel, self.ssh_key_path) thread.start() return None
def __init__(self): self.username = HPC['username'] self.password = HPC['password'] self.host = HPC['host'] self.port = HPC['port'] self.queue = HPC['queue'] self.node_properties = HPC['node_properties'] self.tunnel_username = HPC['ssh_tunnel_username'] self.tunnel_password = HPC['ssh_tunnel_password'] self.tunnel_host = HPC['ssh_tunnel_host'] self.tunnel_port = HPC['ssh_tunnel_port'] self.logger = logging.getLogger(__name__) self.ssh = ShellHandler(self.host, self.username, self.password, self.port, self.tunnel_host, self.tunnel_username, self.tunnel_password, self.tunnel_port)
def run(self): with ShellHandler(self.host, self.username, self.password, self.port, self.tunnel_host, self.tunnel_username, self.tunnel_password, self.tunnel_port, self.use_tunnel, 10020) as handler: out, err = handler.execute_file(self.remote_file, False) logger.debug(out) logger.debug(err)
def execute_command(self, command, ignore_errors=False): logger.info('Execute command: %s' % command) with ShellHandler(self.host, self.username, self.password, self.port, self.tunnel_host, self.tunnel_username, self.tunnel_password, self.tunnel_port, self.use_tunnel, 10024) as handler: (stdout, stderr) = handler.execute(command) logger.debug('Output: %s' % ' '.join(stdout)) logger.debug('ErrorOut: %s' % ' '.join(stderr)) if stderr and not ignore_errors: raise Exception('Error in executing command %s! Error: %s.' % (command, ','.join(stderr))) return stdout
def copy_project_tar(self): with ShellHandler(self.host, self.username, self.password, self.port, self.tunnel_host, self.tunnel_username, self.tunnel_password, self.tunnel_port, self.use_tunnel, 10023) as handler: scp = SCPClient(handler.get_ssh_client().get_transport()) # Copy plugin scp.put('tmp.tar.gz', remote_path=b'~') # Untar plugin self.execute_command('mkdir %s' % self.project_path) self.execute_command('tar -C %s -xvf ~/tmp.tar.gz' % self.project_path) # Delete tar self.execute_command('rm -f ~/tmp.tar.gz')
def get_error_log(self, job): with ShellHandler(self.host, self.username, self.password, self.port, self.tunnel_host, self.tunnel_username, self.tunnel_password, self.tunnel_port, self.use_tunnel, 10022) as handler: sftp_client = handler.get_ssh_client().open_sftp() plugin_execution_output_path = os.path.join(self.log_path, str(job.plugin_execution.id)) try: remote_file = sftp_client.open(os.path.join(plugin_execution_output_path, str(job.id)+'_err.txt')) except FileNotFoundError: return ['File Not Found'] output = [] try: for line in remote_file: output.append(line.strip()) finally: remote_file.close() return output
def copy_plugin(self, plugin): with ShellHandler(self.host, self.username, self.password, self.port, self.tunnel_host, self.tunnel_username, self.tunnel_password, self.tunnel_port, self.use_tunnel, 10023) as handler: scp = SCPClient(handler.get_ssh_client().get_transport()) # Copy plugin scp.put(plugin.get_full_path_to_archive(), remote_path=b'~') try: self.delete_plugin(plugin) except Exception: pass # Untar plugin self.execute_command('mkdir %s/%s' % (self.plugin_path, str(plugin))) self.execute_command('tar -C %s/%s -xvf %s' % (self.plugin_path, str(plugin), plugin.get_name_of_archive())) # Delete tar self.execute_command('rm -f ~/%s' % (plugin.get_name_of_archive()))
class HPCHandler(object): plugin_path = '~/bin/plugins' project_path = '/home/uni08/jgrabow1/bin/projects' tools_path = '~/bin/tools' bsub_output_path = '~/bin/output' home_folder = '/usr/users/jgrabow1' def __init__(self): self.username = HPC['username'] self.password = HPC['password'] self.host = HPC['host'] self.port = HPC['port'] self.queue = HPC['queue'] self.node_properties = HPC['node_properties'] self.tunnel_username = HPC['ssh_tunnel_username'] self.tunnel_password = HPC['ssh_tunnel_password'] self.tunnel_host = HPC['ssh_tunnel_host'] self.tunnel_port = HPC['ssh_tunnel_port'] self.logger = logging.getLogger(__name__) self.ssh = ShellHandler(self.host, self.username, self.password, self.port, self.tunnel_host, self.tunnel_username, self.tunnel_password, self.tunnel_port) def add_parameters_to_command(self, path_to_script, parameters): command = path_to_script for parameter in parameters: command += parameter["value"]+" " return command def create_execution_command(self, plugin, project, parameters): path_to_execute_script = "%s/%s/execute.sh " % (self.plugin_path, str(plugin)) # Make execution script executable self.execute_command("chmod +x %s" % path_to_execute_script) # Add parameters command = self.add_parameters_to_command(path_to_execute_script, parameters) # Substitute stuff return string.Template(command).safe_substitute({ 'db_user': DATABASES['mongodb']['USER'], 'db_password': DATABASES['mongodb']['PASSWORD'], 'db_database': DATABASES['mongodb']['NAME'], 'db_hostname': DATABASES['mongodb']['HOST'], 'db_port': DATABASES['mongodb']['PORT'], 'db_authentication': DATABASES['mongodb']['AUTHENTICATION_DB'], 'url': project.url, 'plugin_path': os.path.join(self.plugin_path, str(plugin)) }) def create_install_command(self, plugin, parameters): path_to_install_script = "%s/%s/install.sh " % (self.plugin_path, str(plugin)) # Make execution script executable self.execute_command("chmod +x %s" % path_to_install_script) # Add parameters command = self.add_parameters_to_command(path_to_install_script, parameters) return string.Template(command).substitute({ 'plugin_path': os.path.join(self.plugin_path, str(plugin)) }) def execute_install(self, plugin, parameters): # Build parameter for install script. command = self.create_install_command(plugin, parameters) self.execute_command(command) def execute_plugin(self, plugin, project, parameters, execution, revisions): org_command = self.create_execution_command(plugin, project, parameters) path_to_repo = os.path.join(self.project_path, project.name, 'base') # Create plugin execution plugin_execution = PluginExecution(project=project, plugin=plugin) plugin_execution.save() # if plugin operates on rep or other level, execute it once if plugin.abstraction_level == 'repo' or plugin.abstraction_level == 'other': command = string.Template(org_command).safe_substitute({ 'path': path_to_repo }) # create command execution self.send_bsub_command(command, plugin, project, None, plugin_execution) elif plugin.abstraction_level == 'rev': # if plugin operated on abstraction level, check execution type all_revisions = self.get_all_revisions(project) revisions_to_execute_plugin_on = [] if execution == 'all': revisions_to_execute_plugin_on = all_revisions elif execution == 'rev': if len(revisions.split(",")) == 1: revisions_to_execute_plugin_on.append(revisions) else: # If only some revisions (comma-separated list) need to be executed, create path and add it to list for revision in revisions.split(","): revisions_to_execute_plugin_on.append(revision) elif execution == 'new': # Get all jobs that were executed with this plugin on this project jobs = plugin.get_all_jobs_for_project(project) job_revision_hashes = [job.revision_hash for job in jobs] # Go through all paths: If the revision was already processed by a job, it is not new, so exclude it for revision in all_revisions: if revision not in job_revision_hashes: revisions_to_execute_plugin_on.append(revision) elif execution == 'error': # Get all revisions on which this plugin failed (in some revisions) on this project. Important: # if the plugin on revision X failed in first run, but worked on revision X in the second it is not longer # marked as failing for this revision revisions = self.get_revisions_for_failed_plugins([plugin], project) for revision in revisions: revisions_to_execute_plugin_on.append(revision) # Create command for revision in revisions_to_execute_plugin_on: # Create command command = string.Template(org_command).safe_substitute({ 'path': path_to_repo, 'revision': revision }) # create command execution self.send_bsub_command(command, plugin, project, revision, plugin_execution) def send_bsub_command(self, command, plugin, project, revision, plugin_execution): (bsub_command, output_path, error_path) = self.generate_bsub_command(command, plugin, project) output = self.execute_command(bsub_command) job_id = None for line in output: if line.startswith("Job"): job_id_match = re.match(r"(\w*) <([0-9]*)>", line) job_id = job_id_match.group(2) # Make db_user and db_password in bsub_command (if they are in the command) anonymous. bsub_command = bsub_command.replace(DATABASES['mongodb']['USER'], 'mongodbUser') bsub_command = bsub_command.replace(DATABASES['mongodb']['PASSWORD'], 'mongoPassword') if plugin.abstraction_level == 'rev': job = Job(job_id=job_id, plugin_execution=plugin_execution, status='WAIT', output_log=output_path, error_log=error_path, submission_string=bsub_command, revision_hash=revision) else: job = Job(job_id=job_id, plugin_execution=plugin_execution, status='WAIT', output_log=output_path, error_log=error_path, submission_string=bsub_command) job.save() def update_job_information(self, jobs): # first check via bjobs output, if the job is found: use this status, if not check if there is an error log # if there are errors in it: there must be an error => state is exit for job in jobs: # set if output exists or not out_exists, error_exists = self.does_output_files_exist(job) job.output_file_exists = out_exists job.error_file_exists = error_exists self.update_single_job_information(job) def update_single_job_information(self, job): # If job is already in exit state, it is fine if job.status in ['DONE', 'EXIT']: return if job.output_file_exists is False or job.error_file_exists is False: job.status = 'WAIT' job.save() return error_log = self.get_error_log(job) if not error_log: job.status = 'DONE' else: job.status = 'EXIT' job.save() def get_history(self, job): out = self.execute_command('bhist -l %s' % job.job_id) out = [line.strip() for line in out] if 'No matching job found' in out[0]: return [] return out def does_output_files_exist(self, job): if job.status in ['DONE', 'EXIT']: return True, True error_path = job.error_log.replace('~', self.home_folder) output_path = job.output_log.replace('~', self.home_folder) error_exists = True try: self.execute_command('head %s' % error_path) except Exception: error_exists = False out_exists = True try: self.execute_command('head %s' % output_path) except Exception: out_exists = False return out_exists, error_exists def get_output_log(self, job): sftp_client = self.ssh.get_ssh_client().open_sftp() try: remote_file = sftp_client.open(job.output_log.replace('~', self.home_folder)) except FileNotFoundError: return None output = [] try: for line in remote_file: output.append(line.strip()) finally: remote_file.close() return output def get_error_log(self, job): error_path = job.error_log sftp_client = self.ssh.get_ssh_client().open_sftp() try: remote_file = sftp_client.open(error_path.replace('~', self.home_folder)) except FileNotFoundError: return None output = [] try: for line in remote_file: output.append(line.strip()) finally: remote_file.close() return output def generate_bsub_command(self, command, plugin, project): jobname = project.name+"_"+str(plugin) plugin_dependencies = plugin.requires.all() generated_uuid = str(uuid.uuid4()) output_path = os.path.join(self.bsub_output_path, generated_uuid+"_out.txt") error_path = os.path.join(self.bsub_output_path, generated_uuid+"_error.txt") bsub_command = 'bsub -q %s -o %s -e %s -J "%s" ' % ( self.queue, output_path, error_path, jobname ) if plugin_dependencies: bsub_command += "-w '" for plugin in plugin_dependencies: bsub_command += "ended(%s_%s) && " % (project.name, str(plugin)) bsub_command = bsub_command[:-4] bsub_command += "' " for node_property in self.node_properties: bsub_command += "-R %s " % node_property bsub_command += "%s" % command return (bsub_command, output_path, error_path) def get_base_directory(self, project): return os.path.join(self.project_path, project.name, 'base') def get_all_revisions(self, project): path = os.path.join(self.project_path, project.name, 'rev') self.execute_command('cd %s' % self.get_base_directory(project)) self.execute_command('git log --pretty=oneline --all --reverse > revisions.txt') sftp_client = self.ssh.get_ssh_client().open_sftp() remote_file = sftp_client.open(os.path.join(self.get_base_directory(project), 'revisions.txt')) revisions = [] try: for line in remote_file: if line: revisions.append(line.strip().split(" ")[0]) finally: remote_file.close() self.execute_command('rm -rf %s' % os.path.join(self.get_base_directory(project), 'revisions.txt')) return revisions def get_revisions_for_failed_plugins(self, plugins, project): revisions = [] for plugin in plugins: revisions.extend(plugin.get_revision_hashes_of_failed_jobs_for_project(project)) return revisions def is_base_repo_existent(self, project): out = self.execute_command('ls %s' % os.path.join(self.project_path, project.name)) folders = [] if out: folders = out[0].strip().split(" ") if 'base' in folders: return True return False def prepare_project(self, project, plugins, execution, revisions): # Create project folder if not existent self.execute_command('mkdir %s' % os.path.join(self.project_path, project.name), ignore_errors=True) # Execute git clone into base folder if self.is_base_repo_existent(project): self.execute_command('cd %s' % self.get_base_directory(project)) self.execute_command('git pull') else: self.execute_command('git clone %s %s' % (project.url, self.get_base_directory(project))) def install_plugin(self, plugin, parameters): self.copy_plugin(plugin) self.execute_install(plugin, parameters) def delete_plugin(self, plugin): self.execute_command('rm -rf %s/%s' % (self.plugin_path, str(plugin))) def copy_plugin(self, plugin): scp = SCPClient(self.ssh.get_ssh_client().get_transport()) # Copy plugin scp.put(plugin.get_full_path_to_archive(), remote_path=b'~') # Untar plugin try: self.delete_plugin(plugin) except Exception: pass self.execute_command('mkdir %s/%s' % (self.plugin_path, str(plugin))) self.execute_command('tar -C %s/%s -xvf %s' % (self.plugin_path, str(plugin), plugin.get_name_of_archive())) # Delete tar self.execute_command('rm -f ~/%s' % (plugin.get_name_of_archive())) def execute_command(self, command, ignore_errors=False): print("Execute command: %s" % command) (stdin, stdout, stderr) = self.ssh.execute(command) if stderr and not ignore_errors: raise Exception('Error in executing command %s! Error: %s.' % (command, ','.join(stderr))) return stdout