def check_ssh_task_status(job_config, schedule_time, pid_file, settings): job_id = job_config.get('id') remote_server_id = job_config.get('remote_server_id') remote_server_config = remote_servers.get(remote_server_id) remote_server_ssh_key = remote_server_config.get('ssh_details').get('ssh_key') remote_server_username = remote_server_config.get('ssh_details').get('username') remote_server_host = remote_server_config.get('ssh_details').get('host') get_pid_id_cmd = ['ssh', '-i', remote_server_ssh_key, '@'.join([remote_server_username, remote_server_host]), 'cat', pid_file] get_pid_id_proc = subprocess.Popen(get_pid_id_cmd, stdout=subprocess.PIPE) pid_id = get_pid_id_proc.communicate()[0].strip() exit_code = get_pid_id_proc.returncode if exit_code != 0: set_succeeded(job_id, schedule_time, db_name=settings.metadata) settings.logger.info('%s %s has succeeded' % (job_id, schedule_time)) return else: check_running_process_cmd = ['ssh', '-i', remote_server_ssh_key, '@'.join([remote_server_username, remote_server_host]), 'ps', '-p', pid_id] check_if_process_still_running = subprocess.Popen(check_running_process_cmd) ps_output = check_if_process_still_running.communicate() process_running = check_if_process_still_running.returncode if process_running == 0: return else: set_failed(job_id, schedule_time, db_name=settings.metadata) settings.logger.warning('%s %s has failed' % (job_id, schedule_time)) return
def launch_ssh_task(job_config, schedule_time, settings): unique_job_id = job_id_generator(job_config.get('id'), schedule_time) remote_server_id = job_config.get('remote_server_id') remote_server_config = remote_servers.get(remote_server_id) remote_action_script = job_config.get('remote_action').get('script') remote_action_script_input_args = job_config.get('remote_action').get('args') remote_action_script_args = [parse_schedule_time(arg, schedule_time) for arg in remote_action_script_input_args] local_action_cmd, uid = ssh_task_cmd_generator(remote_action_script, remote_action_script_args, remote_server_config, unique_job_id, settings) get_pid_id_proc = subprocess.Popen(local_action_cmd, stdout=subprocess.PIPE) exit_code = get_pid_id_proc.returncode #connection check if exit_code == 255 or exit_code == 127: return False, None if local_action_cmd: return True, uid else: return False, None