def check_ssh_task_status(job_config, schedule_time, pid_file, settings):
    job_id = job_config.get('id')

    remote_server_id = job_config.get('remote_server_id')
    remote_server_config = remote_servers.get(remote_server_id)
    remote_server_ssh_key = remote_server_config.get('ssh_details').get('ssh_key')
    remote_server_username = remote_server_config.get('ssh_details').get('username')
    remote_server_host = remote_server_config.get('ssh_details').get('host')

    get_pid_id_cmd = ['ssh', '-i', remote_server_ssh_key, '@'.join([remote_server_username, remote_server_host]), 'cat',
                      pid_file]

    get_pid_id_proc = subprocess.Popen(get_pid_id_cmd, stdout=subprocess.PIPE)
    pid_id = get_pid_id_proc.communicate()[0].strip()
    exit_code = get_pid_id_proc.returncode

    if exit_code != 0:
        set_succeeded(job_id, schedule_time, db_name=settings.metadata)
        settings.logger.info('%s %s has succeeded' % (job_id, schedule_time))
        return
    else:
        check_running_process_cmd = ['ssh', '-i', remote_server_ssh_key,
                                     '@'.join([remote_server_username, remote_server_host]), 'ps', '-p',
                                     pid_id]

        check_if_process_still_running = subprocess.Popen(check_running_process_cmd)
        ps_output = check_if_process_still_running.communicate()
        process_running = check_if_process_still_running.returncode

        if process_running == 0:
            return
        else:
            set_failed(job_id, schedule_time, db_name=settings.metadata)
            settings.logger.warning('%s %s has failed' % (job_id, schedule_time))
            return
def launch_ssh_task(job_config, schedule_time, settings):
    unique_job_id = job_id_generator(job_config.get('id'), schedule_time)

    remote_server_id = job_config.get('remote_server_id')
    remote_server_config = remote_servers.get(remote_server_id)

    remote_action_script = job_config.get('remote_action').get('script')
    remote_action_script_input_args = job_config.get('remote_action').get('args')

    remote_action_script_args = [parse_schedule_time(arg, schedule_time) for arg in remote_action_script_input_args]

    local_action_cmd, uid = ssh_task_cmd_generator(remote_action_script, remote_action_script_args,
                                                   remote_server_config, unique_job_id, settings)

    get_pid_id_proc = subprocess.Popen(local_action_cmd, stdout=subprocess.PIPE)
    exit_code = get_pid_id_proc.returncode

    #connection check
    if exit_code == 255 or exit_code == 127:
        return False, None

    if local_action_cmd:
        return True, uid
    else:
        return False, None