def launch_ssh_task(job_config, schedule_time, settings):
unique_job_id = job_id_generator(job_config.get('id'), schedule_time)
remote_server_id = job_config.get('remote_server_id')
remote_server_config = remote_servers.get(remote_server_id)
remote_action_script = job_config.get('remote_action').get('script')
remote_action_script_input_args = job_config.get('remote_action').get('args')
remote_action_script_args = [parse_schedule_time(arg, schedule_time) for arg in remote_action_script_input_args]
local_action_cmd, uid = ssh_task_cmd_generator(remote_action_script, remote_action_script_args,
remote_server_config, unique_job_id, settings)
get_pid_id_proc = subprocess.Popen(local_action_cmd, stdout=subprocess.PIPE)
exit_code = get_pid_id_proc.returncode
#connection check
if exit_code == 255 or exit_code == 127:
return False, None
if local_action_cmd:
return True, uid
else:
return False, None
def launch_emr_task(job_config, schedule_time, settings):
settings.logger.info('Creating EMR client')
emr_client = boto3.client('emr')
settings.logger.info('EMR client created')
#ugly hack
steps_from_json = job_config.get('Steps', [])
for step_config in steps_from_json:
for arg_id in range(len(step_config.get('HadoopJarStep').get('Args'))):
step_config['HadoopJarStep']['Args'][arg_id] = parse_schedule_time(
step_config['HadoopJarStep']['Args'][arg_id], schedule_time)
try:
aws_response = emr_client.run_job_flow(
Name='_'.join([job_config.get('id'), schedule_time]),
LogUri=job_config.get('LogUri', ''),
AmiVersion=job_config.get('AmiVersion', ''),
Instances=job_config.get('Instances'),
Steps=steps_from_json,
BootstrapActions=job_config.get('BootstrapActions', []),
SupportedProducts=job_config.get('SupportedProducts', []),
NewSupportedProducts=job_config.get('NewSupportedProducts', []),
Applications=job_config.get('Applications', []),
Configurations=job_config.get('Configurations', []),
VisibleToAllUsers=True if job_config.get('VisibleToAllUsers') == "True" else False,
JobFlowRole=job_config.get('JobFlowRole', settings.job_flow_role),
ServiceRole=job_config.get('ServiceRole', settings.service_role)
)
except ClientError:
settings.logger.info('Invalid AWS response for %s' % job_config)
return False, None
settings.logger.info('AWS response\n%s' % aws_response)
success = aws_response.get('ResponseMetadata').get('HTTPStatusCode') == 200
cluster_id = aws_response.get('JobFlowId')
return success, cluster_id
