def __init__(self):
if not os.path.exists('wjc.conf'):
logging.error('we need a valid config, none found!')
raise
parser = ConfigParser()
parser.read('wjc.conf')
self.web = parser.get('parameters', 'web')
self.elk = parser.get('parameters', 'elk')
self.job_timeout = parser.getint('parameters', 'job_timeout')
def __init__(self):
if not os.path.exists('ilm.conf'):
logging.error('we need a valid config, none found!')
raise
parser = ConfigParser()
parser.read('ilm.conf')
self.aws_region = parser.get('aws', 'region')
self.aws_secret = parser.get('aws', 'secret_key')
self.aws_access = parser.get('aws', 'access_key')
self.aws_seqgrp = parser.get('aws', 'security_group')
self.aws_req_max_wait = parser.get('aws', 'request_max_wait_time')
self.auto_remove_failed = parser.getboolean('parameters', 'auto_remove_failed')
self.recycle_workers = parser.getboolean('parameters', 'recycle_workers')
self.max_instances = parser.getint('parameters', 'max_instances')
self.max_storage = parser.getint('parameters', 'max_storage')
def __init__(self):
if not os.path.exists('ilm.conf'):
logging.error('we need a valid config, none found!')
raise
parser = ConfigParser()
parser.read('ilm.conf')
self.aws_region = parser.get('aws', 'region')
self.aws_secret = parser.get('aws', 'secret_key')
self.aws_access = parser.get('aws', 'access_key')
self.aws_seqgrp = parser.get('aws', 'security_group')
self.aws_req_max_wait = parser.get('aws', 'request_max_wait_time')
self.auto_remove_failed = parser.getboolean('parameters',
'auto_remove_failed')
self.recycle_workers = parser.getboolean('parameters',
'recycle_workers')
self.max_instances = parser.getint('parameters', 'max_instances')
self.max_storage = parser.getint('parameters', 'max_storage')
output_filename = 'output.log'
error_filename = 'error.log'
# mwahahaha, buffer outputs and write them to loggers during execution (non blocking)
with io.open(output_filename, 'wb', 0) as output_writer, io.open(output_filename, 'rt', 1) as output_reader, \
io.open(error_filename, 'wb', 0) as error_writer, io.open(error_filename, 'rt', 1) as error_reader:
process = subprocess.Popen('./run', shell=True, stdout=output_writer, stderr=error_writer)
while process.poll() is None:
output_line = output_reader.read()
error_line = error_reader.read()
if output_line is not None and len(output_line) > 0:
requests.post(stdout_url, data = {'stdout': output_line})
logging.info(output_line)
if error_line is not None and len(error_line) > 0:
requests.post(stderr_url, data = {'stderr': error_line})
logging.error(error_line)
process.wait()
output_line = output_reader.read()
if output_line is not None and len(output_line) > 0:
requests.post(stdout_url, data = {'stdout': output_line})
logging.info(output_line)
error_line = error_reader.read()
if error_line is not None and len(error_line) > 0:
requests.post(stderr_url, data = {'stderr': error_line})
logging.error(error_line)
# finished
if process.returncode != 0:
r = requests.post('http://[web]/wjc/jobs/[uuid]/state/run_failed', headers=headers)
logging.error('Job returned error code %s' % str(process.returncode))
else:
r = requests.post('http://[web]/wjc/jobs/[uuid]/state/run_succeeded', headers=headers)
os.chmod('run', st.st_mode | stat.S_IEXEC)
# start the 'run' script
r = requests.post('http://[web]/wjc/jobs/[uuid]/state/running', headers=headers)
output_filename = 'output.log'
error_filename = 'error.log'
# mwahahaha, buffer outputs and write them to loggers during execution (non blocking)
with io.open(output_filename, 'wb') as output_writer, io.open(output_filename, 'rb', 1) as output_reader, \
io.open(error_filename, 'wb') as error_writer, io.open(error_filename, 'rb', 1) as error_reader:
process = subprocess.Popen('./run', shell=True, stdout=output_writer, stderr=error_writer)
while process.poll() is None:
output_line = output_reader.read()
error_line = error_reader.read()
if output_line is not None and len(output_line) > 0:
logging.info(output_line)
if error_line is not None and len(error_line) > 0:
logging.error(error_line)
process.wait()
output_line = output_reader.read()
if output_line is not None and len(output_line) > 0:
logging.info(output_line)
error_line = error_reader.read()
if error_line is not None and len(error_line) > 0:
logging.error(error_line)
# finished
if process.returncode != 0:
r = requests.post('http://[web]/wjc/jobs/[uuid]/state/run_failed', headers=headers)
logging.error('Job returned error code %s' % str(process.returncode))
else:
r = requests.post('http://[web]/wjc/jobs/[uuid]/state/run_succeeded', headers=headers)
logging.info('job [uuid] done')
except Exception, e: