def get_command(self):
jar_filenames = []
for ref in self.task_private['jar_lib']:
obj_store_filename = retrieve_filename_for_ref(ref, self.task_record, False)
with open(obj_store_filename, 'r') as f:
with tempfile.NamedTemporaryFile(suffix='.jar', delete=False) as temp_f:
shutil.copyfileobj(f, temp_f)
jar_filenames.append(temp_f.name)
return ["java", "-Xmx2048M", "-cp", str(':'.join(jar_filenames)), "com.asgow.ciel.executor.Java2Executor"]
def _guarded_run(self, task_private, task_descriptor, task_record):
self.task_private = task_private
self.task_record = task_record
self.task_descriptor = task_descriptor
self.expected_outputs = list(self.task_descriptor['expected_outputs'])
self.error_details = None
if "id" in task_private:
id = task_private['id']
self.process_record = self.process_pool.get_process_record(id)
else:
self.process_record = self.process_pool.get_soft_cache_process(self.__class__, task_descriptor["dependencies"])
if self.process_record is None:
self.process_record = self.process_pool.create_process_record(None, "json")
if "command" in task_private:
# XXX sos22 How is SWRealReference in scope here?
if isinstance(task_private["command"], SWRealReference):
# Command has been passed in as a reference.
command = [retrieve_filename_for_ref(arg, self.task_record, False)]
os.chmod(command_line[0], stat.S_IRWXU)
else:
command = [task_private["command"]]
else:
command = self.get_command()
command.extend(["--write-fifo", self.process_record.get_read_fifo_name(),
"--read-fifo", self.process_record.get_write_fifo_name()])
new_proc_env = os.environ.copy()
new_proc_env.update(self.get_env())
new_proc = subprocess.Popen(command, env=new_proc_env, close_fds=True)
self.process_record.set_pid(new_proc.pid)
# XXX: This will block until the attached process opens the pipes.
reader = self.process_record.get_read_fifo()
writer = self.process_record.get_write_fifo()
self.reader = reader
self.writer = writer
#ciel.log('Got reader and writer FIFOs', 'PROC', logging.INFO)
write_framed_json(("start_task", task_private), writer)
try:
if self.process_record.protocol == 'line':
finished = self.line_event_loop(reader, writer)
elif self.process_record.protocol == 'json':
finished = self.json_event_loop(reader, writer)
else:
raise BlameUserException('Unsupported protocol: %s' % self.process_record.protocol)
except MissingInputException, mie:
self.process_pool.delete_process_record(self.process_record)
raise
def start_process(self, input_files, output_files):
command_line = self.args["command_line"]
for i, arg in enumerate(command_line):
if isinstance(arg, SWRealReference):
# Command line argument has been passed in as a reference.
command_line[i] = retrieve_filename_for_ref(
arg, self.task_record, False)
if i == 0:
# First argument must be executable.
os.chmod(command_line[0], stat.S_IRWXU)
try:
env = self.args['env']
except KeyError:
env = {}
ciel.log.error(
"Executing environ with: %s" % " ".join(map(str, command_line)),
'EXEC', logging.DEBUG)
with tempfile.NamedTemporaryFile(delete=False) as input_filenames_file:
for filename in input_files:
input_filenames_file.write(filename)
input_filenames_file.write('\n')
input_filenames_name = input_filenames_file.name
with tempfile.NamedTemporaryFile(
delete=False) as output_filenames_file:
for filename in output_files:
output_filenames_file.write(filename)
output_filenames_file.write('\n')
output_filenames_name = output_filenames_file.name
environment = {
'INPUT_FILES': input_filenames_name,
'OUTPUT_FILES': output_filenames_name
}
environment.update(os.environ)
environment.update(env)
proc = subprocess.Popen(map(str, command_line),
env=environment,
close_fds=True)
#_ = proc.stdout.read(1)
#print 'Got byte back from Executor'
return proc
def open_ref(self, ref, accept_string=False, make_sweetheart=False):
"""Fetches a reference if it is available, and returns a filename for reading it.
Options to do with eagerness, streaming, etc.
If reference is unavailable, raises a ReferenceUnavailableException."""
ref = self.task_record.retrieve_ref(ref)
if not accept_string:
ctx = retrieve_filename_for_ref(ref, self.task_record, return_ctx=True)
else:
ctx = retrieve_file_or_string_for_ref(ref, self.task_record)
if ctx.completed_ref is not None:
if make_sweetheart:
ctx.completed_ref = SW2_SweetheartReference.from_concrete(ctx.completed_ref, get_own_netloc())
self.task_record.publish_ref(ctx.completed_ref)
return ctx.to_safe_dict()
def open_ref(self, ref, accept_string=False, make_sweetheart=False):
"""Fetches a reference if it is available, and returns a filename for reading it.
Options to do with eagerness, streaming, etc.
If reference is unavailable, raises a ReferenceUnavailableException."""
ref = self.task_record.retrieve_ref(ref)
if not accept_string:
ctx = retrieve_filename_for_ref(ref,
self.task_record,
return_ctx=True)
else:
ctx = retrieve_file_or_string_for_ref(ref, self.task_record)
if ctx.completed_ref is not None:
if make_sweetheart:
ctx.completed_ref = SW2_SweetheartReference.from_concrete(
ctx.completed_ref, get_own_netloc())
self.task_record.publish_ref(ctx.completed_ref)
return ctx.to_safe_dict()
def start_process(self, input_files, output_files):
command_line = self.args["command_line"]
for i, arg in enumerate(command_line):
if isinstance(arg, SWRealReference):
# Command line argument has been passed in as a reference.
command_line[i] = retrieve_filename_for_ref(arg, self.task_record, False)
if i == 0:
# First argument must be executable.
os.chmod(command_line[0], stat.S_IRWXU)
ciel.log.error("Executing stdinout with: %s" % " ".join(map(str, command_line)), 'EXEC', logging.DEBUG)
with open(output_files[0], "w") as temp_output_fp:
# This hopefully avoids the race condition in subprocess.Popen()
return subprocess.Popen(map(str, command_line), stdin=PIPE, stdout=temp_output_fp, close_fds=True)
def start_process(self, input_files, output_files):
command_line = self.args["command_line"]
for i, arg in enumerate(command_line):
if isinstance(arg, SWRealReference):
# Command line argument has been passed in as a reference.
command_line[i] = retrieve_filename_for_ref(arg, self.task_record, False)
if i == 0:
# First argument must be executable.
os.chmod(command_line[0], stat.S_IRWXU)
try:
env = self.args['env']
except KeyError:
env = {}
ciel.log.error("Executing environ with: %s" % " ".join(map(str, command_line)), 'EXEC', logging.DEBUG)
with tempfile.NamedTemporaryFile(delete=False) as input_filenames_file:
for filename in input_files:
input_filenames_file.write(filename)
input_filenames_file.write('\n')
input_filenames_name = input_filenames_file.name
with tempfile.NamedTemporaryFile(delete=False) as output_filenames_file:
for filename in output_files:
output_filenames_file.write(filename)
output_filenames_file.write('\n')
output_filenames_name = output_filenames_file.name
environment = {'INPUT_FILES' : input_filenames_name,
'OUTPUT_FILES' : output_filenames_name}
environment.update(os.environ)
environment.update(env)
proc = subprocess.Popen(map(str, command_line), env=environment, close_fds=True)
#_ = proc.stdout.read(1)
#print 'Got byte back from Executor'
return proc
def start_process(self, input_files, output_files):
command_line = self.args["command_line"]
for i, arg in enumerate(command_line):
if isinstance(arg, SWRealReference):
# Command line argument has been passed in as a reference.
command_line[i] = retrieve_filename_for_ref(
arg, self.task_record, False)
if i == 0:
# First argument must be executable.
os.chmod(command_line[0], stat.S_IRWXU)
ciel.log.error(
"Executing stdinout with: %s" % " ".join(map(str, command_line)),
'EXEC', logging.DEBUG)
with open(output_files[0], "w") as temp_output_fp:
# This hopefully avoids the race condition in subprocess.Popen()
return subprocess.Popen(map(str, command_line),
stdin=PIPE,
stdout=temp_output_fp,
close_fds=True)
def _guarded_run(self, task_private, task_descriptor, task_record):
self.task_private = task_private
self.task_record = task_record
self.task_descriptor = task_descriptor
self.expected_outputs = list(self.task_descriptor['expected_outputs'])
self.error_details = None
if "id" in task_private:
id = task_private['id']
self.process_record = self.process_pool.get_process_record(id)
else:
self.process_record = self.process_pool.get_soft_cache_process(
self.__class__, task_descriptor["dependencies"])
if self.process_record is None:
self.process_record = self.process_pool.create_process_record(
None, "json")
if "command" in task_private:
if isinstance(task_private["command"], SWRealReference):
# Command has been passed in as a reference.
command = [
retrieve_filename_for_ref(arg, self.task_record,
False)
]
os.chmod(command_line[0], stat.S_IRWXU)
else:
command = [task_private["command"]]
else:
command = self.get_command()
command.extend([
"--write-fifo",
self.process_record.get_read_fifo_name(), "--read-fifo",
self.process_record.get_write_fifo_name()
])
new_proc_env = os.environ.copy()
new_proc_env.update(self.get_env())
new_proc = subprocess.Popen(command,
env=new_proc_env,
close_fds=True)
self.process_record.set_pid(new_proc.pid)
# XXX: This will block until the attached process opens the pipes.
reader = self.process_record.get_read_fifo()
writer = self.process_record.get_write_fifo()
self.reader = reader
self.writer = writer
#ciel.log('Got reader and writer FIFOs', 'PROC', logging.INFO)
write_framed_json(("start_task", task_private), writer)
try:
if self.process_record.protocol == 'line':
finished = self.line_event_loop(reader, writer)
elif self.process_record.protocol == 'json':
finished = self.json_event_loop(reader, writer)
else:
raise BlameUserException('Unsupported protocol: %s' %
self.process_record.protocol)
except MissingInputException, mie:
self.process_pool.delete_process_record(self.process_record)
raise
