def _writer():
stdout = FileObjectPosix(read_fd, 'rb', bufsize=0, close=True)
while True:
data = stdout.read(buffer_size)
if len(data) == 0:
return
sock.write(data)
def _reader():
f = FileObjectPosix(write_fd, 'wb', bufsize=0, close=True)
while True:
data = sock.recv(buffer_size)
if len(data) == 0:
return
f.write(data)
def _write_loop(self):
f = FileObjectPosix(self._master_pipe[1], 'w', 0)
while True:
try:
f.write(self._write_queue.get())
except:
self._alive = False
break
def close(self):
try:
if self._sock:
self.flush()
finally:
s = self._sock
self._sock = None
if s is not None:
if self._close:
FileObjectPosix.close(self)
else:
_do_reuse_or_drop(s, "_drop")
def close(self):
try:
if self._sock:
self.flush()
finally:
s = self._sock
self._sock = None
if s is not None:
if self._close:
FileObjectPosix.close(self)
else:
_do_reuse_or_drop(s, '_drop')
def _read_loop(self):
f = FileObjectPosix(self._master_pipe[0], 'r', 0)
while True:
try:
data = f.read(2)
size = struct.unpack('>h', data)[0]
data = f.read(size)
cmd, data = utils.unpack_message(data)
except:
# master is dead probably
self._alive = False
break
def submit_with_download(self, task_name, args, sender, env=None):
task_metadata = self.dispatcher.tasks[task_name]._get_metadata()
schema = task_metadata['schema']
url_list = []
if schema is None:
raise RpcException(
errno.ENOENT,
"Task {0} has no schema associated with it".format(task_name))
for idx, arg in enumerate(schema):
if arg.get('type') == 'fd':
rfd, wfd = os.pipe()
url_list.append("/dispatcher/filedownload?token={0}".format(
self.dispatcher.token_store.issue_token(
FileToken(user=sender.user,
lifetime=60,
direction='download',
file=FileObjectPosix(rfd, 'rb', close=True),
name=args[idx]))))
args[idx] = FileDescriptor(wfd)
task_id = self.submit(task_name, args, sender, env)
return task_id, url_list
def submit_with_upload(self, task_name, args, sender, env=None):
task_metadata = self.dispatcher.tasks[task_name]._get_metadata()
schema = task_metadata['schema']
if schema is None:
raise RpcException(
errno.ENOENT,
"Task {0} has no schema associated with it".format(task_name))
upload_token_list = []
for idx, arg in enumerate(schema):
if arg.get('type') == 'fd':
rfd, wfd = os.pipe()
token = self.dispatcher.token_store.issue_token(
FileToken(user=sender.user,
lifetime=600,
direction='upload',
file=FileObjectPosix(wfd, 'wb', close=True),
name=str(uuid.uuid4()),
size=None))
upload_token_list.append(token)
args[idx] = FileDescriptor(rfd)
task_id = self.submit(task_name, args, sender, env)
return task_id, upload_token_list
def __init__(self, sock, mode="rb", bufsize=-1, close=False):
_do_reuse_or_drop(sock, "_reuse")
self._sock = sock
FileObjectPosix.__init__(self, sock, mode, bufsize, close)
def __init__(self, sock, mode='rb', bufsize=-1, close=False):
_do_reuse_or_drop(sock, '_reuse')
self._sock = sock
FileObjectPosix.__init__(self, sock, mode, bufsize, close)
