def dispatch(source, destinations):
"""Dispatch source file to destinations."""
any_error = False
# check that file actually exists
if not os.path.exists(source):
message = "Source file for dispatching does not exist:{}".format(
str(source))
logger.error(message)
any_error = True
success = {}
# rename and send file with right protocol
for url, params, client in destinations:
# Multiple destinations for one client isn't implemented
if client in success:
raise NotImplementedError(
"Only one destination allowed per client")
try:
logger.debug("Dispatching %s to %s", source, str(clean_url(url)))
move_it(source, url, params)
success[client] = True
except Exception as err:
message = "Could not dispatch to {}: {}".format(
str(clean_url(url)), str(err))
logger.error(message)
any_error = True
success[client] = False
if not any_error:
logger.info("Dispatched all files.")
return success
def ack(self, message):
"""Reply with ack to a publication
"""
for url in gen_dict_extract(message.data, 'uri'):
uri = urlparse(url)
pathname = uri.path
if 'origin' in self._attrs and not fnmatch.fnmatch(
os.path.basename(pathname),
os.path.basename(globify(self._attrs["origin"]))):
LOGGER.warning('Client trying to get invalid file: %s', pathname)
return Message(message.subject,
"err",
data="{0:s} not reacheable".format(pathname))
if (self._attrs.get('compression') or self._attrs.get(
'delete', 'False').lower() in ["1", "yes", "true", "on"]):
self._deleter.add(pathname)
new_msg = Message(message.subject, "ack", data=message.data.copy())
try:
new_msg.data['destination'] = clean_url(new_msg.data[
'destination'])
except KeyError:
pass
return new_msg
def _get_cleaned_ack_message(message):
new_msg = Message(message.subject, "ack", data=message.data.copy())
try:
new_msg.data['destination'] = clean_url(new_msg.data['destination'])
except KeyError:
pass
return new_msg
def _sanitize_message_destination(message):
sanitized_message = Message(rawstr=str(message))
try:
_ = urlparse(message.data['destination'])
except (KeyError, TypeError):
pass
else:
sanitized_message.data['destination'] = clean_url(
message.data['destination'])
return sanitized_message
def push(self, message):
"""Reply to push request."""
new_msg = self._move_files(message)
if new_msg is None:
new_msg = Message(message.subject,
_get_push_message_type(message),
data=message.data.copy())
new_msg.data['destination'] = clean_url(
new_msg.data['destination'])
return new_msg
def dispatch(source, destinations):
"""Dispatch source file to destinations."""
any_error = False
# check that file actually exists
if not os.path.exists(source):
message = "Source file for dispatching does not exist:{}".format(
str(source))
logger.error(message)
any_error = True
# rename and send file with right protocol
for url, params in destinations:
try:
logger.debug("Dispatching %s to %s", source, str(clean_url(url)))
move_it(source, url, params)
except Exception as err:
message = "Could not dispatch to {}: {}".format(
str(clean_url(url)), str(err))
logger.error(message)
any_error = True
if not any_error:
logger.info("Dispatched all files.")
def run(self):
while self._loop:
try:
socks = dict(self._poller.poll(timeout=2000))
except ZMQError:
LOGGER.info("Poller interrupted.")
continue
if socks.get(self.out_socket) == POLLIN:
LOGGER.debug("Received a request")
address, _, payload = self.out_socket.recv_multipart(
NOBLOCK)
message = Message(rawstr=payload)
fake_msg = Message(rawstr=str(message))
try:
urlparse(message.data['destination'])
except (KeyError, TypeError):
pass
else:
fake_msg.data['destination'] = clean_url(message.data[
'destination'])
LOGGER.debug("processing request: %s", str(fake_msg))
if message.type == "ping":
Thread(target=self.reply_and_send,
args=(self.pong, address, message)).start()
elif message.type == "push":
Thread(target=self.reply_and_send,
args=(self.push, address, message)).start()
elif message.type == "ack":
Thread(target=self.reply_and_send,
args=(self.ack, address, message)).start()
elif message.type == "info":
Thread(target=self.reply_and_send,
args=(self.info, address, message)).start()
else: # unknown request
Thread(target=self.reply_and_send,
args=(self.unknown, address, message)).start()
elif socks.get(self.in_socket) == POLLIN:
self.out_socket.send_multipart(
self.in_socket.recv_multipart(NOBLOCK))
else: # timeout
pass
def push(self, message):
"""Reply to push request
"""
for the_dict in gen_dict_contains(message.data, 'uri'):
uri = urlparse(the_dict['uri'])
rel_path = the_dict.get('path', None)
pathname = uri.path
# FIXME: check against file_cache
if 'origin' in self._attrs and not fnmatch.fnmatch(
os.path.basename(pathname),
os.path.basename(globify(self._attrs["origin"]))):
LOGGER.warning('Client trying to get invalid file: %s', pathname)
return Message(message.subject,
"err",
data="{0:s} not reachable".format(pathname))
try:
move_it(pathname, message.data['destination'], self._attrs, rel_path=rel_path)
except Exception as err:
return Message(message.subject, "err", data=str(err))
else:
if (self._attrs.get('compression') or self._attrs.get(
'delete', 'False').lower() in ["1", "yes", "true", "on"]):
self._deleter.add(pathname)
if 'dataset' in message.data:
mtype = 'dataset'
elif 'collection' in message.data:
mtype = 'collection'
elif 'uid' in message.data:
mtype = 'file'
else:
raise KeyError('No known metadata in message.')
new_msg = Message(message.subject,
mtype,
data=message.data.copy())
new_msg.data['destination'] = clean_url(new_msg.data[
'destination'])
return new_msg
def move_it(pathname, destination, attrs=None, hook=None, rel_path=None):
"""Check if the file pointed by *pathname* is in the filelist, and move it
if it is.
The *destination* provided is used, and if *rel_path* is provided, it will
be appended to the destination path.
"""
dest_url = urlparse(destination)
if rel_path is not None:
new_path = os.path.join(dest_url.path, rel_path)
else:
new_path = dest_url.path
new_dest = dest_url._replace(path=new_path)
fake_dest = clean_url(new_dest)
LOGGER.debug("new_dest = %s", new_dest)
LOGGER.debug("Copying to: %s", fake_dest)
try:
LOGGER.debug("Scheme = %s", str(dest_url.scheme))
mover = MOVERS[dest_url.scheme]
except KeyError:
LOGGER.error("Unsupported protocol '" + str(dest_url.scheme) +
"'. Could not copy " + pathname + " to " +
str(destination))
raise
try:
mover(pathname, new_dest, attrs=attrs).copy()
if hook:
hook(pathname, new_dest)
except Exception as err:
exc_type, exc_value, exc_traceback = sys.exc_info()
LOGGER.error("Something went wrong during copy of %s to %s: %s",
pathname, str(fake_dest), str(err))
LOGGER.debug("".join(traceback.format_tb(exc_traceback)))
raise err
else:
LOGGER.info("Successfully copied %s to %s", pathname, str(fake_dest))