def _trash(self, xaf):
if self.failure_policy == "delete":
xaf.delete_or_nothing()
self.add_trace(xaf, ">trash", "delete")
elif self.failure_policy == "keep":
self.add_trace(xaf, ">trash", "keep")
new_filepath = os.path.join(
_get_or_make_trash_dir(self.plugin_name, self.step_name),
xaf.basename(),
)
(success, move) = xaf.move_or_copy(new_filepath)
if success:
xaf.write_tags_in_a_file(new_filepath + ".tags")
xattrfile.XattrFile(new_filepath).clear_tags()
else:
xaf.delete_or_nothing()
elif self.failure_policy == "move":
new_filepath = os.path.join(self.failure_policy_move_dest_dir,
xaf.basename())
(success, move) = xaf.move_or_copy(new_filepath)
if not success:
xaf.delete_or_nothing()
else:
if self.failure_policy_move_absolute_dir:
xaf.write_tags_in_a_file(new_filepath + ".tags")
xattrfile.XattrFile(new_filepath).clear_tags()
self.add_trace(xaf, self.failure_policy_move_dest_dir)
def _trash(self, xaf):
if self.failure_policy == "delete":
xaf.delete_or_nothing()
elif self.failure_policy == "keep":
new_filepath = os.path.join(
_get_or_make_trash_dir(self.plugin_name, self.step_name),
xaf.basename(),
)
(success, move) = xaf.move_or_copy(new_filepath)
if success:
xaf.write_tags_in_a_file(new_filepath + ".tags")
xattrfile.XattrFile(new_filepath).clear_tags()
else:
xaf.delete_or_nothing()
elif self.failure_policy == "move":
new_filepath = os.path.join(self.args.failure_policy_move_dest_dir,
xaf.basename())
(success, move) = xaf.move_or_copy(new_filepath)
if success:
if self.failure_policy_move_keep_tags:
suffix = self.failure_policy_move_keep_tags_suffix
xaf.write_tags_in_a_file(new_filepath + suffix)
xattrfile.XattrFile(new_filepath).clear_tags()
else:
xaf.delete_or_nothing()
def __run_in_debug_mode(self, filepath):
self.info("Start the debug mode with filepath=%s", filepath)
self.__get_logger().setLevel(0)
self._debug_mode = True
self._ping()
tmp_filepath = self.get_tmp_filepath()
original_xaf = xattrfile.XattrFile(filepath)
xaf = original_xaf.copy(tmp_filepath)
return self._process(xaf)
def _on_message(self, client, userdata, message):
self.info("message received on %s (size: %i)", message.topic,
len(message.payload))
self.debug("message qos: %s", message.qos)
self.debug("message retain flag: %s", message.retain)
self.debug("message info: %s", message.info)
basename = mfutil.get_unique_hexa_identifier()
filepath = os.path.join(self.args.dest_dir, basename)
tmp_filepath = ".".join((filepath, self.args.tmp_suffix))
with open(tmp_filepath, "wb") as f:
f.write(message.payload)
xaf = xattrfile.XattrFile(tmp_filepath)
self.set_tag(xaf, "mqtt_listener_subscription_topic",
self.args.subscription_topic)
self.set_tag(xaf, "mqtt_listener_received_topic", message.topic)
self.set_tag(xaf, "mqtt_listener_broker_hostname",
self.args.broker_hostname)
self.set_tag(xaf, "mqtt_listener_broker_port",
str(self.args.broker_port))
self._set_before_tags(xaf)
xaf.rename(filepath)
def __on_message(self, client, userdata, message):
self.info("message received on %s (size: %i)", message.topic,
len(message.payload))
self.debug("message qos: %s", message.qos)
self.debug("message retain flag: %s", message.retain)
tmp_filepath = self.get_tmp_filepath()
with open(tmp_filepath, "wb") as f:
f.write(message.payload)
xaf = xattrfile.XattrFile(tmp_filepath)
self.set_tag(xaf, "mqttlistener_topic", message.topic,
add_latest=False)
self.set_tag(xaf, "mqttlistener_hostname", self.mqtt_hostname,
add_latest=False)
self.set_tag(xaf, "mqttlistener_port", str(self.mqtt_port),
add_latest=False)
self._set_before_tags(xaf)
target = self.get_target_filepath(xaf)
res, _ = xaf.move_or_copy(target)
if res:
self.info("message saved in %s" % target)
else:
self.warning("can't save message in %s" % target)
def _on_message(
self,
_unused_channel,
basic_deliver,
properties,
body,
add_extra_tags_func=None,
):
basename = mfutil.get_unique_hexa_identifier()
self.debug("basename: %s" % (basename))
filename = os.path.join(self.args.dest_dir, basename)
tmp_filename = ".".join((filename, self.args.tmp_suffix))
self.debug("Created tmp file name : %s" % tmp_filename)
with open(tmp_filename, "wb") as f:
f.write(body)
xaf = xattrfile.XattrFile(tmp_filename)
self.set_tag(
xaf,
"amqp_listener.subscription_exchange",
self.args.subscription_exchange,
)
self.set_tag(
xaf,
"amqp_listener_subscription_queue",
self.args.subscription_queue,
)
self.set_tag(xaf, "amqp_listener_broker_hostname",
self.args.broker_hostname)
self.set_tag(xaf, "amqp_listener_broker_port",
str(self.args.broker_port))
if add_extra_tags_func:
add_extra_tags_func(self, xaf, _unused_channel, basic_deliver,
properties, body)
self._set_before_tags(xaf)
xaf.rename(filename)
self.debug("Created file name : %s" % filename)
self.info("Received message %s from %s (size: %i)" %
(basic_deliver.delivery_tag, properties.app_id, len(body)))
def on_message(client, userdata, message):
client.get_logger().debug("message received: %s" ,str(message.payload.decode("utf-8")))
client.get_logger().debug("message topic: %s",message.topic)
client.get_logger().debug("message qos: %s",message.qos)
client.get_logger().debug("message retain flag: %s",message.retain)
client.get_logger().debug("message info: %s",message.info)
client.get_logger().debug("userdata: %s",userdata)
basename = mfutil.get_unique_hexa_identifier()
client.get_logger().debug("basename: %s" % (basename))
filepath = os.path.join(client.args.dest_dir, basename)
tmp_filepath = '.'.join((filepath, client.args.tmp_suffix))
client.get_logger().debug("Created tmp file name : %s" % (tmp_filepath))
with open(tmp_filepath, "w") as fichier:
fichier.write(str(message.payload.decode("utf-8")))
xaf = xattrfile.XattrFile(tmp_filepath)
xaf.tags['mqtt_listener_subscription_topic'] = client.args.subscription_topic
xaf.tags['mqtt_listener_received_topic'] = message.topic
xaf.tags['mqtt_listener_broker_hostname'] = client.args.broker_hostname
xaf.tags['mqtt_listener_broker_port'] = str(client.args.broker_port)
if userdata is not None:
xaf.tags['mqtt_listener_user_data'] = userdata
xaf.rename(filepath)
client.get_logger().debug("Created file name : %s" % (filepath))
def on_message(self, basic_deliver, properties, body):
LOGGER.info('Received message # %s from %s: %s',
basic_deliver.delivery_tag, properties.app_id, body)
tmp_filepath = self.get_tmp_filepath()
with open(tmp_filepath, "wb") as f:
f.write(body)
xaf = xattrfile.XattrFile(tmp_filepath)
# FIXME: other tags
self.set_tag(xaf,
"amqplistener_hostname",
self.amqp_hostname,
add_latest=False)
self.set_tag(xaf,
"amqplistener_port",
str(self.amqp_port),
add_latest=False)
self._set_before_tags(xaf)
target = self.get_target_filepath(xaf)
res, _ = xaf.move_or_copy(target)
if res:
self.info("message saved in %s" % target)
else:
self.warning("can't save message in %s" % target)
return res
def __run_in_daemon_mode(self,
redis_host,
redis_port,
redis_queue,
redis_unix_socket_path=None):
self.info("Start the daemon mode with redis_queue=%s", redis_queue)
if redis_unix_socket_path:
r = redis.StrictRedis(unix_socket_path=redis_unix_socket_path)
else:
r = redis.StrictRedis(host=redis_host, port=redis_port)
redis_connection_exceptions_counter = 0
while not self.stop_flag:
self.__ping_if_needed()
try:
msg = r.brpop(redis_queue, 1)
except KeyboardInterrupt:
self.stop_flag = True
continue
except redis.exceptions.ConnectionError:
redis_connection_exceptions_counter = \
redis_connection_exceptions_counter + 1
if redis_connection_exceptions_counter >= 10:
self.critical("Can't connect to redis after 10s => exit")
self.stop_flag = True
continue
else:
self.debug("Can't connect to redis => "
"I will try again after 1s sleep")
time.sleep(1)
continue
redis_connection_exceptions_counter = 0
if msg is None:
# brpop timeout
continue
try:
decoded_msg = json.loads(msg[1].decode('utf8'))
filepath = os.path.join(decoded_msg['directory'],
decoded_msg['filename'])
except Exception:
self.warning("wrong message type popped on bus "
"=> stopping to force a restart")
self.stop_flag = True
continue
if not os.path.exists(filepath):
self.debug(
"filepath: %s does not exist anymore "
"=> ignoring event", filepath)
continue
xaf = xattrfile.XattrFile(filepath)
counter = "counter.%s.%s" % (self.plugin_name, self.step_name)
latest = "latest.%s.%s" % (self.plugin_name, self.step_name)
try:
r.incr(counter)
except Exception:
self.warning("can't update redis counter: %s" % counter)
try:
r.set(latest, str(get_utc_unix_timestamp()))
except Exception:
self.warning("can't set latest timestamp: %s" % latest)
self._process(xaf)
self.debug("Stop to brpop queue %s" % redis_queue)
