def _trash(self, xaf): if self.failure_policy == "delete": xaf.delete_or_nothing() self.add_trace(xaf, ">trash", "delete") elif self.failure_policy == "keep": self.add_trace(xaf, ">trash", "keep") new_filepath = os.path.join( _get_or_make_trash_dir(self.plugin_name, self.step_name), xaf.basename(), ) (success, move) = xaf.move_or_copy(new_filepath) if success: xaf.write_tags_in_a_file(new_filepath + ".tags") xattrfile.XattrFile(new_filepath).clear_tags() else: xaf.delete_or_nothing() elif self.failure_policy == "move": new_filepath = os.path.join(self.failure_policy_move_dest_dir, xaf.basename()) (success, move) = xaf.move_or_copy(new_filepath) if not success: xaf.delete_or_nothing() else: if self.failure_policy_move_absolute_dir: xaf.write_tags_in_a_file(new_filepath + ".tags") xattrfile.XattrFile(new_filepath).clear_tags() self.add_trace(xaf, self.failure_policy_move_dest_dir)
def _trash(self, xaf): if self.failure_policy == "delete": xaf.delete_or_nothing() elif self.failure_policy == "keep": new_filepath = os.path.join( _get_or_make_trash_dir(self.plugin_name, self.step_name), xaf.basename(), ) (success, move) = xaf.move_or_copy(new_filepath) if success: xaf.write_tags_in_a_file(new_filepath + ".tags") xattrfile.XattrFile(new_filepath).clear_tags() else: xaf.delete_or_nothing() elif self.failure_policy == "move": new_filepath = os.path.join(self.args.failure_policy_move_dest_dir, xaf.basename()) (success, move) = xaf.move_or_copy(new_filepath) if success: if self.failure_policy_move_keep_tags: suffix = self.failure_policy_move_keep_tags_suffix xaf.write_tags_in_a_file(new_filepath + suffix) xattrfile.XattrFile(new_filepath).clear_tags() else: xaf.delete_or_nothing()
def __run_in_debug_mode(self, filepath): self.info("Start the debug mode with filepath=%s", filepath) self.__get_logger().setLevel(0) self._debug_mode = True self._ping() tmp_filepath = self.get_tmp_filepath() original_xaf = xattrfile.XattrFile(filepath) xaf = original_xaf.copy(tmp_filepath) return self._process(xaf)
def _on_message(self, client, userdata, message): self.info("message received on %s (size: %i)", message.topic, len(message.payload)) self.debug("message qos: %s", message.qos) self.debug("message retain flag: %s", message.retain) self.debug("message info: %s", message.info) basename = mfutil.get_unique_hexa_identifier() filepath = os.path.join(self.args.dest_dir, basename) tmp_filepath = ".".join((filepath, self.args.tmp_suffix)) with open(tmp_filepath, "wb") as f: f.write(message.payload) xaf = xattrfile.XattrFile(tmp_filepath) self.set_tag(xaf, "mqtt_listener_subscription_topic", self.args.subscription_topic) self.set_tag(xaf, "mqtt_listener_received_topic", message.topic) self.set_tag(xaf, "mqtt_listener_broker_hostname", self.args.broker_hostname) self.set_tag(xaf, "mqtt_listener_broker_port", str(self.args.broker_port)) self._set_before_tags(xaf) xaf.rename(filepath)
def __on_message(self, client, userdata, message): self.info("message received on %s (size: %i)", message.topic, len(message.payload)) self.debug("message qos: %s", message.qos) self.debug("message retain flag: %s", message.retain) tmp_filepath = self.get_tmp_filepath() with open(tmp_filepath, "wb") as f: f.write(message.payload) xaf = xattrfile.XattrFile(tmp_filepath) self.set_tag(xaf, "mqttlistener_topic", message.topic, add_latest=False) self.set_tag(xaf, "mqttlistener_hostname", self.mqtt_hostname, add_latest=False) self.set_tag(xaf, "mqttlistener_port", str(self.mqtt_port), add_latest=False) self._set_before_tags(xaf) target = self.get_target_filepath(xaf) res, _ = xaf.move_or_copy(target) if res: self.info("message saved in %s" % target) else: self.warning("can't save message in %s" % target)
def _on_message( self, _unused_channel, basic_deliver, properties, body, add_extra_tags_func=None, ): basename = mfutil.get_unique_hexa_identifier() self.debug("basename: %s" % (basename)) filename = os.path.join(self.args.dest_dir, basename) tmp_filename = ".".join((filename, self.args.tmp_suffix)) self.debug("Created tmp file name : %s" % tmp_filename) with open(tmp_filename, "wb") as f: f.write(body) xaf = xattrfile.XattrFile(tmp_filename) self.set_tag( xaf, "amqp_listener.subscription_exchange", self.args.subscription_exchange, ) self.set_tag( xaf, "amqp_listener_subscription_queue", self.args.subscription_queue, ) self.set_tag(xaf, "amqp_listener_broker_hostname", self.args.broker_hostname) self.set_tag(xaf, "amqp_listener_broker_port", str(self.args.broker_port)) if add_extra_tags_func: add_extra_tags_func(self, xaf, _unused_channel, basic_deliver, properties, body) self._set_before_tags(xaf) xaf.rename(filename) self.debug("Created file name : %s" % filename) self.info("Received message %s from %s (size: %i)" % (basic_deliver.delivery_tag, properties.app_id, len(body)))
def on_message(client, userdata, message): client.get_logger().debug("message received: %s" ,str(message.payload.decode("utf-8"))) client.get_logger().debug("message topic: %s",message.topic) client.get_logger().debug("message qos: %s",message.qos) client.get_logger().debug("message retain flag: %s",message.retain) client.get_logger().debug("message info: %s",message.info) client.get_logger().debug("userdata: %s",userdata) basename = mfutil.get_unique_hexa_identifier() client.get_logger().debug("basename: %s" % (basename)) filepath = os.path.join(client.args.dest_dir, basename) tmp_filepath = '.'.join((filepath, client.args.tmp_suffix)) client.get_logger().debug("Created tmp file name : %s" % (tmp_filepath)) with open(tmp_filepath, "w") as fichier: fichier.write(str(message.payload.decode("utf-8"))) xaf = xattrfile.XattrFile(tmp_filepath) xaf.tags['mqtt_listener_subscription_topic'] = client.args.subscription_topic xaf.tags['mqtt_listener_received_topic'] = message.topic xaf.tags['mqtt_listener_broker_hostname'] = client.args.broker_hostname xaf.tags['mqtt_listener_broker_port'] = str(client.args.broker_port) if userdata is not None: xaf.tags['mqtt_listener_user_data'] = userdata xaf.rename(filepath) client.get_logger().debug("Created file name : %s" % (filepath))
def on_message(self, basic_deliver, properties, body): LOGGER.info('Received message # %s from %s: %s', basic_deliver.delivery_tag, properties.app_id, body) tmp_filepath = self.get_tmp_filepath() with open(tmp_filepath, "wb") as f: f.write(body) xaf = xattrfile.XattrFile(tmp_filepath) # FIXME: other tags self.set_tag(xaf, "amqplistener_hostname", self.amqp_hostname, add_latest=False) self.set_tag(xaf, "amqplistener_port", str(self.amqp_port), add_latest=False) self._set_before_tags(xaf) target = self.get_target_filepath(xaf) res, _ = xaf.move_or_copy(target) if res: self.info("message saved in %s" % target) else: self.warning("can't save message in %s" % target) return res
def __run_in_daemon_mode(self, redis_host, redis_port, redis_queue, redis_unix_socket_path=None): self.info("Start the daemon mode with redis_queue=%s", redis_queue) if redis_unix_socket_path: r = redis.StrictRedis(unix_socket_path=redis_unix_socket_path) else: r = redis.StrictRedis(host=redis_host, port=redis_port) redis_connection_exceptions_counter = 0 while not self.stop_flag: self.__ping_if_needed() try: msg = r.brpop(redis_queue, 1) except KeyboardInterrupt: self.stop_flag = True continue except redis.exceptions.ConnectionError: redis_connection_exceptions_counter = \ redis_connection_exceptions_counter + 1 if redis_connection_exceptions_counter >= 10: self.critical("Can't connect to redis after 10s => exit") self.stop_flag = True continue else: self.debug("Can't connect to redis => " "I will try again after 1s sleep") time.sleep(1) continue redis_connection_exceptions_counter = 0 if msg is None: # brpop timeout continue try: decoded_msg = json.loads(msg[1].decode('utf8')) filepath = os.path.join(decoded_msg['directory'], decoded_msg['filename']) except Exception: self.warning("wrong message type popped on bus " "=> stopping to force a restart") self.stop_flag = True continue if not os.path.exists(filepath): self.debug( "filepath: %s does not exist anymore " "=> ignoring event", filepath) continue xaf = xattrfile.XattrFile(filepath) counter = "counter.%s.%s" % (self.plugin_name, self.step_name) latest = "latest.%s.%s" % (self.plugin_name, self.step_name) try: r.incr(counter) except Exception: self.warning("can't update redis counter: %s" % counter) try: r.set(latest, str(get_utc_unix_timestamp())) except Exception: self.warning("can't set latest timestamp: %s" % latest) self._process(xaf) self.debug("Stop to brpop queue %s" % redis_queue)