Пример #1
0
 def test_get_unique_hexa_identifier(self):
     x = get_unique_hexa_identifier()
     self.assertTrue(len(x) == 32)
     for c in x:
         if c not in string.hexdigits.lower():
             raise ("Invalid char in identifier")
     y = get_unique_hexa_identifier()
     self.assertTrue(x != y)
Пример #2
0
 def test_get_unique_hexa_identifier(self):
     x = get_unique_hexa_identifier()
     self.assertTrue(len(x) == 32)
     for c in x:
         if c not in string.hexdigits.lower():
             raise("Invalid char in identifier")
     y = get_unique_hexa_identifier()
     self.assertTrue(x != y)
Пример #3
0
 def init(self):
     tmp = self.get_custom_config_value("mqtt_clean_session", default=1,
                                        transform=int)
     self.mqtt_clean_session = (tmp == 1)
     self.mqtt_client_id = self.get_custom_config_value("mqtt_client_id",
                                                        default=None)
     if self.mqtt_client_id is None or self.mqtt_client_id == "" or \
             self.mqtt_client_id == "null":
         if self.mqtt_clean_session is False:
             raise Exception("you must set a mqtt_client_id when "
                             "clean_session==0")
         self.mqtt_client_id = get_unique_hexa_identifier()
     self.mqtt_qos = self.get_custom_config_value("mqtt_qos", default="0",
                                                  transform=int)
     if self.mqtt_qos not in (0, 1, 2):
         raise Exception("mqtt_qos must be 0, 1 or 2")
     self.mqtt_hostname = self.get_custom_config_value("mqtt_hostname",
                                                       default="localhost")
     self.mqtt_port = self.get_custom_config_value("mqtt_port",
                                                   default="1883",
                                                   transform=int)
     self.mqtt_topic = self.get_custom_config_value("mqtt_topic",
                                                    default="#")
     self.mqtt_keepalive = self.get_custom_config_value(
         "mqtt_keepalive", default="60", transform=int)
     tmp = self.get_custom_config_value("dest_dir", default=None)
     self.dest_dir, _ = dest_dir_to_absolute(tmp, allow_absolute=True)
     self.client = mqtt.Client(client_id=self.mqtt_client_id,
                               clean_session=self.mqtt_clean_session)
     self.client.on_message = self._on_message
     self.client.on_connect = self._on_connect
     self.client.on_disconnect = self._on_disconnect
     signal.signal(signal.SIGTERM, self.__sigterm_handler)
Пример #4
0
 def setUp(self):
     self.base_path = os.path.join(TEST_DIRECTORY,
                                   get_unique_hexa_identifier())
     init_plugins_base(self.base_path)
     self.original_metwork_layers_path = os.environ["METWORK_LAYERS_PATH"]
     os.environ["METWORK_LAYERS_PATH"] = \
         os.environ["METWORK_LAYERS_PATH"] + ":%s" % self.base_path
Пример #5
0
 def _set_original_uid_if_necessary(self, xaf, forced_uid=None):
     tag_name = self._get_original_uid_tag_name()
     if tag_name not in xaf.tags:
         if forced_uid is None:
             original_uid = get_unique_hexa_identifier()
         else:
             original_uid = forced_uid
         self._set_tag(xaf, tag_name, original_uid, info=True)
Пример #6
0
 def _copy(self, xaf, directory):
     target_path = os.path.join(directory, get_unique_hexa_identifier())
     try:
         xaf.copy(target_path)
     except Exception:
         self.warning("Can't copy %s to %s" % (xaf.filepath, target_path))
         return False
     self.info("File %s copied to %s" % (xaf.filepath, target_path))
     return True
Пример #7
0
def main():
    parser = argparse.ArgumentParser("Inject a file into a plugin/step")
    parser.add_argument("filepath", type=str, help="filepath to inject")
    parser.add_argument("--plugin",
                        type=str,
                        help="plugin name (default :guess_file_type)",
                        default="guess_file_type")
    parser.add_argument("--step",
                        type=str,
                        help="step name (default: main)",
                        default="main")
    parser.add_argument("--move",
                        action="store_true",
                        help="move the file instead of copying it "
                        "(default: copy)")
    parser.add_argument("--random-basename",
                        action="store_true",
                        help="use a random basename for copying/moving "
                        "the file (default: keep the original basename)")
    parser.add_argument(
        "--incoming",
        action="store_true",
        help="ignore plugin and step parameter and inject "
        "the file into the first configured directory listened"
        " by the MFDATA_INTERNAL_PLUGINS_WATCHED_DIRECTORIES env var")

    args = parser.parse_args()
    try:
        x = XattrFile(args.filepath)
    except Exception:
        logger.warning("can't open %s" % args.filepath)
        sys.exit(1)
    if args.random_basename:
        basename = get_unique_hexa_identifier()
    else:
        basename = os.path.basename(args.filepath)
    if args.incoming:
        env_var = 'MFDATA_INTERNAL_PLUGINS_WATCHED_DIRECTORIES'
        first_directory = os.environ[env_var].split(',')[0]
        new_filepath = os.path.join(os.environ['MFDATA_DATA_IN_DIR'],
                                    first_directory, basename)
    else:
        new_filepath = \
            os.path.join(get_plugin_step_directory_path(args.plugin,
                                                        args.step), basename)
    if args.move:
        res, moved = x.move_or_copy(new_filepath)
        if not res:
            logger.warning("can't move %s to %s" %
                           (args.filepath, new_filepath))
            sys.exit(1)
    else:
        res = x.copy_or_nothing(new_filepath)
        if not res:
            logger.warning("can't copy %s to %s" %
                           (args.filepath, new_filepath))
            sys.exit(1)
Пример #8
0
 def reinject(self, xaf, retry_attempt):
     self._set_tag_latest(xaf, "attempt", str(retry_attempt + 1))
     filepath = xaf.filepath
     self.info("reinjecting %s into %s/... attempt %d", filepath,
               self.args.reinject_dir, retry_attempt + 1)
     new_filepath = os.path.join(self.args.reinject_dir,
                                 get_unique_hexa_identifier())
     self._set_after_tags(xaf, True)
     xaf.move_or_copy(new_filepath)
Пример #9
0
 def reinject(self, xaf, retry_attempt):
     self._set_tag_latest(xaf, "attempt", str(retry_attempt + 1))
     filepath = xaf.filepath
     self.info("reinjecting %s into %s/... attempt %d", filepath,
               self.args.reinject_dir, retry_attempt + 1)
     new_filepath = os.path.join(self.args.reinject_dir,
                                 get_unique_hexa_identifier())
     xaf.move_or_copy(new_filepath)
     self.get_stats_client().incr("number_of_processed_files", 1)
     self.get_stats_client().incr("bytes_of_processed_files", xaf.getsize())
Пример #10
0
 def test_mkdir_p(self):
     tmp = create_tmp_dirpath()
     x = get_unique_hexa_identifier()
     new_tmp = os.path.join(tmp, x)
     res = mkdir_p(new_tmp, nowarning=True)
     self.assertTrue(res)
     self.assertTrue(os.path.isdir(new_tmp))
     res = mkdir_p(new_tmp, nowarning=True)
     self.assertTrue(res)
     self.assertTrue(os.path.isdir(new_tmp))
     shutil.rmtree(tmp)
Пример #11
0
 def test_mkdir_p(self):
     tmp = create_tmp_dirpath()
     x = get_unique_hexa_identifier()
     new_tmp = os.path.join(tmp, x)
     res = mkdir_p(new_tmp, nowarning=True)
     self.assertTrue(res)
     self.assertTrue(os.path.isdir(new_tmp))
     res = mkdir_p(new_tmp, nowarning=True)
     self.assertTrue(res)
     self.assertTrue(os.path.isdir(new_tmp))
     shutil.rmtree(tmp)
Пример #12
0
def build_plugin(plugin_path, plugins_base_dir=None):
    plugin_path = os.path.abspath(plugin_path)
    plugins_base_dir = _get_plugins_base_dir(plugins_base_dir)
    base = os.path.join(plugins_base_dir, "base")
    pwd = os.getcwd()
    parser = ExtendedConfigParser(config=os.environ.get('MFCONFIG', 'GENERIC'),
                                  strict=False,
                                  inheritance="im")
    with open(os.path.join(plugin_path, "config.ini"), "r") as f:
        config_content = f.read()
    if six.PY2:
        parser.read_string(config_content.decode('utf-8'))
    else:
        parser.read_string(config_content)
    name = parser['general']['name']
    version = parser['general']['version']
    summary = parser['general']['summary']
    license = parser['general']['license']
    try:
        packager = parser['general']['packager']
    except Exception:
        packager = parser['general']['maintainer']
    vendor = parser['general']['vendor']
    url = parser['general']['url']
    tmpdir = os.path.join(RUNTIME_HOME, "tmp",
                          "plugin_%s" % get_unique_hexa_identifier())
    mkdir_p_or_die(os.path.join(tmpdir, "BUILD"))
    mkdir_p_or_die(os.path.join(tmpdir, "RPMS"))
    mkdir_p_or_die(os.path.join(tmpdir, "SRPMS"))
    _make_plugin_spec(os.path.join(tmpdir, "specfile.spec"), name, version,
                      summary, license, packager, vendor, url)
    cmd = "source %s/lib/bash_utils.sh ; " % MFEXT_HOME
    cmd = cmd + "layer_load rpm@mfext ; "
    cmd = cmd + 'rpmbuild --define "_topdir %s" --define "pwd %s" ' \
        '--define "prefix %s" --dbpath %s ' \
        '-bb %s/specfile.spec' % (tmpdir, plugin_path, tmpdir,
                                  base, tmpdir)
    x = BashWrapperOrRaise(cmd, MFUtilPluginCantBuild,
                           "can't build plugin %s" % plugin_path)
    tmp = glob.glob(os.path.join(tmpdir, "RPMS", "x86_64", "*.rpm"))
    if len(tmp) == 0:
        raise MFUtilPluginCantBuild("can't find generated plugin" %
                                    plugin_path,
                                    bash_wrapper=x)
    plugin_path = tmp[0]
    new_basename = \
        os.path.basename(plugin_path).replace("x86_64.rpm",
                                              "metwork.%s.plugin" %
                                              MODULE_LOWERCASE)
    new_plugin_path = os.path.join(pwd, new_basename)
    shutil.move(plugin_path, new_plugin_path)
    shutil.rmtree(tmpdir, True)
    os.chdir(pwd)
    return new_plugin_path
Пример #13
0
 def test_get_domainname_from_resolv_conf(self):
     tmp_file = get_unique_hexa_identifier()
     with open(tmp_file, "w") as f:
         f.write("# Generated by NetworkManager\n")
         f.write("search foo.bar\n")
         f.write("nameserver 1.2.3.4\n")
         f.write("nameserver 1.2.3.5\n")
         f.write("domain foo.bar\n")
     tmp = _get_domainname_from_resolv_conf(tmp_file)
     self.assertTrue(isinstance(tmp, str))
     self.assertEquals(tmp, "foo.bar")
     os.unlink(tmp_file)
Пример #14
0
 def test_get_domainname_from_resolv_conf(self):
     tmp_file = get_unique_hexa_identifier()
     with open(tmp_file, "w") as f:
         f.write("# Generated by NetworkManager\n")
         f.write("search foo.bar\n")
         f.write("nameserver 1.2.3.4\n")
         f.write("nameserver 1.2.3.5\n")
         f.write("domain foo.bar\n")
     tmp = _get_domainname_from_resolv_conf(tmp_file)
     self.assertTrue(isinstance(tmp, str))
     self.assertEquals(tmp, "foo.bar")
     os.unlink(tmp_file)
Пример #15
0
def build_plugin(plugin_path, plugins_base_dir=None):
    plugin_path = os.path.abspath(plugin_path)
    plugins_base_dir = _get_plugins_base_dir(plugins_base_dir)
    base = os.path.join(plugins_base_dir, "base")
    pwd = os.getcwd()
    parser = ExtendedConfigParser(config=os.environ.get('MFCONFIG', 'GENERIC'),
                                  strict=False, inheritance="im")
    with open(os.path.join(plugin_path, "config.ini"), "r") as f:
        config_content = f.read()
    if six.PY2:
        parser.read_string(config_content.decode('utf-8'))
    else:
        parser.read_string(config_content)
    name = parser['general']['name']
    version = parser['general']['version']
    summary = parser['general']['summary']
    license = parser['general']['license']
    try:
        packager = parser['general']['packager']
    except Exception:
        packager = parser['general']['maintainer']
    vendor = parser['general']['vendor']
    url = parser['general']['url']
    tmpdir = os.path.join(RUNTIME_HOME, "tmp",
                          "plugin_%s" % get_unique_hexa_identifier())
    mkdir_p_or_die(os.path.join(tmpdir, "BUILD"))
    mkdir_p_or_die(os.path.join(tmpdir, "RPMS"))
    mkdir_p_or_die(os.path.join(tmpdir, "SRPMS"))
    _make_plugin_spec(os.path.join(tmpdir, "specfile.spec"), name, version,
                      summary, license, packager, vendor, url)
    cmd = "source %s/lib/bash_utils.sh ; " % MFEXT_HOME
    cmd = cmd + "layer_load rpm@mfext ; "
    cmd = cmd + 'rpmbuild --define "_topdir %s" --define "pwd %s" ' \
        '--define "prefix %s" --dbpath %s ' \
        '-bb %s/specfile.spec' % (tmpdir, plugin_path, tmpdir,
                                  base, tmpdir)
    x = BashWrapperOrRaise(cmd, MFUtilPluginCantBuild,
                           "can't build plugin %s" % plugin_path)
    tmp = glob.glob(os.path.join(tmpdir, "RPMS", "x86_64", "*.rpm"))
    if len(tmp) == 0:
        raise MFUtilPluginCantBuild("can't find generated plugin" %
                                    plugin_path, bash_wrapper=x)
    plugin_path = tmp[0]
    new_basename = \
        os.path.basename(plugin_path).replace("x86_64.rpm",
                                              "metwork.%s.plugin" %
                                              MODULE_LOWERCASE)
    new_plugin_path = os.path.join(pwd, new_basename)
    shutil.move(plugin_path, new_plugin_path)
    shutil.rmtree(tmpdir, True)
    os.chdir(pwd)
    return new_plugin_path
Пример #16
0
 def get_stor_name(self, xaf):
     step_counter = self._get_counter_tag_value(xaf, not_found_value='999')
     replaces = {
         "{RANDOM_ID}": get_unique_hexa_identifier(),
         "{ORIGINAL_BASENAME}": self.get_original_basename(xaf),
         "{ORIGINAL_DIRNAME}": self.get_original_dirname(xaf),
         "{ORIGINAL_UID}": self.get_original_uid(xaf),
         "{STEP_COUNTER}": str(step_counter)
     }
     stor_name = self.ftp_basename
     for to_replace, replaced in replaces.items():
         stor_name = stor_name.replace(to_replace, replaced)
     return time.strftime(stor_name)
Пример #17
0
 def __xxx_to_plugin_step_get_basename(self, xaf, keep_original_basename):
     if keep_original_basename is None:
         if "first.core.original_uid" in xaf.tags:
             return xaf.tags["first.core.original_uid"].decode('utf8')
         else:
             keep_original_basename = False
     if keep_original_basename is True:
         return xaf.basename()
     elif keep_original_basename is False:
         return get_unique_hexa_identifier()
     else:
         raise Exception(
             "invalid value for keep_original_basename: %s "
             "(must be True, False or None)", keep_original_basename)
Пример #18
0
 def _hardlink_or_copy(self, xaf, directory):
     old_filepath = xaf.filepath
     target_path = os.path.join(directory, get_unique_hexa_identifier())
     result, hardlinked = xaf.hardlink_or_copy(target_path)
     if not result:
         self.warning("Can't hardlink or copy %s to %s" %
                      (old_filepath, target_path))
         return False
     if hardlinked:
         self.info("File %s hardlinked to %s" % (old_filepath, target_path))
     else:
         self.info("File %s copied to %s (can't hardlink)" %
                   (old_filepath, target_path))
     return True
Пример #19
0
 def repackage_plugin(self, name):
     p = self.get_plugin(name)
     p.load_full()
     if p.is_dev_linked:
         raise Exception("can't repackage a devlinked plugin")
     tmpdir = os.path.join(MFMODULE_RUNTIME_HOME, "tmp",
                           "plugin_%s" % get_unique_hexa_identifier())
     shutil.copytree(p.home, tmpdir, symlinks=True)
     # FIXME: ? clean ?
     newp = self.make_plugin(tmpdir, dont_read_config_overrides=True)
     newp.load_full()
     x = configupdater.ConfigUpdater(delimiters=('=', ),
                                     comment_prefixes=('#', ))
     x.optionxform = str
     x.read("%s/config.ini" % tmpdir)
     sections = p.configuration._doc.keys()
     for section in sections:
         for option in p.configuration._doc[section].keys():
             if option.startswith('_'):
                 continue
             val = p.configuration._doc[section][option]
             try:
                 newval = newp.configuration._doc[section][option]
             except Exception:
                 # probably a new section
                 try:
                     x.add_section(section)
                 except Exception:
                     pass
                 newval = None
             try:
                 if newval is None:
                     print("NEW [%s]/%s = %s" % (section, option, val),
                           file=sys.stderr)
                 else:
                     if newval == val:
                         continue
                     print("CHANGED [%s]/%s: %s => %s" %
                           (section, option, newval, val),
                           file=sys.stderr)
                 if isinstance(val, bool):
                     x.set(section, option, "1" if val else "0")
                 else:
                     x.set(section, option, val)
             except Exception:
                 pass
     x.update_file()
     new_p = self.make_plugin(tmpdir)
     return new_p.build()
Пример #20
0
def _get_tmp_filepath(plugin_name, step_name):
    """Get a full temporary filepath (including unique filename).

    Args:
        plugin_name (string): plugin name.
        step_name (string): step name.


    Returns:
        (string) full temporary filepath (including unique filename).

    """
    tmp_dir = _get_or_make_tmp_dir(plugin_name, step_name)
    tmp_name = get_unique_hexa_identifier()
    return os.path.join(tmp_dir, tmp_name)
Пример #21
0
 def process(self, xaf):
     original_dirname = self.get_original_dirname(xaf)
     original_basename = self.get_original_basename(xaf)
     original_uid = self.get_original_uid(xaf)
     random_basename = get_unique_hexa_identifier()
     step_counter = self._get_counter_tag_value(xaf, not_found_value='999')
     if step_counter != 999 and step_counter != 0:
         step_counter_minus_1 = step_counter - 1
     else:
         step_counter_minus_1 = step_counter
     rendered_template = os.path.join(self.archive_dir,
                                      self.strftime_template)
     rendered_template = rendered_template.replace('{RANDOM_ID}',
                                                   random_basename)
     rendered_template = rendered_template.replace('{ORIGINAL_BASENAME}',
                                                   original_basename)
     rendered_template = rendered_template.replace('{ORIGINAL_UID}',
                                                   original_uid)
     rendered_template = rendered_template.replace('{ORIGINAL_DIRNAME}',
                                                   original_dirname)
     rendered_template = rendered_template.replace('{STEP_COUNTER}',
                                                   str(step_counter))
     rendered_template = \
         rendered_template.replace('{STEP_COUNTER_MINUS_1}',
                                   str(step_counter_minus_1))
     new_filepath = time.strftime(rendered_template)
     dirname = os.path.dirname(new_filepath)
     res = mkdir_p(dirname)
     if not res:
         self.warning("can't mkdir %s" % dirname)
         return False
     # Store old xaf filepath to display in the logs
     old_filepath = xaf.filepath
     success, moved = xaf.move_or_copy(new_filepath)
     if success:
         if moved:
             self.info("%s moved into %s", old_filepath, new_filepath)
         else:
             self.info("%s copied into %s", xaf.filepath, new_filepath)
         if self.keep_tags:
             tags_filepath = new_filepath + self.keep_tags_suffix
             xaf.write_tags_in_a_file(tags_filepath)
         XattrFile(new_filepath).clear_tags()
         return True
     else:
         self.warning("Can't move/copy %s to %s", xaf.filepath,
                      new_filepath)
         return False
Пример #22
0
 def compute_basename(self, xaf):
     step_counter = self._get_counter_tag_value(xaf, not_found_value='999')
     replaces = {
         "{RANDOM_ID}": get_unique_hexa_identifier(),
         "{ORIGINAL_BASENAME}": self.get_original_basename(xaf),
         "{ORIGINAL_DIRNAME}": self.get_original_dirname(xaf),
         "{ORIGINAL_UID}": self.get_original_uid(xaf),
         "{STEP_COUNTER}": str(step_counter)
     }
     tmp = self.dest_basename
     for to_replace, replaced in replaces.items():
         tmp = tmp.replace(to_replace, replaced)
     if '%' in tmp:
         return datetime.datetime.now().strftime(tmp)
     else:
         return tmp
Пример #23
0
 def __init__(
     self,
     name_prefix: str,
     cmd: Cmd,
 ):
     self.cmd: Cmd = cmd
     self.id: str = get_unique_hexa_identifier()[0:10]
     self.name: str = f"{name_prefix}.managed_process.{self.id}"
     self.logger = mflog.get_logger("alwaysup.managed_process").bind(
         id=self.name)
     StateMixin.__init__(self, logger=self.logger)
     self.process: Optional[Process] = None
     self.pid: Optional[int] = None
     self.returncode: Optional[int] = None
     self.set_state(ManagedProcessState.READY)
     self._wait_for_process_end_task: Optional[asyncio.Task] = None
     self.cmd_line: Optional[str] = None
Пример #24
0
    def copy_to_plugin_step(self, xaf, plugin_name, step_name):
        """Copy a XattrFile (with tags) to another plugin/step.

        Args:
            xaf (XattrFile): XattrFile to move
            plugin_name (string): plugin name
            step_name (string): step name

        Returns:
            boolean: True if ok

        """
        target_path = os.path.join(
            get_plugin_step_directory_path(plugin_name, step_name),
            get_unique_hexa_identifier())
        self._set_after_tags(xaf, True)
        result = xaf.copy_or_nothing(target_path)
        return result
Пример #25
0
    def move_to_plugin_step(self, xaf, plugin_name, step_name):
        """Move a XattrFile to another plugin/step.

        Args:
            xaf (XattrFile): XattrFile to move
            plugin_name (string): plugin name
            step_name (string): step name

        Returns:
            boolean: True if ok

        """
        target_path = os.path.join(
            get_plugin_step_directory_path(plugin_name, step_name),
            get_unique_hexa_identifier(),
        )
        result, _ = xaf.move_or_copy(target_path)
        return result
Пример #26
0
def _get_tmp_filepath(plugin_name, step_name, forced_basename=None):
    """Get a full temporary filepath (including unique filename).

    Args:
        plugin_name (string): plugin name.
        step_name (string): step name.
        forced_basename (string): if not None, use the given string as
            basename. If None, the basename is a random identifier.

    Returns:
        (string) full temporary filepath (including unique filename).

    """
    tmp_dir = _get_or_make_tmp_dir(plugin_name, step_name)
    if forced_basename is None:
        tmp_name = get_unique_hexa_identifier()
    else:
        tmp_name = forced_basename
    return os.path.join(tmp_dir, tmp_name)
Пример #27
0
 def _on_message(self, client, userdata, message):
     self.info("message received on %s (size: %i)", message.topic,
               len(message.payload))
     self.debug("message qos: %s", message.qos)
     self.debug("message retain flag: %s", message.retain)
     self.debug("message info: %s", message.info)
     basename = mfutil.get_unique_hexa_identifier()
     filepath = os.path.join(self.args.dest_dir, basename)
     tmp_filepath = ".".join((filepath, self.args.tmp_suffix))
     with open(tmp_filepath, "wb") as f:
         f.write(message.payload)
     xaf = xattrfile.XattrFile(tmp_filepath)
     self.set_tag(xaf, "mqtt_listener_subscription_topic",
                  self.args.subscription_topic)
     self.set_tag(xaf, "mqtt_listener_received_topic", message.topic)
     self.set_tag(xaf, "mqtt_listener_broker_hostname",
                  self.args.broker_hostname)
     self.set_tag(xaf, "mqtt_listener_broker_port",
                  str(self.args.broker_port))
     self._set_before_tags(xaf)
     xaf.rename(filepath)
Пример #28
0
 def _on_message(
     self,
     _unused_channel,
     basic_deliver,
     properties,
     body,
     add_extra_tags_func=None,
 ):
     basename = mfutil.get_unique_hexa_identifier()
     self.debug("basename: %s" % (basename))
     filename = os.path.join(self.args.dest_dir, basename)
     tmp_filename = ".".join((filename, self.args.tmp_suffix))
     self.debug("Created tmp file name : %s" % tmp_filename)
     with open(tmp_filename, "wb") as f:
         f.write(body)
     xaf = xattrfile.XattrFile(tmp_filename)
     self.set_tag(
         xaf,
         "amqp_listener.subscription_exchange",
         self.args.subscription_exchange,
     )
     self.set_tag(
         xaf,
         "amqp_listener_subscription_queue",
         self.args.subscription_queue,
     )
     self.set_tag(xaf, "amqp_listener_broker_hostname",
                  self.args.broker_hostname)
     self.set_tag(xaf, "amqp_listener_broker_port",
                  str(self.args.broker_port))
     if add_extra_tags_func:
         add_extra_tags_func(self, xaf, _unused_channel, basic_deliver,
                             properties, body)
     self._set_before_tags(xaf)
     xaf.rename(filename)
     self.debug("Created file name : %s" % filename)
     self.info("Received message %s from %s (size: %i)" %
               (basic_deliver.delivery_tag, properties.app_id, len(body)))
Пример #29
0
def on_message(client, userdata, message):
    client.get_logger().debug("message received: %s" ,str(message.payload.decode("utf-8")))
    client.get_logger().debug("message topic: %s",message.topic)
    client.get_logger().debug("message qos: %s",message.qos)
    client.get_logger().debug("message retain flag: %s",message.retain)
    client.get_logger().debug("message info: %s",message.info)
    client.get_logger().debug("userdata: %s",userdata)
    basename = mfutil.get_unique_hexa_identifier()
    client.get_logger().debug("basename: %s" % (basename))

    filepath = os.path.join(client.args.dest_dir, basename)
    tmp_filepath = '.'.join((filepath, client.args.tmp_suffix))
    client.get_logger().debug("Created tmp file name : %s" % (tmp_filepath))
    with open(tmp_filepath, "w") as fichier:
        fichier.write(str(message.payload.decode("utf-8")))
    xaf = xattrfile.XattrFile(tmp_filepath)
    xaf.tags['mqtt_listener_subscription_topic'] = client.args.subscription_topic
    xaf.tags['mqtt_listener_received_topic'] = message.topic
    xaf.tags['mqtt_listener_broker_hostname'] = client.args.broker_hostname
    xaf.tags['mqtt_listener_broker_port'] = str(client.args.broker_port)
    if userdata is not None:
        xaf.tags['mqtt_listener_user_data'] = userdata
    xaf.rename(filepath)
    client.get_logger().debug("Created file name : %s" % (filepath))
Пример #30
0
#!/usr/bin/env python3

import datetime
import sys
import os
import time
from mfutil import BashWrapperOrRaise, get_unique_hexa_identifier

NGINX_PORT = int(os.environ['MFSERV_NGINX_PORT'])

UNIQUE = get_unique_hexa_identifier()
BashWrapperOrRaise("rm -Rf foobar")
BashWrapperOrRaise("plugins.uninstall foobar || true")

print(
    BashWrapperOrRaise("bootstrap_plugin.py create --template=python3_noweb "
                       "--no-input foobar"))
with open("foobar/config.ini", "a") as f:
    f.write("\n\n[extra_daemon_foo]\n")
    f.write("_cmd_and_args = foo.sh %s\n" % UNIQUE)
    f.write("numprocesses=1\n")
BashWrapperOrRaise("mkdir -p foobar/bin")
BashWrapperOrRaise("cp -f foo.sh foobar/bin/")
BashWrapperOrRaise("chmod +x foobar/bin/foo.sh")

print(BashWrapperOrRaise("cd foobar && make develop"))

now_fn = datetime.datetime.now
before = now_fn()
code = 1
while (now_fn() - before).total_seconds() <= 30:
Пример #31
0
 def __set_original_uid_if_necessary(self, xaf):
     tag_name = self.__get_original_uid_tag_name()
     if tag_name not in xaf.tags:
         original_uid = get_unique_hexa_identifier()
         self.__set_tag(xaf, tag_name, original_uid)
Пример #32
0
 def test_get_tmp_filepath(self):
     tmpdir = os.path.join(TEST_DIRECTORY, get_unique_hexa_identifier())
     tmp = get_tmp_filepath(tmpdir, "foo")
     self.assertTrue(tmp.startswith(os.path.join(tmpdir, "foo")))
     shutil.rmtree(tmpdir, True)
Пример #33
0
 def test_get_tmp_filepath(self):
     tmpdir = os.path.join(TEST_DIRECTORY, get_unique_hexa_identifier())
     tmp = get_tmp_filepath(tmpdir, "foo")
     self.assertTrue(tmp.startswith(os.path.join(tmpdir, "foo")))
     shutil.rmtree(tmpdir, True)
Пример #34
0
def build_plugin(plugin_path, plugins_base_dir=None):
    """Build a plugin.

    Args:
        plugin_path (string): the plugin path to build
        plugins_base_dir (string): (optional) the plugin base directory path.
            If not set, the default plugins base directory path is used.

    Raises:
        MFUtilPluginCantBuild: if a error occurs during build

    """
    plugin_path = os.path.abspath(plugin_path)
    plugins_base_dir = _get_plugins_base_dir(plugins_base_dir)
    base = os.path.join(plugins_base_dir, "base")
    pwd = os.getcwd()
    parser = OpinionatedConfigParser()
    with open(os.path.join(plugin_path, "config.ini"), "r") as f:
        config_content = f.read()
    if six.PY2:
        parser.read_string(config_content.decode('utf-8'))
    else:
        parser.read_string(config_content)
    with open(os.path.join(plugin_path, ".layerapi2_label"), "r") as f:
        name = f.read().replace('plugin_', '', 1).split('@')[0]
    version = parser['general']['version']
    summary = parser['general']['summary']
    license = parser['general']['license']
    try:
        packager = parser['general']['packager']
    except Exception:
        packager = parser['general']['maintainer']
    vendor = parser['general']['vendor']
    url = parser['general']['url']
    tmpdir = os.path.join(RUNTIME_HOME, "tmp",
                          "plugin_%s" % get_unique_hexa_identifier())
    mkdir_p_or_die(os.path.join(tmpdir, "BUILD"))
    mkdir_p_or_die(os.path.join(tmpdir, "RPMS"))
    mkdir_p_or_die(os.path.join(tmpdir, "SRPMS"))
    _make_plugin_spec(os.path.join(tmpdir, "specfile.spec"), name, version,
                      summary, license, packager, vendor, url)
    cmd = "source %s/lib/bash_utils.sh ; " % MFEXT_HOME
    cmd = cmd + "layer_load rpm@mfext ; "
    cmd = cmd + 'rpmbuild --define "_topdir %s" --define "pwd %s" ' \
        '--define "prefix %s" --dbpath %s ' \
        '-bb %s/specfile.spec' % (tmpdir, plugin_path, tmpdir,
                                  base, tmpdir)
    x = BashWrapperOrRaise(cmd, MFUtilPluginCantBuild,
                           "can't build plugin %s" % plugin_path)
    tmp = glob.glob(os.path.join(tmpdir, "RPMS", "x86_64", "*.rpm"))
    if len(tmp) == 0:
        raise MFUtilPluginCantBuild("can't find generated plugin" %
                                    plugin_path, bash_wrapper=x)
    plugin_path = tmp[0]
    new_basename = \
        os.path.basename(plugin_path).replace("x86_64.rpm",
                                              "metwork.%s.plugin" %
                                              MFMODULE_LOWERCASE)
    new_plugin_path = os.path.join(pwd, new_basename)
    shutil.move(plugin_path, new_plugin_path)
    shutil.rmtree(tmpdir, True)
    os.chdir(pwd)
    return new_plugin_path
Пример #35
0
 def __init__(self, max_size, max_wait):
     self.id = get_unique_hexa_identifier()
     self._xafs = []
     self.max_size = max_size
     self.max_wait = max_wait