Beispiel #1
0
def __read_configuration(configuration_handler,
                         filename="tuf.interposition.json",
                         parent_repository_directory=None,
                         parent_ssl_certificates_directory=None):
    """
  A generic function to read TUF interposition configurations off a file, and
  then handle those configurations with a given function. configuration_handler
  must be a function which accepts a tuf.interposition.Configuration
  instance.

  Returns the parsed configurations as a dictionary of configurations indexed
  by hostnames."""

    INVALID_TUF_CONFIGURATION = "Invalid configuration for {network_location}!"
    INVALID_TUF_INTERPOSITION_JSON = "Invalid configuration in {filename}!"
    NO_CONFIGURATIONS = "No configurations found in configuration in {filename}!"

    # Configurations indexed by hostnames.
    parsed_configurations = {}

    try:
        with open(filename) as tuf_interposition_json:
            tuf_interpositions = json.load(tuf_interposition_json)
            configurations = tuf_interpositions.get("configurations", {})

            if len(configurations) == 0:
                raise InvalidConfiguration(
                    NO_CONFIGURATIONS.format(filename=filename))

            else:
                for network_location, configuration in configurations.iteritems(
                ):
                    try:
                        configuration_parser = ConfigurationParser(
                            network_location,
                            configuration,
                            parent_repository_directory=
                            parent_repository_directory,
                            parent_ssl_certificates_directory=
                            parent_ssl_certificates_directory)

                        configuration = configuration_parser.parse()
                        configuration_handler(configuration)
                        parsed_configurations[
                            configuration.hostname] = configuration

                    except:
                        Logger.exception(
                            INVALID_TUF_CONFIGURATION.format(
                                network_location=network_location))
                        raise

    except:
        Logger.exception(
            INVALID_TUF_INTERPOSITION_JSON.format(filename=filename))
        raise

    else:
        return parsed_configurations
Beispiel #2
0
    def auto_updater(self):
        configuration = ConfigurationParser('dashboard.conf')
        Timer(configuration.refresh_rate(), self.auto_updater).start()
        now = datetime.now()
        self.last_update = str(now.hour).zfill(2) + ":" + str(now.minute % 60).zfill(2)

        # Load jenkins info
        url, user, password, key = configuration.jenkins_credentials()
        zentyal_jenkins = JenkinsHelper(url, user, password, key, configuration)
        self.jobs = zentyal_jenkins.get_jobs()
        self.components = zentyal_jenkins.get_components()

        # Load public tracker info
        url, key = configuration.public_tracker_credentials()
        self.public_tracker = RedmineHelper(url, key)

        # Load github pull requests
        client_id, client_secret = configuration.github_credentials()
        oauth_token, retest_message = configuration.github_retest()
        self.github = GitHubHelper(client_id, client_secret, oauth_token, retest_message)
        repositories = configuration.github_repositories()

        for repo in repositories:
            self.github.add_pull_requests(repo['organization'], repo['repository'])

        self.pullrequests = self.github.get_pull_requests()
        self.base_branchs = self.github.base_branchs()

        # Load packages data
        repo_path = configuration.zentyal_repo_path()
        zentyal_git = ZentyalGitHelper(repo_path, self.pullrequests)
        self.pending_packages = zentyal_git.get_pending_packages()
def __read_configuration(configuration_handler,
                         filename="tuf.interposition.json",
                         parent_repository_directory=None,
                         parent_ssl_certificates_directory=None):
  """
  A generic function to read TUF interposition configurations off a file, and
  then handle those configurations with a given function. configuration_handler
  must be a function which accepts a tuf.interposition.Configuration
  instance.

  Returns the parsed configurations as a dictionary of configurations indexed
  by hostnames."""

  INVALID_TUF_CONFIGURATION = "Invalid configuration for {network_location}!"
  INVALID_TUF_INTERPOSITION_JSON = "Invalid configuration in {filename}!"
  NO_CONFIGURATIONS = "No configurations found in configuration in {filename}!"

  # Configurations indexed by hostnames.
  parsed_configurations = {}

  try:
    with open(filename) as tuf_interposition_json:
      tuf_interpositions = json.load(tuf_interposition_json)
      configurations = tuf_interpositions.get("configurations", {})

      if len(configurations) == 0:
        raise InvalidConfiguration(NO_CONFIGURATIONS.format(filename=filename))

      else:
        for network_location, configuration in configurations.iteritems():
          try:
            configuration_parser = ConfigurationParser(network_location,
              configuration, parent_repository_directory=parent_repository_directory,
              parent_ssl_certificates_directory=parent_ssl_certificates_directory)

            configuration = configuration_parser.parse()
            configuration_handler(configuration)
            parsed_configurations[configuration.hostname] = configuration

          except:
            Logger.exception(INVALID_TUF_CONFIGURATION.format(network_location=network_location))
            raise

  except:
    Logger.exception(INVALID_TUF_INTERPOSITION_JSON.format(filename=filename))
    raise

  else:
    return parsed_configurations
Beispiel #4
0
Datei: run.py Projekt: tswsxk/XKT
def sym_run(stage: (int, str) = "viz"):  # pragma: no cover
    if isinstance(stage, str):
        stage = {
            "viz": 0,
            "pseudo": 1,
            "real": 2,
            "cli": 3,
        }[stage]

    if stage <= 1:
        cfg = Configuration(
            hyper_params={
                "ku_num": 835,
                "latent_dim": 600,
                "hidden_num": 900,
            },
            nettype="EmbedDKT",
        )

        net = get_net(**cfg.hyper_params)

        if stage == 0:
            # ############################## Net Visualization ###########################
            net_viz(net, cfg, False)
        else:
            # ############################## Pseudo Test #################################
            pseudo_numerical_check(net, cfg)

    elif stage == 2:
        # ################################# Simple Train ###############################
        import mxnet as mx
        train(
            "$data_dir/train.json",
            "$data_dir/test.json",
            dataset="assistment_2009_2010",
            ctx=mx.gpu(2),
            optimizer_params={"learning_rate": 0.001},
            hyper_params={
                "ku_num": 146,
                "hidden_num": 200,
                "dropout": 0.5
            },
            batch_size=16,
        )

    elif stage == 3:
        # ################################# CLI ###########################
        cfg_parser = ConfigurationParser(Configuration, commands=[train])
        cfg_kwargs = cfg_parser()
        assert "subcommand" in cfg_kwargs
        subcommand = cfg_kwargs["subcommand"]
        del cfg_kwargs["subcommand"]
        print(cfg_kwargs)
        eval("%s" % subcommand)(**cfg_kwargs)

    else:
        raise TypeError
Beispiel #5
0
def sym_run(stage: (int, str) = "viz"):  # pragma: no cover
    if isinstance(stage, str):
        stage = {
            "viz": 0,
            "pseudo": 1,
            "real": 2,
            "cli": 3,
        }[stage]

    if stage <= 1:
        cfg = Configuration(hyper_params={})
        net = get_net(**cfg.hyper_params)
        net.initialize()

        if stage == 0:
            # ############################## Net Visualization ###########################
            net_viz(net, cfg, False)
        else:
            # ############################## Pseudo Test #################################
            pseudo_numerical_check(net, cfg)

    elif stage == 2:
        # ################################# Simple Train ###############################
        import mxnet as mx
        train(
            "$data_dir/train.csv",
            "$data_dir/test.csv",
            ctx=mx.cpu(),
            optimizer_params={
                "learning_rate": 0.01,
                "clip_gradient": 1,
            },
            dataset="a0910",
            # init_params_update={"initial_user_item": False},
            # workspace="random",
            workspace="init",
            # params_save=True
        )

    elif stage == 3:
        # ################################# CLI ###########################
        cfg_parser = ConfigurationParser(Configuration, commands=[train])
        cfg_kwargs = cfg_parser()
        assert "subcommand" in cfg_kwargs
        subcommand = cfg_kwargs["subcommand"]
        del cfg_kwargs["subcommand"]
        print(cfg_kwargs)
        eval("%s" % subcommand)(**cfg_kwargs)

    else:
        raise TypeError
Beispiel #6
0
def __read_configuration(configuration_handler,
                         filename="tuf.interposition.json",
                         parent_repository_directory=None,
                         parent_ssl_certificates_directory=None):
  """
  A generic function to read a TUF interposition configuration off the disk,
  and handle it. configuration_handler must be a function which accepts a
  tuf.interposition.Configuration instance."""

  INVALID_TUF_CONFIGURATION = "Invalid configuration for {network_location}!"
  INVALID_TUF_INTERPOSITION_JSON = "Invalid configuration in {filename}!"
  NO_CONFIGURATIONS = "No configurations found in configuration in {filename}!"

  try:
    with open(filename) as tuf_interposition_json:
      tuf_interpositions = json.load(tuf_interposition_json)
      configurations = tuf_interpositions.get("configurations", {})

      if len(configurations) == 0:
        raise InvalidConfiguration(NO_CONFIGURATIONS.format(filename=filename))

      else:
        for network_location, configuration in configurations.iteritems():
          try:
            configuration_parser = ConfigurationParser(network_location,
              configuration, parent_repository_directory=parent_repository_directory,
              parent_ssl_certificates_directory=parent_ssl_certificates_directory)

            configuration = configuration_parser.parse()
            configuration_handler(configuration)

          except:
            Logger.exception(INVALID_TUF_CONFIGURATION.format(network_location=network_location))
            raise

  except:
    Logger.exception(INVALID_TUF_INTERPOSITION_JSON.format(filename=filename))
    raise
Beispiel #7
0
def sym_run(stage: (int, str) = "pesudo"):  # pragma: no cover
    if isinstance(stage, str):
        stage = {
            "pseudo": 0,
            "real": 1,
            "cli": 2,
        }[stage]

    if stage == 0:
        # ############################## Pesudo Test #################################
        cfg = Configuration(
            hyper_params={
                "ku_num": 835,
                "hidden_num": 835,
            },
            ctx="cuda:0,1,2",
        )
        net = get_net(**cfg.hyper_params)
        pseudo_numerical_check(net, cfg)

    elif stage == 1:
        # ################################# Simple Train ###############################
        train(
            "$data_dir/train",
            "$data_dir/test",
            dataset="junyi",
            ctx="cuda:0",
            optimizer_params={"lr": 0.001},
            hyper_params={
                "ku_num": 835,
                "hidden_num": 835
            },
            batch_size=16,
        )

    elif stage == 2:
        # ################################# CLI ###########################
        cfg_parser = ConfigurationParser(Configuration, commands=[train])
        cfg_kwargs = cfg_parser()
        assert "subcommand" in cfg_kwargs
        subcommand = cfg_kwargs["subcommand"]
        del cfg_kwargs["subcommand"]
        print(cfg_kwargs)
        eval("%s" % subcommand)(**cfg_kwargs)

    else:
        raise TypeError
Beispiel #8
0
def sym_run(stage: (int, str) = "viz"):  # pragma: no cover
    if isinstance(stage, str):
        stage = {
            "viz": 0,
            "pseudo": 1,
            "real": 2,
            "cli": 3,
        }[stage]

    if stage <= 1:
        cfg = Configuration(
            hyper_params=dict(
                ku_num=835,
                key_embedding_dim=50,
                value_embedding_dim=200,
                hidden_num=900,
                key_memory_size=20,
                value_memory_size=20,
                key_memory_state_dim=50,
                value_memory_state_dim=200,
            )
        )

        net = get_net(**cfg.hyper_params)

        if stage == 0:
            # ############################## Net Visualization ###########################
            net_viz(net, cfg, False)
        else:
            # ############################## Pseudo Test #################################
            pseudo_numerical_check(net, cfg)

    elif stage == 2:
        # ################################# Simple Train ###############################
        import mxnet as mx
        train(
            "$data_dir/train.json",
            "$data_dir/test.json",
            dataset="junyi",
            ctx=mx.gpu(),
            optimizer_params={
                "learning_rate": 0.001
            },
            hyper_params=dict(
                ku_num=835,
                key_embedding_dim=200,
                value_embedding_dim=200,
                hidden_num=835,
                key_memory_size=40,
                dropout=0.5,
            ),
            batch_size=16,
            root="../../../",
            data_dir="$root_data_dir",
            end_epoch=10,
        )

    elif stage == 3:
        # ################################# CLI ###########################
        cfg_parser = ConfigurationParser(Configuration, commands=[train])
        cfg_kwargs = cfg_parser()
        assert "subcommand" in cfg_kwargs
        subcommand = cfg_kwargs["subcommand"]
        del cfg_kwargs["subcommand"]
        print(cfg_kwargs)
        eval("%s" % subcommand)(**cfg_kwargs)

    else:
        raise TypeError
Beispiel #9
0
def sym_run(stage: (int, str) = "viz"):  # pragma: no cover
    if isinstance(stage, str):
        stage = {
            "viz": 0,
            "pseudo": 1,
            "real": 2,
            "cli": 3,
        }[stage]

    if stage <= 1:
        cfg = Configuration(
            hyper_params={
                "model_type": "wclstm",
                "class_num": 32,
                "embedding_dim": 256,
            }
        )
        embedding_size = {
            "w": 100,
            "c": 100,
        }
        net = get_net(embedding_size=embedding_size, **cfg.hyper_params)

        if stage == 0:
            # ############################## Net Visualization ###########################
            net_viz(net, cfg, False)
        else:
            # ############################## Pseudo Test #################################
            pseudo_numerical_check(net, cfg)

    elif stage == 2:
        # ################################# Simple Train ###############################
        import mxnet as mx
        train(
            "$data_dir/train.json",
            "$data_dir/test.json",
            "w:$vec_dir/word.vec.dat,c:$vec_dir/char.vec.dat",
            ctx=mx.gpu(),
            hyper_params={
                "model_type": "wclstm",
                "class_num": 32,
                "embedding_dim": 256,
            },
            root="../../../..",
            dataset="ctc32",
            data_dir="$root_data_dir",
            end_epoch=1,
        )

    elif stage == 3:
        # ################################# CLI ###########################
        cfg_parser = ConfigurationParser(Configuration, commands=[train])
        cfg_kwargs = cfg_parser()
        assert "subcommand" in cfg_kwargs
        subcommand = cfg_kwargs["subcommand"]
        del cfg_kwargs["subcommand"]
        print(cfg_kwargs)
        eval("%s" % subcommand)(**cfg_kwargs)

    else:
        raise TypeError
Beispiel #10
0
def sym_run(stage: (int, str) = "viz"):  # pragma: no cover
    if isinstance(stage, str):
        stage = {
            "viz": 0,
            "pseudo": 1,
            "real": 2,
            "cli": 3,
        }[stage]

    if stage <= 1:
        cfg = Configuration(
            hyper_params=dict(
                ku_num=835,
            )
        )

        net = get_net(**cfg.hyper_params)

        if stage == 0:
            # ############################## Net Visualization ###########################
            net_viz(net, cfg, False)
        else:
            # ############################## Pseudo Test #################################
            pseudo_numerical_check(net, cfg)

    elif stage == 2:
        # ################################# Simple Train ###############################
        import mxnet as mx
        train(
            "$data_dir/train.json",
            "$data_dir/test.json",
            ctx=mx.gpu(),
            optimizer_params={
                "learning_rate": 0.001
            },
            dump=True,
            data_dir="$root_data_dir",
            # end_epoch=20,
            # root="../../../",
            # dataset="assistment_2009_2010",
            # hyper_params=dict(
            #     ku_num=124,
            #     hidden_num=32,
            #     latent_dim=32,
            #     dropout=0.5,
            # ),
            end_epoch=10,
            hyper_params=dict(
                ku_num=835,
                hidden_num=32,
                latent_dim=32,
                dropout=0.5,
            ),
            dataset="junyi",
            root="/data/tongshiwei/KT",
            root_data_dir="$root/$dataset",
        )

    elif stage == 3:
        # ################################# CLI ###########################
        cfg_parser = ConfigurationParser(Configuration, commands=[train])
        cfg_kwargs = cfg_parser()
        assert "subcommand" in cfg_kwargs
        subcommand = cfg_kwargs["subcommand"]
        del cfg_kwargs["subcommand"]
        print(cfg_kwargs)
        eval("%s" % subcommand)(**cfg_kwargs)

    else:
        raise TypeError
Beispiel #11
0
def sym_run(stage: (int, str) = "viz"):  # pragma: no cover
    if isinstance(stage, str):
        stage = {
            "viz": 0,
            "pseudo": 1,
            "real": 2,
            "cli": 3,
        }[stage]

    if stage <= 1:
        cfg = Configuration(hyper_params={
            "user_num": 1000,
            "item_num": 100,
            "vec_dim": 100,
            "op": "mlp"
        },
                            eval_params={
                                "unlabeled_value": 0,
                                "k": [1, 3],
                                "pointwise": True,
                            })
        net = get_net(**cfg.hyper_params)

        if stage == 0:
            # ############################## Net Visualization ###########################
            net_viz(net, cfg, False)
        else:
            # ############################## Pseudo Test #################################
            pseudo_numerical_check(net, cfg)

    elif stage == 2:
        # ################################# Simple Train ###############################
        import mxnet as mx
        train(
            "$data_dir/ml-1m/train.jsonl",
            "$data_dir/ml-1m/test.jsonl",
            hyper_params={
                "num_a": 6040,
                "num_b": 3900,
                "vec_dim": 128,
                "op": "linear"
            },
            root_data_dir="../../../../",
            optimizer_params={"learning_rate": 0.001},
            # ctx=[mx.gpu(3)],
            ctx=[mx.gpu(5)],
            batch_size=16,
            # optimizer_params={
            #     "learning_rate": 0.01
            # },
            # ctx=[mx.gpu(5), mx.gpu(6), mx.gpu(7), mx.gpu(8)],
            # batch_size=256,
        )

    elif stage == 3:
        # ################################# CLI ###########################
        cfg_parser = ConfigurationParser(Configuration, commands=[train])
        cfg_kwargs = cfg_parser()
        assert "subcommand" in cfg_kwargs
        subcommand = cfg_kwargs["subcommand"]
        del cfg_kwargs["subcommand"]
        print(cfg_kwargs)
        eval("%s" % subcommand)(**cfg_kwargs)

    else:
        raise TypeError
Beispiel #12
0
def check_auth(request_username, request_password):
    configuration = ConfigurationParser('dashboard.conf')
    username, password = configuration.authentication_credentials()

    return request_username == username and request_password == password
Beispiel #13
0
def configure(filename="tuf.interposition.json",
              parent_repository_directory=None,
              parent_ssl_certificates_directory=None):

  """
  The optional parent_repository_directory parameter is used to specify the
  containing parent directory of the "repository_directory" specified in a
  configuration for *all* network locations, because sometimes the absolute
  location of the "repository_directory" is only known at runtime. If you
  need to specify a different parent_repository_directory for other
  network locations, simply call this method again with different parameters.

  Ditto for the optional parent_ssl_certificates_directory parameter.

  Example of a TUF interposition configuration JSON object:

  {
      "configurations": {
          "seattle.cs.washington.edu": {
              "repository_directory": "client/",
              "repository_mirrors" : {
                  "mirror1": {
                      "url_prefix": "http://seattle-tuf.cs.washington.edu",
                      "metadata_path": "metadata",
                      "targets_path": "targets",
                      "confined_target_dirs": [ "" ]
                  }
              },
              ("target_paths": [
                  { ".*/(simple/\\w+)/$": "{0}/index.html" },
                  { ".*/(packages/.+)$": "{0}" }
              ],
              "ssl_certificates": "cacert.pem")
          }
      }
  }

  "target_paths" is optional: If you do not tell TUF to selectively match
  paths with regular expressions, TUF will work over any path under the given
  network location. However, if you do specify it, you are then telling TUF
  how to transform a specified path into another one, and TUF will *not*
  recognize any unspecified path for the given network location.

  Unless any "url_prefix" begins with "https://", "ssl_certificates" is
  optional; it must specify certificates bundled as PEM (RFC 1422).
  """

  INVALID_TUF_CONFIGURATION = "Invalid configuration for {network_location}!"
  INVALID_TUF_INTERPOSITION_JSON = "Invalid configuration in {filename}!"
  NO_CONFIGURATIONS = "No configurations found in configuration in {filename}!"

  try:
    with open(filename) as tuf_interposition_json:
      tuf_interpositions = json.load(tuf_interposition_json)
      configurations = tuf_interpositions.get("configurations", {})

      if len(configurations) == 0:
        raise InvalidConfiguration(NO_CONFIGURATIONS.format(filename=filename))

      else:
        for network_location, configuration in configurations.iteritems():
          try:
            configuration_parser = ConfigurationParser(network_location,
              configuration, parent_repository_directory=parent_repository_directory,
              parent_ssl_certificates_directory=parent_ssl_certificates_directory)

            configuration = configuration_parser.parse()
            __updater_controller.add(configuration)

          except:
            Logger.exception(INVALID_TUF_CONFIGURATION.format(network_location=network_location))
            raise

  except:
    Logger.exception(INVALID_TUF_INTERPOSITION_JSON.format(filename=filename))
    raise
Beispiel #14
0
            day_list.append(current_day)
            current_day = current_day + timedelta(days=1)
        return day_list

    @staticmethod
    def __build_measurement_doc(source: str, attribute: str, date_origin: datetime, value: float) -> dict:
        return {
            'source': source,
            'attribute': attribute,
            'timestamp': date_origin + random.random() * timedelta(days=1),
            'value': value
        }


if __name__ == '__main__':
    arg_parser = ArgumentParser()
    arg_parser.add_argument('-c', '--config',
                            required=True,
                            dest='config_file',
                            help='Path to the input configuration file',
                            metavar='FILE')
    arg_parser.add_argument('-o', '--output',
                            required=True,
                            dest='validation_file',
                            help='Output file with stats from the input data, mey be used later to check if data '
                                 'processor calculate the statistical data correctly',
                            metavar='FILE')
    args = arg_parser.parse_args()
    configuration = ConfigurationParser.parse_from_file(args.config_file)
    DataProcessor().process_data(configuration, args.validation_file)