def __read_configuration(configuration_handler, filename="tuf.interposition.json", parent_repository_directory=None, parent_ssl_certificates_directory=None): """ A generic function to read TUF interposition configurations off a file, and then handle those configurations with a given function. configuration_handler must be a function which accepts a tuf.interposition.Configuration instance. Returns the parsed configurations as a dictionary of configurations indexed by hostnames.""" INVALID_TUF_CONFIGURATION = "Invalid configuration for {network_location}!" INVALID_TUF_INTERPOSITION_JSON = "Invalid configuration in {filename}!" NO_CONFIGURATIONS = "No configurations found in configuration in {filename}!" # Configurations indexed by hostnames. parsed_configurations = {} try: with open(filename) as tuf_interposition_json: tuf_interpositions = json.load(tuf_interposition_json) configurations = tuf_interpositions.get("configurations", {}) if len(configurations) == 0: raise InvalidConfiguration( NO_CONFIGURATIONS.format(filename=filename)) else: for network_location, configuration in configurations.iteritems( ): try: configuration_parser = ConfigurationParser( network_location, configuration, parent_repository_directory= parent_repository_directory, parent_ssl_certificates_directory= parent_ssl_certificates_directory) configuration = configuration_parser.parse() configuration_handler(configuration) parsed_configurations[ configuration.hostname] = configuration except: Logger.exception( INVALID_TUF_CONFIGURATION.format( network_location=network_location)) raise except: Logger.exception( INVALID_TUF_INTERPOSITION_JSON.format(filename=filename)) raise else: return parsed_configurations
def auto_updater(self): configuration = ConfigurationParser('dashboard.conf') Timer(configuration.refresh_rate(), self.auto_updater).start() now = datetime.now() self.last_update = str(now.hour).zfill(2) + ":" + str(now.minute % 60).zfill(2) # Load jenkins info url, user, password, key = configuration.jenkins_credentials() zentyal_jenkins = JenkinsHelper(url, user, password, key, configuration) self.jobs = zentyal_jenkins.get_jobs() self.components = zentyal_jenkins.get_components() # Load public tracker info url, key = configuration.public_tracker_credentials() self.public_tracker = RedmineHelper(url, key) # Load github pull requests client_id, client_secret = configuration.github_credentials() oauth_token, retest_message = configuration.github_retest() self.github = GitHubHelper(client_id, client_secret, oauth_token, retest_message) repositories = configuration.github_repositories() for repo in repositories: self.github.add_pull_requests(repo['organization'], repo['repository']) self.pullrequests = self.github.get_pull_requests() self.base_branchs = self.github.base_branchs() # Load packages data repo_path = configuration.zentyal_repo_path() zentyal_git = ZentyalGitHelper(repo_path, self.pullrequests) self.pending_packages = zentyal_git.get_pending_packages()
def __read_configuration(configuration_handler, filename="tuf.interposition.json", parent_repository_directory=None, parent_ssl_certificates_directory=None): """ A generic function to read TUF interposition configurations off a file, and then handle those configurations with a given function. configuration_handler must be a function which accepts a tuf.interposition.Configuration instance. Returns the parsed configurations as a dictionary of configurations indexed by hostnames.""" INVALID_TUF_CONFIGURATION = "Invalid configuration for {network_location}!" INVALID_TUF_INTERPOSITION_JSON = "Invalid configuration in {filename}!" NO_CONFIGURATIONS = "No configurations found in configuration in {filename}!" # Configurations indexed by hostnames. parsed_configurations = {} try: with open(filename) as tuf_interposition_json: tuf_interpositions = json.load(tuf_interposition_json) configurations = tuf_interpositions.get("configurations", {}) if len(configurations) == 0: raise InvalidConfiguration(NO_CONFIGURATIONS.format(filename=filename)) else: for network_location, configuration in configurations.iteritems(): try: configuration_parser = ConfigurationParser(network_location, configuration, parent_repository_directory=parent_repository_directory, parent_ssl_certificates_directory=parent_ssl_certificates_directory) configuration = configuration_parser.parse() configuration_handler(configuration) parsed_configurations[configuration.hostname] = configuration except: Logger.exception(INVALID_TUF_CONFIGURATION.format(network_location=network_location)) raise except: Logger.exception(INVALID_TUF_INTERPOSITION_JSON.format(filename=filename)) raise else: return parsed_configurations
def sym_run(stage: (int, str) = "viz"): # pragma: no cover if isinstance(stage, str): stage = { "viz": 0, "pseudo": 1, "real": 2, "cli": 3, }[stage] if stage <= 1: cfg = Configuration( hyper_params={ "ku_num": 835, "latent_dim": 600, "hidden_num": 900, }, nettype="EmbedDKT", ) net = get_net(**cfg.hyper_params) if stage == 0: # ############################## Net Visualization ########################### net_viz(net, cfg, False) else: # ############################## Pseudo Test ################################# pseudo_numerical_check(net, cfg) elif stage == 2: # ################################# Simple Train ############################### import mxnet as mx train( "$data_dir/train.json", "$data_dir/test.json", dataset="assistment_2009_2010", ctx=mx.gpu(2), optimizer_params={"learning_rate": 0.001}, hyper_params={ "ku_num": 146, "hidden_num": 200, "dropout": 0.5 }, batch_size=16, ) elif stage == 3: # ################################# CLI ########################### cfg_parser = ConfigurationParser(Configuration, commands=[train]) cfg_kwargs = cfg_parser() assert "subcommand" in cfg_kwargs subcommand = cfg_kwargs["subcommand"] del cfg_kwargs["subcommand"] print(cfg_kwargs) eval("%s" % subcommand)(**cfg_kwargs) else: raise TypeError
def sym_run(stage: (int, str) = "viz"): # pragma: no cover if isinstance(stage, str): stage = { "viz": 0, "pseudo": 1, "real": 2, "cli": 3, }[stage] if stage <= 1: cfg = Configuration(hyper_params={}) net = get_net(**cfg.hyper_params) net.initialize() if stage == 0: # ############################## Net Visualization ########################### net_viz(net, cfg, False) else: # ############################## Pseudo Test ################################# pseudo_numerical_check(net, cfg) elif stage == 2: # ################################# Simple Train ############################### import mxnet as mx train( "$data_dir/train.csv", "$data_dir/test.csv", ctx=mx.cpu(), optimizer_params={ "learning_rate": 0.01, "clip_gradient": 1, }, dataset="a0910", # init_params_update={"initial_user_item": False}, # workspace="random", workspace="init", # params_save=True ) elif stage == 3: # ################################# CLI ########################### cfg_parser = ConfigurationParser(Configuration, commands=[train]) cfg_kwargs = cfg_parser() assert "subcommand" in cfg_kwargs subcommand = cfg_kwargs["subcommand"] del cfg_kwargs["subcommand"] print(cfg_kwargs) eval("%s" % subcommand)(**cfg_kwargs) else: raise TypeError
def __read_configuration(configuration_handler, filename="tuf.interposition.json", parent_repository_directory=None, parent_ssl_certificates_directory=None): """ A generic function to read a TUF interposition configuration off the disk, and handle it. configuration_handler must be a function which accepts a tuf.interposition.Configuration instance.""" INVALID_TUF_CONFIGURATION = "Invalid configuration for {network_location}!" INVALID_TUF_INTERPOSITION_JSON = "Invalid configuration in {filename}!" NO_CONFIGURATIONS = "No configurations found in configuration in {filename}!" try: with open(filename) as tuf_interposition_json: tuf_interpositions = json.load(tuf_interposition_json) configurations = tuf_interpositions.get("configurations", {}) if len(configurations) == 0: raise InvalidConfiguration(NO_CONFIGURATIONS.format(filename=filename)) else: for network_location, configuration in configurations.iteritems(): try: configuration_parser = ConfigurationParser(network_location, configuration, parent_repository_directory=parent_repository_directory, parent_ssl_certificates_directory=parent_ssl_certificates_directory) configuration = configuration_parser.parse() configuration_handler(configuration) except: Logger.exception(INVALID_TUF_CONFIGURATION.format(network_location=network_location)) raise except: Logger.exception(INVALID_TUF_INTERPOSITION_JSON.format(filename=filename)) raise
def sym_run(stage: (int, str) = "pesudo"): # pragma: no cover if isinstance(stage, str): stage = { "pseudo": 0, "real": 1, "cli": 2, }[stage] if stage == 0: # ############################## Pesudo Test ################################# cfg = Configuration( hyper_params={ "ku_num": 835, "hidden_num": 835, }, ctx="cuda:0,1,2", ) net = get_net(**cfg.hyper_params) pseudo_numerical_check(net, cfg) elif stage == 1: # ################################# Simple Train ############################### train( "$data_dir/train", "$data_dir/test", dataset="junyi", ctx="cuda:0", optimizer_params={"lr": 0.001}, hyper_params={ "ku_num": 835, "hidden_num": 835 }, batch_size=16, ) elif stage == 2: # ################################# CLI ########################### cfg_parser = ConfigurationParser(Configuration, commands=[train]) cfg_kwargs = cfg_parser() assert "subcommand" in cfg_kwargs subcommand = cfg_kwargs["subcommand"] del cfg_kwargs["subcommand"] print(cfg_kwargs) eval("%s" % subcommand)(**cfg_kwargs) else: raise TypeError
def sym_run(stage: (int, str) = "viz"): # pragma: no cover if isinstance(stage, str): stage = { "viz": 0, "pseudo": 1, "real": 2, "cli": 3, }[stage] if stage <= 1: cfg = Configuration( hyper_params=dict( ku_num=835, key_embedding_dim=50, value_embedding_dim=200, hidden_num=900, key_memory_size=20, value_memory_size=20, key_memory_state_dim=50, value_memory_state_dim=200, ) ) net = get_net(**cfg.hyper_params) if stage == 0: # ############################## Net Visualization ########################### net_viz(net, cfg, False) else: # ############################## Pseudo Test ################################# pseudo_numerical_check(net, cfg) elif stage == 2: # ################################# Simple Train ############################### import mxnet as mx train( "$data_dir/train.json", "$data_dir/test.json", dataset="junyi", ctx=mx.gpu(), optimizer_params={ "learning_rate": 0.001 }, hyper_params=dict( ku_num=835, key_embedding_dim=200, value_embedding_dim=200, hidden_num=835, key_memory_size=40, dropout=0.5, ), batch_size=16, root="../../../", data_dir="$root_data_dir", end_epoch=10, ) elif stage == 3: # ################################# CLI ########################### cfg_parser = ConfigurationParser(Configuration, commands=[train]) cfg_kwargs = cfg_parser() assert "subcommand" in cfg_kwargs subcommand = cfg_kwargs["subcommand"] del cfg_kwargs["subcommand"] print(cfg_kwargs) eval("%s" % subcommand)(**cfg_kwargs) else: raise TypeError
def sym_run(stage: (int, str) = "viz"): # pragma: no cover if isinstance(stage, str): stage = { "viz": 0, "pseudo": 1, "real": 2, "cli": 3, }[stage] if stage <= 1: cfg = Configuration( hyper_params={ "model_type": "wclstm", "class_num": 32, "embedding_dim": 256, } ) embedding_size = { "w": 100, "c": 100, } net = get_net(embedding_size=embedding_size, **cfg.hyper_params) if stage == 0: # ############################## Net Visualization ########################### net_viz(net, cfg, False) else: # ############################## Pseudo Test ################################# pseudo_numerical_check(net, cfg) elif stage == 2: # ################################# Simple Train ############################### import mxnet as mx train( "$data_dir/train.json", "$data_dir/test.json", "w:$vec_dir/word.vec.dat,c:$vec_dir/char.vec.dat", ctx=mx.gpu(), hyper_params={ "model_type": "wclstm", "class_num": 32, "embedding_dim": 256, }, root="../../../..", dataset="ctc32", data_dir="$root_data_dir", end_epoch=1, ) elif stage == 3: # ################################# CLI ########################### cfg_parser = ConfigurationParser(Configuration, commands=[train]) cfg_kwargs = cfg_parser() assert "subcommand" in cfg_kwargs subcommand = cfg_kwargs["subcommand"] del cfg_kwargs["subcommand"] print(cfg_kwargs) eval("%s" % subcommand)(**cfg_kwargs) else: raise TypeError
def sym_run(stage: (int, str) = "viz"): # pragma: no cover if isinstance(stage, str): stage = { "viz": 0, "pseudo": 1, "real": 2, "cli": 3, }[stage] if stage <= 1: cfg = Configuration( hyper_params=dict( ku_num=835, ) ) net = get_net(**cfg.hyper_params) if stage == 0: # ############################## Net Visualization ########################### net_viz(net, cfg, False) else: # ############################## Pseudo Test ################################# pseudo_numerical_check(net, cfg) elif stage == 2: # ################################# Simple Train ############################### import mxnet as mx train( "$data_dir/train.json", "$data_dir/test.json", ctx=mx.gpu(), optimizer_params={ "learning_rate": 0.001 }, dump=True, data_dir="$root_data_dir", # end_epoch=20, # root="../../../", # dataset="assistment_2009_2010", # hyper_params=dict( # ku_num=124, # hidden_num=32, # latent_dim=32, # dropout=0.5, # ), end_epoch=10, hyper_params=dict( ku_num=835, hidden_num=32, latent_dim=32, dropout=0.5, ), dataset="junyi", root="/data/tongshiwei/KT", root_data_dir="$root/$dataset", ) elif stage == 3: # ################################# CLI ########################### cfg_parser = ConfigurationParser(Configuration, commands=[train]) cfg_kwargs = cfg_parser() assert "subcommand" in cfg_kwargs subcommand = cfg_kwargs["subcommand"] del cfg_kwargs["subcommand"] print(cfg_kwargs) eval("%s" % subcommand)(**cfg_kwargs) else: raise TypeError
def sym_run(stage: (int, str) = "viz"): # pragma: no cover if isinstance(stage, str): stage = { "viz": 0, "pseudo": 1, "real": 2, "cli": 3, }[stage] if stage <= 1: cfg = Configuration(hyper_params={ "user_num": 1000, "item_num": 100, "vec_dim": 100, "op": "mlp" }, eval_params={ "unlabeled_value": 0, "k": [1, 3], "pointwise": True, }) net = get_net(**cfg.hyper_params) if stage == 0: # ############################## Net Visualization ########################### net_viz(net, cfg, False) else: # ############################## Pseudo Test ################################# pseudo_numerical_check(net, cfg) elif stage == 2: # ################################# Simple Train ############################### import mxnet as mx train( "$data_dir/ml-1m/train.jsonl", "$data_dir/ml-1m/test.jsonl", hyper_params={ "num_a": 6040, "num_b": 3900, "vec_dim": 128, "op": "linear" }, root_data_dir="../../../../", optimizer_params={"learning_rate": 0.001}, # ctx=[mx.gpu(3)], ctx=[mx.gpu(5)], batch_size=16, # optimizer_params={ # "learning_rate": 0.01 # }, # ctx=[mx.gpu(5), mx.gpu(6), mx.gpu(7), mx.gpu(8)], # batch_size=256, ) elif stage == 3: # ################################# CLI ########################### cfg_parser = ConfigurationParser(Configuration, commands=[train]) cfg_kwargs = cfg_parser() assert "subcommand" in cfg_kwargs subcommand = cfg_kwargs["subcommand"] del cfg_kwargs["subcommand"] print(cfg_kwargs) eval("%s" % subcommand)(**cfg_kwargs) else: raise TypeError
def check_auth(request_username, request_password): configuration = ConfigurationParser('dashboard.conf') username, password = configuration.authentication_credentials() return request_username == username and request_password == password
def configure(filename="tuf.interposition.json", parent_repository_directory=None, parent_ssl_certificates_directory=None): """ The optional parent_repository_directory parameter is used to specify the containing parent directory of the "repository_directory" specified in a configuration for *all* network locations, because sometimes the absolute location of the "repository_directory" is only known at runtime. If you need to specify a different parent_repository_directory for other network locations, simply call this method again with different parameters. Ditto for the optional parent_ssl_certificates_directory parameter. Example of a TUF interposition configuration JSON object: { "configurations": { "seattle.cs.washington.edu": { "repository_directory": "client/", "repository_mirrors" : { "mirror1": { "url_prefix": "http://seattle-tuf.cs.washington.edu", "metadata_path": "metadata", "targets_path": "targets", "confined_target_dirs": [ "" ] } }, ("target_paths": [ { ".*/(simple/\\w+)/$": "{0}/index.html" }, { ".*/(packages/.+)$": "{0}" } ], "ssl_certificates": "cacert.pem") } } } "target_paths" is optional: If you do not tell TUF to selectively match paths with regular expressions, TUF will work over any path under the given network location. However, if you do specify it, you are then telling TUF how to transform a specified path into another one, and TUF will *not* recognize any unspecified path for the given network location. Unless any "url_prefix" begins with "https://", "ssl_certificates" is optional; it must specify certificates bundled as PEM (RFC 1422). """ INVALID_TUF_CONFIGURATION = "Invalid configuration for {network_location}!" INVALID_TUF_INTERPOSITION_JSON = "Invalid configuration in {filename}!" NO_CONFIGURATIONS = "No configurations found in configuration in {filename}!" try: with open(filename) as tuf_interposition_json: tuf_interpositions = json.load(tuf_interposition_json) configurations = tuf_interpositions.get("configurations", {}) if len(configurations) == 0: raise InvalidConfiguration(NO_CONFIGURATIONS.format(filename=filename)) else: for network_location, configuration in configurations.iteritems(): try: configuration_parser = ConfigurationParser(network_location, configuration, parent_repository_directory=parent_repository_directory, parent_ssl_certificates_directory=parent_ssl_certificates_directory) configuration = configuration_parser.parse() __updater_controller.add(configuration) except: Logger.exception(INVALID_TUF_CONFIGURATION.format(network_location=network_location)) raise except: Logger.exception(INVALID_TUF_INTERPOSITION_JSON.format(filename=filename)) raise
day_list.append(current_day) current_day = current_day + timedelta(days=1) return day_list @staticmethod def __build_measurement_doc(source: str, attribute: str, date_origin: datetime, value: float) -> dict: return { 'source': source, 'attribute': attribute, 'timestamp': date_origin + random.random() * timedelta(days=1), 'value': value } if __name__ == '__main__': arg_parser = ArgumentParser() arg_parser.add_argument('-c', '--config', required=True, dest='config_file', help='Path to the input configuration file', metavar='FILE') arg_parser.add_argument('-o', '--output', required=True, dest='validation_file', help='Output file with stats from the input data, mey be used later to check if data ' 'processor calculate the statistical data correctly', metavar='FILE') args = arg_parser.parse_args() configuration = ConfigurationParser.parse_from_file(args.config_file) DataProcessor().process_data(configuration, args.validation_file)