def __read_configuration(configuration_handler,
filename="tuf.interposition.json",
parent_repository_directory=None,
parent_ssl_certificates_directory=None):
"""
A generic function to read TUF interposition configurations off a file, and
then handle those configurations with a given function. configuration_handler
must be a function which accepts a tuf.interposition.Configuration
instance.
Returns the parsed configurations as a dictionary of configurations indexed
by hostnames."""
INVALID_TUF_CONFIGURATION = "Invalid configuration for {network_location}!"
INVALID_TUF_INTERPOSITION_JSON = "Invalid configuration in {filename}!"
NO_CONFIGURATIONS = "No configurations found in configuration in {filename}!"
# Configurations indexed by hostnames.
parsed_configurations = {}
try:
with open(filename) as tuf_interposition_json:
tuf_interpositions = json.load(tuf_interposition_json)
configurations = tuf_interpositions.get("configurations", {})
if len(configurations) == 0:
raise InvalidConfiguration(
NO_CONFIGURATIONS.format(filename=filename))
else:
for network_location, configuration in configurations.iteritems(
):
try:
configuration_parser = ConfigurationParser(
network_location,
configuration,
parent_repository_directory=
parent_repository_directory,
parent_ssl_certificates_directory=
parent_ssl_certificates_directory)
configuration = configuration_parser.parse()
configuration_handler(configuration)
parsed_configurations[
configuration.hostname] = configuration
except:
Logger.exception(
INVALID_TUF_CONFIGURATION.format(
network_location=network_location))
raise
except:
Logger.exception(
INVALID_TUF_INTERPOSITION_JSON.format(filename=filename))
raise
else:
return parsed_configurations
def auto_updater(self):
configuration = ConfigurationParser('dashboard.conf')
Timer(configuration.refresh_rate(), self.auto_updater).start()
now = datetime.now()
self.last_update = str(now.hour).zfill(2) + ":" + str(now.minute % 60).zfill(2)
# Load jenkins info
url, user, password, key = configuration.jenkins_credentials()
zentyal_jenkins = JenkinsHelper(url, user, password, key, configuration)
self.jobs = zentyal_jenkins.get_jobs()
self.components = zentyal_jenkins.get_components()
# Load public tracker info
url, key = configuration.public_tracker_credentials()
self.public_tracker = RedmineHelper(url, key)
# Load github pull requests
client_id, client_secret = configuration.github_credentials()
oauth_token, retest_message = configuration.github_retest()
self.github = GitHubHelper(client_id, client_secret, oauth_token, retest_message)
repositories = configuration.github_repositories()
for repo in repositories:
self.github.add_pull_requests(repo['organization'], repo['repository'])
self.pullrequests = self.github.get_pull_requests()
self.base_branchs = self.github.base_branchs()
# Load packages data
repo_path = configuration.zentyal_repo_path()
zentyal_git = ZentyalGitHelper(repo_path, self.pullrequests)
self.pending_packages = zentyal_git.get_pending_packages()
def __read_configuration(configuration_handler,
filename="tuf.interposition.json",
parent_repository_directory=None,
parent_ssl_certificates_directory=None):
"""
A generic function to read TUF interposition configurations off a file, and
then handle those configurations with a given function. configuration_handler
must be a function which accepts a tuf.interposition.Configuration
instance.
Returns the parsed configurations as a dictionary of configurations indexed
by hostnames."""
INVALID_TUF_CONFIGURATION = "Invalid configuration for {network_location}!"
INVALID_TUF_INTERPOSITION_JSON = "Invalid configuration in {filename}!"
NO_CONFIGURATIONS = "No configurations found in configuration in {filename}!"
# Configurations indexed by hostnames.
parsed_configurations = {}
try:
with open(filename) as tuf_interposition_json:
tuf_interpositions = json.load(tuf_interposition_json)
configurations = tuf_interpositions.get("configurations", {})
if len(configurations) == 0:
raise InvalidConfiguration(NO_CONFIGURATIONS.format(filename=filename))
else:
for network_location, configuration in configurations.iteritems():
try:
configuration_parser = ConfigurationParser(network_location,
configuration, parent_repository_directory=parent_repository_directory,
parent_ssl_certificates_directory=parent_ssl_certificates_directory)
configuration = configuration_parser.parse()
configuration_handler(configuration)
parsed_configurations[configuration.hostname] = configuration
except:
Logger.exception(INVALID_TUF_CONFIGURATION.format(network_location=network_location))
raise
except:
Logger.exception(INVALID_TUF_INTERPOSITION_JSON.format(filename=filename))
raise
else:
return parsed_configurations
def sym_run(stage: (int, str) = "viz"): # pragma: no cover
if isinstance(stage, str):
stage = {
"viz": 0,
"pseudo": 1,
"real": 2,
"cli": 3,
}[stage]
if stage <= 1:
cfg = Configuration(
hyper_params={
"ku_num": 835,
"latent_dim": 600,
"hidden_num": 900,
},
nettype="EmbedDKT",
)
net = get_net(**cfg.hyper_params)
if stage == 0:
# ############################## Net Visualization ###########################
net_viz(net, cfg, False)
else:
# ############################## Pseudo Test #################################
pseudo_numerical_check(net, cfg)
elif stage == 2:
# ################################# Simple Train ###############################
import mxnet as mx
train(
"$data_dir/train.json",
"$data_dir/test.json",
dataset="assistment_2009_2010",
ctx=mx.gpu(2),
optimizer_params={"learning_rate": 0.001},
hyper_params={
"ku_num": 146,
"hidden_num": 200,
"dropout": 0.5
},
batch_size=16,
)
elif stage == 3:
# ################################# CLI ###########################
cfg_parser = ConfigurationParser(Configuration, commands=[train])
cfg_kwargs = cfg_parser()
assert "subcommand" in cfg_kwargs
subcommand = cfg_kwargs["subcommand"]
del cfg_kwargs["subcommand"]
print(cfg_kwargs)
eval("%s" % subcommand)(**cfg_kwargs)
else:
raise TypeError
def sym_run(stage: (int, str) = "viz"): # pragma: no cover
if isinstance(stage, str):
stage = {
"viz": 0,
"pseudo": 1,
"real": 2,
"cli": 3,
}[stage]
if stage <= 1:
cfg = Configuration(hyper_params={})
net = get_net(**cfg.hyper_params)
net.initialize()
if stage == 0:
# ############################## Net Visualization ###########################
net_viz(net, cfg, False)
else:
# ############################## Pseudo Test #################################
pseudo_numerical_check(net, cfg)
elif stage == 2:
# ################################# Simple Train ###############################
import mxnet as mx
train(
"$data_dir/train.csv",
"$data_dir/test.csv",
ctx=mx.cpu(),
optimizer_params={
"learning_rate": 0.01,
"clip_gradient": 1,
},
dataset="a0910",
# init_params_update={"initial_user_item": False},
# workspace="random",
workspace="init",
# params_save=True
)
elif stage == 3:
# ################################# CLI ###########################
cfg_parser = ConfigurationParser(Configuration, commands=[train])
cfg_kwargs = cfg_parser()
assert "subcommand" in cfg_kwargs
subcommand = cfg_kwargs["subcommand"]
del cfg_kwargs["subcommand"]
print(cfg_kwargs)
eval("%s" % subcommand)(**cfg_kwargs)
else:
raise TypeError
def __read_configuration(configuration_handler,
filename="tuf.interposition.json",
parent_repository_directory=None,
parent_ssl_certificates_directory=None):
"""
A generic function to read a TUF interposition configuration off the disk,
and handle it. configuration_handler must be a function which accepts a
tuf.interposition.Configuration instance."""
INVALID_TUF_CONFIGURATION = "Invalid configuration for {network_location}!"
INVALID_TUF_INTERPOSITION_JSON = "Invalid configuration in {filename}!"
NO_CONFIGURATIONS = "No configurations found in configuration in {filename}!"
try:
with open(filename) as tuf_interposition_json:
tuf_interpositions = json.load(tuf_interposition_json)
configurations = tuf_interpositions.get("configurations", {})
if len(configurations) == 0:
raise InvalidConfiguration(NO_CONFIGURATIONS.format(filename=filename))
else:
for network_location, configuration in configurations.iteritems():
try:
configuration_parser = ConfigurationParser(network_location,
configuration, parent_repository_directory=parent_repository_directory,
parent_ssl_certificates_directory=parent_ssl_certificates_directory)
configuration = configuration_parser.parse()
configuration_handler(configuration)
except:
Logger.exception(INVALID_TUF_CONFIGURATION.format(network_location=network_location))
raise
except:
Logger.exception(INVALID_TUF_INTERPOSITION_JSON.format(filename=filename))
raise
def sym_run(stage: (int, str) = "pesudo"): # pragma: no cover
if isinstance(stage, str):
stage = {
"pseudo": 0,
"real": 1,
"cli": 2,
}[stage]
if stage == 0:
# ############################## Pesudo Test #################################
cfg = Configuration(
hyper_params={
"ku_num": 835,
"hidden_num": 835,
},
ctx="cuda:0,1,2",
)
net = get_net(**cfg.hyper_params)
pseudo_numerical_check(net, cfg)
elif stage == 1:
# ################################# Simple Train ###############################
train(
"$data_dir/train",
"$data_dir/test",
dataset="junyi",
ctx="cuda:0",
optimizer_params={"lr": 0.001},
hyper_params={
"ku_num": 835,
"hidden_num": 835
},
batch_size=16,
)
elif stage == 2:
# ################################# CLI ###########################
cfg_parser = ConfigurationParser(Configuration, commands=[train])
cfg_kwargs = cfg_parser()
assert "subcommand" in cfg_kwargs
subcommand = cfg_kwargs["subcommand"]
del cfg_kwargs["subcommand"]
print(cfg_kwargs)
eval("%s" % subcommand)(**cfg_kwargs)
else:
raise TypeError
def sym_run(stage: (int, str) = "viz"): # pragma: no cover
if isinstance(stage, str):
stage = {
"viz": 0,
"pseudo": 1,
"real": 2,
"cli": 3,
}[stage]
if stage <= 1:
cfg = Configuration(
hyper_params=dict(
ku_num=835,
key_embedding_dim=50,
value_embedding_dim=200,
hidden_num=900,
key_memory_size=20,
value_memory_size=20,
key_memory_state_dim=50,
value_memory_state_dim=200,
)
)
net = get_net(**cfg.hyper_params)
if stage == 0:
# ############################## Net Visualization ###########################
net_viz(net, cfg, False)
else:
# ############################## Pseudo Test #################################
pseudo_numerical_check(net, cfg)
elif stage == 2:
# ################################# Simple Train ###############################
import mxnet as mx
train(
"$data_dir/train.json",
"$data_dir/test.json",
dataset="junyi",
ctx=mx.gpu(),
optimizer_params={
"learning_rate": 0.001
},
hyper_params=dict(
ku_num=835,
key_embedding_dim=200,
value_embedding_dim=200,
hidden_num=835,
key_memory_size=40,
dropout=0.5,
),
batch_size=16,
root="../../../",
data_dir="$root_data_dir",
end_epoch=10,
)
elif stage == 3:
# ################################# CLI ###########################
cfg_parser = ConfigurationParser(Configuration, commands=[train])
cfg_kwargs = cfg_parser()
assert "subcommand" in cfg_kwargs
subcommand = cfg_kwargs["subcommand"]
del cfg_kwargs["subcommand"]
print(cfg_kwargs)
eval("%s" % subcommand)(**cfg_kwargs)
else:
raise TypeError
def sym_run(stage: (int, str) = "viz"): # pragma: no cover
if isinstance(stage, str):
stage = {
"viz": 0,
"pseudo": 1,
"real": 2,
"cli": 3,
}[stage]
if stage <= 1:
cfg = Configuration(
hyper_params={
"model_type": "wclstm",
"class_num": 32,
"embedding_dim": 256,
}
)
embedding_size = {
"w": 100,
"c": 100,
}
net = get_net(embedding_size=embedding_size, **cfg.hyper_params)
if stage == 0:
# ############################## Net Visualization ###########################
net_viz(net, cfg, False)
else:
# ############################## Pseudo Test #################################
pseudo_numerical_check(net, cfg)
elif stage == 2:
# ################################# Simple Train ###############################
import mxnet as mx
train(
"$data_dir/train.json",
"$data_dir/test.json",
"w:$vec_dir/word.vec.dat,c:$vec_dir/char.vec.dat",
ctx=mx.gpu(),
hyper_params={
"model_type": "wclstm",
"class_num": 32,
"embedding_dim": 256,
},
root="../../../..",
dataset="ctc32",
data_dir="$root_data_dir",
end_epoch=1,
)
elif stage == 3:
# ################################# CLI ###########################
cfg_parser = ConfigurationParser(Configuration, commands=[train])
cfg_kwargs = cfg_parser()
assert "subcommand" in cfg_kwargs
subcommand = cfg_kwargs["subcommand"]
del cfg_kwargs["subcommand"]
print(cfg_kwargs)
eval("%s" % subcommand)(**cfg_kwargs)
else:
raise TypeError
def sym_run(stage: (int, str) = "viz"): # pragma: no cover
if isinstance(stage, str):
stage = {
"viz": 0,
"pseudo": 1,
"real": 2,
"cli": 3,
}[stage]
if stage <= 1:
cfg = Configuration(
hyper_params=dict(
ku_num=835,
)
)
net = get_net(**cfg.hyper_params)
if stage == 0:
# ############################## Net Visualization ###########################
net_viz(net, cfg, False)
else:
# ############################## Pseudo Test #################################
pseudo_numerical_check(net, cfg)
elif stage == 2:
# ################################# Simple Train ###############################
import mxnet as mx
train(
"$data_dir/train.json",
"$data_dir/test.json",
ctx=mx.gpu(),
optimizer_params={
"learning_rate": 0.001
},
dump=True,
data_dir="$root_data_dir",
# end_epoch=20,
# root="../../../",
# dataset="assistment_2009_2010",
# hyper_params=dict(
# ku_num=124,
# hidden_num=32,
# latent_dim=32,
# dropout=0.5,
# ),
end_epoch=10,
hyper_params=dict(
ku_num=835,
hidden_num=32,
latent_dim=32,
dropout=0.5,
),
dataset="junyi",
root="/data/tongshiwei/KT",
root_data_dir="$root/$dataset",
)
elif stage == 3:
# ################################# CLI ###########################
cfg_parser = ConfigurationParser(Configuration, commands=[train])
cfg_kwargs = cfg_parser()
assert "subcommand" in cfg_kwargs
subcommand = cfg_kwargs["subcommand"]
del cfg_kwargs["subcommand"]
print(cfg_kwargs)
eval("%s" % subcommand)(**cfg_kwargs)
else:
raise TypeError
def sym_run(stage: (int, str) = "viz"): # pragma: no cover
if isinstance(stage, str):
stage = {
"viz": 0,
"pseudo": 1,
"real": 2,
"cli": 3,
}[stage]
if stage <= 1:
cfg = Configuration(hyper_params={
"user_num": 1000,
"item_num": 100,
"vec_dim": 100,
"op": "mlp"
},
eval_params={
"unlabeled_value": 0,
"k": [1, 3],
"pointwise": True,
})
net = get_net(**cfg.hyper_params)
if stage == 0:
# ############################## Net Visualization ###########################
net_viz(net, cfg, False)
else:
# ############################## Pseudo Test #################################
pseudo_numerical_check(net, cfg)
elif stage == 2:
# ################################# Simple Train ###############################
import mxnet as mx
train(
"$data_dir/ml-1m/train.jsonl",
"$data_dir/ml-1m/test.jsonl",
hyper_params={
"num_a": 6040,
"num_b": 3900,
"vec_dim": 128,
"op": "linear"
},
root_data_dir="../../../../",
optimizer_params={"learning_rate": 0.001},
# ctx=[mx.gpu(3)],
ctx=[mx.gpu(5)],
batch_size=16,
# optimizer_params={
# "learning_rate": 0.01
# },
# ctx=[mx.gpu(5), mx.gpu(6), mx.gpu(7), mx.gpu(8)],
# batch_size=256,
)
elif stage == 3:
# ################################# CLI ###########################
cfg_parser = ConfigurationParser(Configuration, commands=[train])
cfg_kwargs = cfg_parser()
assert "subcommand" in cfg_kwargs
subcommand = cfg_kwargs["subcommand"]
del cfg_kwargs["subcommand"]
print(cfg_kwargs)
eval("%s" % subcommand)(**cfg_kwargs)
else:
raise TypeError
def check_auth(request_username, request_password):
configuration = ConfigurationParser('dashboard.conf')
username, password = configuration.authentication_credentials()
return request_username == username and request_password == password
def configure(filename="tuf.interposition.json",
parent_repository_directory=None,
parent_ssl_certificates_directory=None):
"""
The optional parent_repository_directory parameter is used to specify the
containing parent directory of the "repository_directory" specified in a
configuration for *all* network locations, because sometimes the absolute
location of the "repository_directory" is only known at runtime. If you
need to specify a different parent_repository_directory for other
network locations, simply call this method again with different parameters.
Ditto for the optional parent_ssl_certificates_directory parameter.
Example of a TUF interposition configuration JSON object:
{
"configurations": {
"seattle.cs.washington.edu": {
"repository_directory": "client/",
"repository_mirrors" : {
"mirror1": {
"url_prefix": "http://seattle-tuf.cs.washington.edu",
"metadata_path": "metadata",
"targets_path": "targets",
"confined_target_dirs": [ "" ]
}
},
("target_paths": [
{ ".*/(simple/\\w+)/$": "{0}/index.html" },
{ ".*/(packages/.+)$": "{0}" }
],
"ssl_certificates": "cacert.pem")
}
}
}
"target_paths" is optional: If you do not tell TUF to selectively match
paths with regular expressions, TUF will work over any path under the given
network location. However, if you do specify it, you are then telling TUF
how to transform a specified path into another one, and TUF will *not*
recognize any unspecified path for the given network location.
Unless any "url_prefix" begins with "https://", "ssl_certificates" is
optional; it must specify certificates bundled as PEM (RFC 1422).
"""
INVALID_TUF_CONFIGURATION = "Invalid configuration for {network_location}!"
INVALID_TUF_INTERPOSITION_JSON = "Invalid configuration in {filename}!"
NO_CONFIGURATIONS = "No configurations found in configuration in {filename}!"
try:
with open(filename) as tuf_interposition_json:
tuf_interpositions = json.load(tuf_interposition_json)
configurations = tuf_interpositions.get("configurations", {})
if len(configurations) == 0:
raise InvalidConfiguration(NO_CONFIGURATIONS.format(filename=filename))
else:
for network_location, configuration in configurations.iteritems():
try:
configuration_parser = ConfigurationParser(network_location,
configuration, parent_repository_directory=parent_repository_directory,
parent_ssl_certificates_directory=parent_ssl_certificates_directory)
configuration = configuration_parser.parse()
__updater_controller.add(configuration)
except:
Logger.exception(INVALID_TUF_CONFIGURATION.format(network_location=network_location))
raise
except:
Logger.exception(INVALID_TUF_INTERPOSITION_JSON.format(filename=filename))
raise
day_list.append(current_day)
current_day = current_day + timedelta(days=1)
return day_list
@staticmethod
def __build_measurement_doc(source: str, attribute: str, date_origin: datetime, value: float) -> dict:
return {
'source': source,
'attribute': attribute,
'timestamp': date_origin + random.random() * timedelta(days=1),
'value': value
}
if __name__ == '__main__':
arg_parser = ArgumentParser()
arg_parser.add_argument('-c', '--config',
required=True,
dest='config_file',
help='Path to the input configuration file',
metavar='FILE')
arg_parser.add_argument('-o', '--output',
required=True,
dest='validation_file',
help='Output file with stats from the input data, mey be used later to check if data '
'processor calculate the statistical data correctly',
metavar='FILE')
args = arg_parser.parse_args()
configuration = ConfigurationParser.parse_from_file(args.config_file)
DataProcessor().process_data(configuration, args.validation_file)
