class Command(amass.commands.DjangoCommand):
usage = CommonArgs("""
Delete gateway from AMASS
""", [
Arg("gateway", "Name of gateway to delete"),
], [])
def __init__(self):
amass.commands.Command.__init__(self)
self.file = __file__
def run(self, config, args):
arg_vals = self.parse_args(args)
[source_gateway] = self.get_django_models(config, "SourceGateway")
gw = None
try:
gw = source_gateway.objects.get(name=arg_vals["gateway"])
except:
amass.abort("Gateway '%s' does not exist" % arg_vals["gateway"])
try:
gw.delete()
print "Gateway '%s' sucessfully deleted" % arg_vals["gateway"]
except Exception as e:
amass.abort("Problem deleting gateway '%s': %s" %
(arg_vals["gateway"], str(e)))
class Command(amass.commands.Command):
usage = CommonArgs("""
List the cached features
""", [], [
Opt("names", "Display the names for each feature in vector", "False")
])
def __init__(self):
amass.commands.Command.__init__(self)
def run(self, config, args):
arg_vals = self.parse_args(args)
features = amass.features.Features(config)
table = features.list_cached_features()
if not self.is_arg_true(arg_vals["names"]):
amass.print_table(
["FEATURES", "GATEWAY", "COUNT", "START DATE", "END DATE"],
table)
sys.exit(0)
new_table = []
for row in table:
feature_names = features.list_feature_names(row[0], row[1])
for i, feature_name in enumerate(feature_names):
new_table.append([row[0], row[1], str(i), feature_name])
amass.print_table(["FEATURES", "GATEWAY", "INDEX", "NAME"], new_table)
class Command(amass.commands.DjangoCommand):
usage = CommonArgs("""
Delete gateway error type
""", [
Arg("gateway type", "Type of gateway"),
Arg("error", "Name of gateway error type to remove"),
], [])
def __init__(self):
amass.commands.Command.__init__(self)
self.file = __file__
def run(self, config, args):
arg_vals = self.parse_args(args)
[source_gateway_type, source_gateway_error
] = self.get_django_models(config, "SourceGatewayType",
"SourceGatewayError")
gw_type = None
try:
gw_type = source_gateway_type.objects.get(
type=arg_vals["gateway type"])
except:
amass.abort("Gateway type '%s' does not exist" %
arg_vals["gateway type"])
try:
e = source_gateway_error.objects.get(gateway_type=gw_type,
error=arg_vals["error"])
e.delete()
print "Gateway error type '%s' for gateway '%s' sucessfully deleted" % (
arg_vals["error"], arg_vals["gateway type"])
except Exception as e:
amass.abort(
"Problem deleting gateway error type '%s' for gateway '%s': %s"
% (arg_vals["error"], arg_vals["gateway type"], str(e)))
class Command(amass.commands.Command):
usage = CommonArgs(
"""
List all errors present in cached features
""",
[
Arg("features", "Name of defined feature set"),
Arg("gateway", "Name of gateway")
],
[
Opt("printerror", "Print out all errors matching specified type", "")
]
)
def __init__(self):
amass.commands.Command.__init__(self)
self.file = __file__
def run(self, config, args):
arg_vals = self.parse_args(args)
features = amass.features.Features(config)
table, unmatched_errors = features.list_cached_feature_errors(
arg_vals["features"],arg_vals["gateway"], arg_vals["printerror"])
if arg_vals["printerror"] != "":
amass.print_table(["ERROR"], table)
else:
amass.print_table(["ERROR TYPE", "ERROR", "COUNT"], table)
if len(unmatched_errors) > 0:
print "\nThe following errors are unmatched:"
for error in unmatched_errors:
print error
class Command(amass.commands.DjangoCommand):
usage = CommonArgs("""
Add a new gateway
""", [
Arg("gateway", "Name of gateway to add"),
Arg("type", "Type of gateway"),
], [])
def __init__(self):
amass.commands.Command.__init__(self)
self.file = __file__
def run(self, config, args):
arg_vals = self.parse_args(args)
[source_gateway,
source_gateway_type] = self.get_django_models(config, "SourceGateway",
"SourceGatewayType")
gw_type = None
try:
gw_type = source_gateway_type.objects.get(type=arg_vals["type"])
except:
amass.abort("'%s' is not a known gateway type" % arg_vals["type"])
try:
source_gateway.objects.create(name=arg_vals["gateway"],
type=gw_type)
print "Gateway '%s' sucessfully added" % arg_vals["gateway"]
except Exception as e:
amass.abort("Problem adding gateway '%s': %s" %
(arg_vals["gateway"], str(e)))
class Command(amass.commands.DjangoCommand):
usage = CommonArgs("""
Add a new gateway error type for this type of gateway
""",
[
Arg("gateway type", "Name of gateway type to add config parameter to"),
Arg("error", "Name of gateway error type"),
Arg("regex", "Regular expression that captures errors of this type"),
], [])
def __init__(self):
amass.commands.Command.__init__(self)
self.file = __file__
def run(self, config, args):
arg_vals = self.parse_args(args)
[source_gateway_type, source_gateway_error] = self.get_django_models(config, "SourceGatewayType", "SourceGatewayError")
gw = None
try:
gw_type = source_gateway_type.objects.get(type=arg_vals["gateway type"])
except:
amass.abort("'%s' is not a known gateway type" % arg_vals["gateway type"])
try:
source_gateway_error.objects.create(gateway_type=gw_type, error=arg_vals["error"], regex=arg_vals["regex"])
print "Error type '%s' sucessfully added for gateway '%s'" % (arg_vals["error"], arg_vals["gateway type"])
except Exception as e:
amass.abort("Problem adding gateway error '%s' for gateway '%s': %s" % (arg_vals["error"], arg_vals["gateway type"], str(e)))
class Command(amass.commands.list.DjangoListCommand):
usage = CommonArgs("""
List resources
""", [], [])
def __init__(self):
amass.commands.Command.__init__(self)
self.file = __file__
class Command(amass.commands.list.DjangoListCommand):
usage = CommonArgs("""
List available gateway types
""", [], [])
def __init__(self):
amass.commands.Command.__init__(self)
self.file = __file__
class Command(amass.commands.list.DjangoListCommand):
usage = CommonArgs("""
List configured gateways
""", [], [])
def __init__(self):
amass.commands.Command.__init__(self)
self.file = __file__
class Command(amass.commands.Command):
usage = CommonArgs("""
List the cached sources
""", [], [])
def __init__(self):
amass.commands.Command.__init__(self)
def run(self, config, args):
arg_vals = self.parse_args(args)
features = amass.features.Features(config)
table = features.list_cached_sources()
amass.print_table(["SOURCE", "COUNT", "START DATE", "END DATE"], table)
class Command(amass.commands.Command):
usage = CommonArgs("""
List available defined features
""", [], [])
def __init__(self):
amass.commands.Command.__init__(self)
self.file = __file__
def run(self, config, args):
self.parse_args(args)
features = amass.features.Features(config)
table = features.list_features()
amass.print_table(["FEATURE SET NAME", "FEATURE NAMES"], table)
class Command(amass.commands.DjangoCommand):
usage = CommonArgs("""
Generate features for selected gateway
""", [
Arg("features", "Name of cached feature set to run prediction on"),
Arg("gateway", "Name of cached gateway features"),
Arg("startdate", "Start date of jobs in 'YYYY-MM-DD HH:MM:SS' format"),
Arg("enddate", "End date of jobs in 'YYYY-MM-DD HH:MM:SS' format")
], [
Opt("refresh-features", "Re-generate all features", "False"),
Opt("refresh-sources",
"Re-generate specified sources (comma separated)", ""),
Opt("jobid", "Only generate features for specified job", "")
])
def __init__(self):
amass.commands.Command.__init__(self)
self.file = __file__
def run(self, config, args):
arg_vals = self.parse_args(args)
server_model_names = ["SourceConfig", "Resource", "SourceResource"]
server_models = self.get_django_models(config, *server_model_names)
server_config = {}
for i, model in enumerate(server_models):
server_config[server_model_names[i]] = model
startDate = "\'2016-03-01 03:00:00\'"
endDate = "\'2016-06-01 4:00:00\'"
features = amass.features.Features(config, server_config)
features.cache_init()
refresh_features = self.is_arg_true(arg_vals["refresh-features"])
refresh_sources = []
if arg_vals["refresh-sources"] != "":
refresh_sources = re.split("\s*,\s*", arg_vals["refresh-sources"])
if not features.load_or_fetch_sources(
arg_vals["features"], arg_vals["gateway"],
arg_vals["startdate"], arg_vals["enddate"], refresh_sources):
amass.abort("Unable to fetch source data for features %s" %
arg_vals["features"])
if arg_vals["jobid"] == "":
features.generate_features(arg_vals["features"],
arg_vals["gateway"], refresh_features)
else:
gateway = features.get_gateway(arg_vals["gateway"])
job = gateway.query_job(arg_vals["jobid"])
print features.generate_feature(arg_vals["features"], job)
class Command(amass.commands.DjangoCommand):
usage = CommonArgs("""
Add a new resource
""", [Arg("resource", "Name of resource to add")], [])
def __init__(self):
amass.commands.Command.__init__(self)
self.file = __file__
def run(self, config, args):
arg_vals = self.parse_args(args)
[resource] = self.get_django_models(config, "Resource")
try:
resource.objects.create(name=arg_vals["resource"], )
print "Resource '%s' sucessfully added" % arg_vals["resource"]
except Exception as e:
amass.abort("Problem adding resource '%s': %s" %
(arg_vals["resource"], str(e)))
class Command(amass.commands.DjangoCommand):
usage = CommonArgs("""
Add a new gateway type
""",
[
Arg("type", "Name of new gateway type"),
], [])
def __init__(self):
amass.commands.Command.__init__(self)
self.file = __file__
def run(self, config, args):
arg_vals = self.parse_args(args)
[source_gateway_type] = self.get_django_models(config, "SourceGatewayType")
try:
source_gateway_type.objects.create(type=arg_vals["type"])
print "Gateway type '%s' sucessfully added" % arg_vals["type"]
except:
amass.abort("Problem creating gatewa type '%s'" % arg_vals["type"])
class Command(amass.commands.Command):
usage = CommonArgs("""
Run predictions.
""", [
Arg("features", "Name of cached feature set to run prediction on"),
Arg("gateway", "Name of cached gateway features")
], [
Opt("filtererrors",
"Only run training and prediction on selected job errors", ""),
Opt("split", "Split the features into a train/test split", "0.6")
])
def __init__(self):
amass.commands.Command.__init__(self)
self.file = __file__
def run(self, config, args):
arg_vals = self.parse_args(args)
self.logger.info("Running prediction")
features = amass.features.Features(config)
filter_errors = None
if arg_vals["filtererrors"] != "":
filter_errors = re.split("\s*,\s*", arg_vals["filtererrors"])
features, results = features.get_features_results(
arg_vals["features"], arg_vals["gateway"], filter_errors)
if len(results) < 1:
amass.abort("Unable to find matching features to run prediction")
split_percentage = float(arg_vals["split"])
split_index = int(split_percentage * len(features))
train_features = features[:split_index]
test_features = features[split_index:]
train_results = results[:split_index]
test_results = results[split_index:]
predict = amass.predict.Prediction()
print predict.train(train_features, train_results)
class Command(amass.commands.DjangoCommand):
usage = CommonArgs("""
Add a new feature source configuration parameter
""",
[
Arg("source", "Name of source to add config parameter to"),
Arg("name", "Name of source config parameter"),
Arg("value", "Value of source config parameter"),
], [])
def __init__(self):
amass.commands.Command.__init__(self)
self.file = __file__
def run(self, config, args):
arg_vals = self.parse_args(args)
[source_config] = self.get_django_models(config, "SourceConfig")
try:
source_config.objects.create(source=arg_vals["source"], name=arg_vals["name"], value=arg_vals["value"])
print "Config parameter '%s' sucessfully added for source '%s'" % (arg_vals["name"], arg_vals["source"])
except Exception as e:
amass.abort("Problem adding config '%s' for source '%s': %s" % (arg_vals["name"], arg_vals["source"], str(e)))
class Command(amass.commands.DjangoCommand):
usage = CommonArgs("""
Delete source configuration parameter
""",
[
Arg("source", "Name of feature source"),
Arg("name", "Name of source configuration parameter to remove"),
], [])
def __init__(self):
amass.commands.Command.__init__(self)
self.file = __file__
def run(self, config, args):
arg_vals = self.parse_args(args)
[source_config] = self.get_django_models(config, "SourceConfig")
try:
cfg = source_config.objects.get(source=arg_vals["source"], name=arg_vals["name"])
cfg.delete()
print "Configuration parameter '%s' for source '%s' sucessfully deleted" % (arg_vals["name"], arg_vals["source"])
except Exception as e:
amass.abort("Problem deleting configuration parameter '%s' for source '%s': %s" % (arg_vals["name"], arg_vals["source"], str(e)))
class Command(amass.commands.DjangoCommand):
usage = CommonArgs(
"""
Add a new source resource. E.g., if CIPRES refers to Comet as 'comet' and
Inca refers to Comet as 'sdsc-comet'
""", [
Arg("resource", "Name of AMASS resource"),
Arg("source", "Name of source data -- e.g., inca, gateway.cipres"),
Arg("source resource", "Name of source resource"),
], [])
def __init__(self):
amass.commands.Command.__init__(self)
self.file = __file__
def run(self, config, args):
arg_vals = self.parse_args(args)
[resource,
source_resource] = self.get_django_models(config, "Resource",
"SourceResource")
amass_resource = None
try:
amass_resource = resource.objects.get(name=arg_vals["resource"])
except Exception as e:
amass.abort("'%s' is not a known AMASS resource" %
arg_vals["resource"])
try:
source_resource.objects.create(
resource=amass_resource,
source=arg_vals["source"],
source_name=arg_vals["source resource"])
print "Successfully added source resource '%s' for source '%s'" % (
arg_vals["source resource"], arg_vals["source"])
except Exception as e:
amass.abort("Problem adding source resource '%s': %s" %
(arg_vals["resource"], str(e)))