def handle(self, args):
"""
CLI to extract entities from text.
"""
if args.text is None and args.path is None:
return color.format('* no text source specified', color.RED)
registry = Registry.find_or_create(args.registry,
key_prefix=args.prefix)
entity = registry.get_dict(args.entity)
engine = Extractor(entity, args.grammar)
if args.path is not None:
text = open(args.path).read()
else:
text = args.text
results = engine.extract(text, args.lang)
entities = []
for element in results[0]:
if element['entity_found'] == 1:
entities = list(
set(entities).union(element['entity_candidates']))
if len(entities) > 0:
print(
color.format('%d entities detected' % len(entities),
color.GREEN))
print('\n'.join(entities))
else:
print(color.format('no entities detected', color.RED))
# print(color.format('%d' % results[1].elapsed, color.LIGHT_MAGENTA))
return '* text processed according to %s entity' %\
(color.format(args.entity, color.GREEN))
def state_light(self, full=True):
"""
Returns the colorized state light. If full is true, returns the name.
"""
scolor = self.STATE_COLORS[self.state]
if full:
return color.format(u"● {}", scolor, self.state)
return color.format(u"●", scolor)
def handle(self, args):
"""
Handle the destroy command
"""
# Load the managed instances
manager = ManagedInstances.load()
# Validate instances and create a new manager
if args.instances:
n_unmanaged = 0
for instance in args.instances:
if instance not in manager:
n_unmanaged += 1
# Cannot destroy any unmanaged instances
if n_unmanaged > 0:
raise ValueError(
"cannot destroy {} unmanaged instances".format(n_unmanaged)
)
# Filter the manager to the instances we'll be destroying
manager = manager.filter(args.instances, instances=True)
# Return if no instances are managed
if len(manager) == 0:
return color.format(
"no instances under management", color.LIGHT_YELLOW
)
# Prompt to confirm
if not args.force:
prompt = color.format("destroy {}?", color.LIGHT_YELLOW, manager)
if not self.prompt(prompt):
print(color.format("stopping instance termination", color.LIGHT_CYAN))
return
print(color.format(u"destroying {} instances …\n", color.LIGHT_RED, len(manager)))
# Destroy instances
reports = manager.terminate()
# Report destruction
table = [['Region', 'Instance', 'State']]
table.extend([
[
report.region.name, report["InstanceId"], unicode(report),
]
for report in reports
])
print(tabulate(table, tablefmt="simple", headers='firstrow'))
# Remove instances from management
# Ensure we reload the full manager list and not use the filtered one.
manager = ManagedInstances.load()
for report in reports:
manager.discard(report["InstanceId"])
manager.dump()
class BaleenUtility(ConsoleProgram):
description = color.format(DESCRIPTION, color.CYAN)
epilog = color.format(EPILOG, color.MAGENTA)
version = color.format("baleen v{}", color.CYAN, get_version())
@classmethod
def load(klass, commands=COMMANDS):
utility = klass()
for command in commands:
utility.register(command)
return utility
def handle(self, args):
"""
CLI to drop an entity dictionary.
"""
registry = Registry.find_or_create(args.registry,
key_prefix=args.prefix)
if registry.del_dict(args.entity):
return '* %s dictionary dropped' % (color.format(
args.entity, color.GREEN))
else:
return '* %s unknown dictionary' % (color.format(
args.entity, color.RED))
class ExampleUtility(ConsoleProgram):
description = color.format(DESCRIPTION, color.CYAN)
epilog = color.format(EPILOG, color.LIGHT_MAGENTA)
version = commis.__version__
@classmethod
def load(klass, commands=COMMANDS):
utility = klass()
for command in commands:
utility.register(command)
return utility
class PasswordUtility(commis.ConsoleProgram):
description = color.format(DESCRIPTION, color.CYAN)
epilog = color.format(EPILOG, color.MAGENTA)
version = color.format(VERSION, color.CYAN)
@classmethod
def load(klass):
utility = klass()
utility.register(Generate)
utility.register(Configure)
return utility
class FeetApp(ConsoleProgram):
"""
Feet CLI app
"""
description = color.format(DESCRIPTION, color.CYAN)
epilog = color.format(EPILOG, color.MAGENTA)
version = color.format("feet v{}", color.CYAN, __version__)
@classmethod
def load(klass, commands=COMMANDS):
utility = klass()
for command in commands:
utility.register(command)
return utility
def handle(self, args):
"""
Create packages for the specified datasets
"""
# Ensure there is an uploads directory
if not os.path.exists(args.uploads) or not os.path.isdir(args.uploads):
if args.force:
os.makedirs(args.uploads)
else:
raise ConsoleError(
"no uploads directory at '{}' use -f to create".format(
args.uploads))
for dataset in args.datasets:
# Remove the trailing slash
dataset = dataset.rstrip(os.path.sep)
# Check if the dataset is valid
if not args.force and not is_valid(dataset, False):
print(
color.format("cannot package invalid dataset at {}",
color.LIGHT_RED, dataset))
continue
name = os.path.basename(dataset)
out = os.path.join(args.uploads, name)
if not args.force and os.path.exists(out + ".zip"):
print(
color.format("dataset exists at {} use -f to overwrite",
color.LIGHT_YELLOW, out))
continue
try:
out = shutil.make_archive(out,
"zip",
root_dir=os.path.dirname(dataset),
base_dir=name)
self.update_manifest(out)
except Exception as e:
print(
color.format("could not package dataset {} at {}: {}",
color.LIGHT_RED, dataset,
out.rstrip(".zip") + ".zip", e))
continue
print(
color.format("packaged dataset at {}", color.LIGHT_GREEN, out))
def handle_all(self, args):
"""
Run all datasets through the checker and print out readiness table.
"""
fixtures = args.dataset[0]
datasets = [
path for path in glob(os.path.join(fixtures, "*"))
if os.path.isdir(path)
]
if len(datasets) == 0:
print(
color.format("no datasets found in '{}'", color.LIGHT_YELLOW,
fixtures))
return
table = [["valid", "dataset", "type"]]
for dataset in datasets:
if is_valid_standard(dataset):
table.append([CHECKMARK, dataset, "standard"])
continue
elif is_valid_corpus(dataset):
table.append([CHECKMARK, dataset, "corpus"])
continue
else:
table.append([CROSSMARK, dataset, ""])
print(tabulate(table, headers="firstrow", tablefmt="simple"))
def execute(self):
"""
Entry point to the execution of the program.
"""
# Ensure that we have commands registered
if not self.commands or self._parser is None:
raise NotImplementedError(
"No commands registered with this program!"
)
# Handle input from the command line
args = self.parser.parse_args() # Parse the arguments
try:
handle_default_args(args) # Handle the default args
msg = "{}\n".format(args.func(args)) # Call the default function
self.exit(0, msg) # Exit cleanly with message
except Exception as e:
if hasattr(args, 'traceback') and args.traceback:
traceback.print_exc()
msg = color.format(str(e), color.RED)
self.exit(1, msg) # Exit with error
def execute(self):
"""
Entry point to the execution of the program.
"""
# Ensure that we have commands registered
if not self.commands or self._parser is None:
raise NotImplementedError(
"No commands registered with this program!"
)
# Handle input from the command line
args = self.parser.parse_args() # Parse the arguments
try:
handle_default_args(args) # Handle the default args
msg = "{}\n".format(args.func(args)) # Call the default function
self.exit(0, msg) # Exit cleanly with message
except Exception as e:
if hasattr(args, 'traceback') and args.traceback:
traceback.print_exc()
msg = color.format(str(e), color.RED)
self.exit(1, msg) # Exit with error
def handle(self, args):
"""
Handles the config command with arguments from the command line.
"""
# Load the instance manager
manager = ManagedInstances.load()
# Filter by regions
manager = manager.filter(args.regions, regions=True)
# Filter by instance ids
if args.instances:
manager = manager.filter(args.instances, instances=True)
# Return if no instances are managed
if len(manager) == 0:
return color.format("no instances under management",
color.LIGHT_YELLOW)
# Load the instances
instances = manager.status()
# Create the region table
table = [[
instance.state_light(), instance.region.name, instance.name,
str(instance),
instance.uptime()
] for instance in instances]
print(tabulate(table, tablefmt="plain"))
def handle(self, args):
"""
Handle the launch command
"""
# Load the regions for the launch command
regions = Regions(region for region in Regions.load()
if str(region) in args.regions)
# Get the templates associated with each region
self.templates = regions.launch_templates()
# Launch the instances with the specified template
instances = Instances.collect(
wait(
(partial(self.launch_in_region, region) for region in regions),
args=(args, )))
# Rename the instances given
wait((partial(self.tag_instance, instance, idx)
for idx, instance in enumerate(instances)),
args=(args, ))
# Update the instances under management
manager = ManagedInstances.load()
for instance in instances:
manager.add(str(instance), instance.region)
manager.dump()
# Report what went down
print(
color.format("created {} instances in {} regions",
color.LIGHT_GREEN, len(instances), len(regions)))
def handle(self, args):
"""
Run the dataset through each checker and print out a checklist table.
"""
if args.all:
return self.handle_all(args)
if args.type == "standard":
checklist = standard_package_checklist(args.dataset[0],
not args.exclude_optional)
valid = is_valid_standard(args.dataset[0], False)
elif args.type == "corpus":
checklist = corpus_package_checklist(args.dataset[0],
not args.exclude_optional)
valid = is_valid_corpus(args.dataset[0], False)
else:
raise ValueError("did not understand type '{}'".format(args.type))
table = [["", args.dataset[0]]]
for item, checked in checklist.items():
table.append([CHECKMARK if checked else CROSSMARK, item])
print(tabulate(table, tablefmt="simple", headers="firstrow"))
valid_msg, valid_color = {
True: ("is valid and ready to be uploaded", color.LIGHT_GREEN),
False:
("needs additional requirements before upload", color.LIGHT_RED),
}[valid]
print(color.format("\n{} {}", valid_color, args.dataset[0], valid_msg))
def handle(self, args):
"""
Start all instances currently being managed.
"""
manager = ManagedInstances.load()
# Filter by regions
manager = manager.filter(args.regions, regions=True)
# Filter by instance ids
if args.instances:
manager = manager.filter(args.instances, instances=True)
# Return if no instances are managed
if len(manager) == 0:
return color.format("no instances under management",
color.LIGHT_YELLOW)
table = [['Region', 'Instance', 'State']]
table.extend([[
report.region.name,
report["InstanceId"],
unicode(report),
] for report in manager.start()])
print(tabulate(table, tablefmt="simple", headers='firstrow'))
def handle(self, args):
"""
Convert the dataset from one format to another.
"""
stype = self.get_data_type(args.src)
dtype = self.get_data_type(args.dst)
if stype not in VALID_EXTS:
print(color.format("Unknown source type '{}'", color.LIGHT_RED, stype))
return
if dtype not in VALID_EXTS:
print(color.format("Unknown convert type '{}'", color.LIGHT_RED, dtype))
return
# Load source data
if stype.startswith(".csv"):
# Load data with pandas
X, y = self.load_pandas(args.src)
elif stype == ".npz":
# Load data with numpy
data = np.load(args.src)
X, y = data.get("X"), data.get("y")
else:
raise NotImplementedError("conversion from {} to {} not implemented yet".format(stype, dtype))
# Perform the conversion and save
if dtype == ".npz":
# Convert to the appropriate array type
if args.X_dtype is not None:
X = X.astype(args.X_dtype)
if args.y_dtype is not None:
y = y.astype(args.y_dtype)
# Save as numpy compressed
np.savez_compressed(args.dst, X=X, y=y, allow_pickle=False)
elif dtype.startswith(".csv"):
# Save as pandas data frame
df = pd.concat([pd.DataFrame(X), pd.Series(y)], axis=1)
meta = self.load_meta(args.src)
header = meta['features'] + [meta['target']]
compression = "gzip" if dtype.endswith(".gz") else None
df.to_csv(args.dst, index=False, header=header, compression=compression)
else:
raise NotImplementedError("conversion from {} to {} not implemented yet".format(stype, dtype))
def handle(self, args):
self.instances = ManagedInstances.load()
instance_ids = list(self.instances) if args.all else args.instance_id
if len(instance_ids) == 0:
print(color.format("no instances to create", color.YELLOW))
results = [self.handle_instance_id(iid, args) for iid in instance_ids]
print(tabulate(results, tablefmt="simple", headers="keys"))
def get_password(self, attempts=0, confirm=settings.password_master):
password = getpass("Enter master passphrase: ").strip()
confirm = confirm or getpass("Enter same passphrase again: ").strip()
if not self.confirm_password(password, confirm):
if attempts <= 3:
print color.format(
"Password doesn't match configuration or confirmation, try again.",
color.YELLOW)
return self.get_password(attempts + 1)
else:
raise ConsoleError(
"Password attempt maximum, stretch fingers and try again!")
if not password:
raise ConsoleError(
"You must supply a base password for the generator!")
return password
def test_colorize_alias(self):
"""
Assert that there is a colorize alias and use args instead of kwargs.
"""
templ = "It will cost you ${:0.2f} to purchase the {}."
args = (19.993, "cookie press")
render = templ.format(*args)
for cm, cr in colors:
expected = cr + render + colorama.Fore.RESET
self.assertEqual(color.format(templ, cm, *args), expected)
def handle(self, args):
kahu = Kahu()
if len(args.replicas) == 0 and not args.deactivate:
return color.format(
"specify replicas to activate or use --deactivate",
color.LIGHT_YELLOW)
if args.deactivate and len(args.replicas) > 0:
print(
color.format("warning: ignoring deactivate flag",
color.LIGHT_YELLOW))
resp = kahu.activate(args.replicas)
print(
color.format("{:>3} active replicas", color.LIGHT_GREEN,
resp["activated"]))
print(
color.format("{:>3} inactive replicas", color.LIGHT_RED,
resp["deactivated"]))
def test_colorize_alias(self):
"""
Assert that there is a colorize alias and use args instead of kwargs.
"""
templ = "It will cost you ${:0.2f} to purchase the {}."
args = (19.993, "cookie press")
render = templ.format(*args)
for cm, cr in colors:
expected = cr + render + colorama.Fore.RESET
self.assertEqual(color.format(templ, cm, *args), expected)
def handle(self, args):
"""
CLI to load an entity dictionary.
"""
file_path = args.txt
if args.csv is not None:
registry = Registry.find_or_create(args.registry,
dict_class=CSVDictionary,
key_prefix=args.prefix)
file_path = args.csv
else:
registry = Registry.find_or_create(args.registry,
key_prefix=args.prefix)
file_path = args.txt
dictionary = registry.get_dict(args.entity)
count = dictionary.load_file(file_path, args.lang)
print('+ %d entities processed' % count)
return '* %s dictionary loaded' % (color.format(args.entity,
color.GREEN))
from geonet.ec2 import Instance, Volume
from geonet.utils.serialize import to_json
# Resource Types
GROUPS = "security-groups"
TEMPLATES = "launch-templates"
AMIS = "images"
KEYS = "key-pairs"
VOLUMES = "volumes"
INSTANCES = "instances"
PLACEMENTS = "placement-groups"
ZONES = "availability-zones"
# Checks
CHECKS = {
True: color.format(u"✓", color.LIGHT_GREEN),
False: color.format(u"✗", color.LIGHT_RED)
}
##########################################################################
## Helper Methods
##########################################################################
def rtype(s):
"""
Compute aliases for the resource type
"""
s = s.lower()
if s in {'m', 'machine', 'machines', 'vm', 'vms', 'instance', INSTANCES}:
## Imports
##########################################################################
from commis import color
from commis import Command
from tabulate import tabulate
from geonet.region import Regions
from geonet.utils.async import wait
from geonet.config import settings
from functools import partial
GROUP_NAME = "alia"
CHECKMARK = color.format(u"✓", color.LIGHT_GREEN)
CROSSMARK = color.format(u"✗", color.LIGHT_RED)
##########################################################################
## Command Description
##########################################################################
class SecurityGroupCreateCommand(Command):
name = "sg:create"
help = "create the default alia security group"
args = {
'--tablefmt': {
'choices': ('plain', 'simple', 'pipe', 'psql', 'html', 'latex'),
'default': 'simple',
##########################################################################
## Imports
##########################################################################
import os
from commis import color
from commis import Command
from tabulate import tabulate
from geonet.ec2 import connect, Instance
from geonet.utils.editor import edit_file
from geonet.utils.serialize import to_json
from geonet.region import Regions, REGIONDATA
CHECKMARK = color.format(u"✓", color.LIGHT_GREEN)
##########################################################################
## Command Description
##########################################################################
class RegionsCommand(Command):
name = "regions"
help = "describe configured regions"
args = {
('-e', '--edit'): {
'action': 'store_true',
'help': 'edit region locale names',
},
