def while_waiting(self):
""" Update fields periodically if nothing is happening """
# give a little extra time for file descriptors to close
time.sleep(0.1)
self.addfield.value = Timestamp()
self.addfield.display()
self.addfield2.value = Uptime()
self.addfield2.display()
self.addfield3.value = str(len(Containers())) + ' running'
if len(Containers()) > 0:
self.addfield3.labelColor = 'GOOD'
else:
self.addfield3.labelColor = 'DEFAULT'
self.addfield3.display()
# if file drop location changes deal with it
logger = Logger(__name__)
if self.file_drop.value != DropLocation()[1]:
logger.info('Starting: file drop restart')
try:
self.file_drop.value = DropLocation()[1]
logger.info('Path given: ' + str(self.file_drop.value))
except Exception as e: # pragma no cover
logger.error('file drop restart failed with error: ' + str(e))
logger.info('Finished: file drop restart')
self.file_drop.display()
return
def popup(original, orig_type, thr, title):
"""
Start the thread and display a popup of info
until the thread is finished
"""
thr.start()
info_str = ""
while thr.is_alive():
if orig_type == 'containers':
info = diff(Containers(), original)
elif orig_type == 'images':
info = diff(Images(), original)
if info:
info_str = ""
for entry in info:
info_str = entry[0] + ": " + entry[1] + "\n" + info_str
if self.action['action_name'] != 'configure':
npyscreen.notify_wait(info_str, title=title)
time.sleep(1)
thr.join()
try:
result = self.api_action.queue.get(False)
if isinstance(result, tuple) and isinstance(result[1], tuple):
running, failed = result[1]
r_str = ''
for container in running:
r_str += container + ": successful\n"
for container in failed:
r_str += container + ": failed\n"
npyscreen.notify_confirm(r_str)
except Exception as e: # pragma: no cover
pass
return
def on_ok(self):
"""
Take the tool selections and start them
"""
def diff(first, second):
"""
Get the elements that exist in the first list and not in the second
"""
second = set(second)
return [item for item in first if item not in second]
def popup(original, orig_type, thr, title):
"""
Start the thread and display a popup of info
until the thread is finished
"""
thr.start()
info_str = ""
while thr.is_alive():
if orig_type == 'containers':
info = diff(Containers(), original)
elif orig_type == 'images':
info = diff(Images(), original)
if info:
info_str = ""
for entry in info:
# TODO limit length of info_str to fit box
info_str += entry[0] + ": " + entry[1] + "\n"
npyscreen.notify_wait(info_str, title=title)
time.sleep(1)
return
original_containers = Containers()
api_action = Action()
tool_dict = {}
for repo in self.tools_tc:
for tool in self.tools_tc[repo]:
self.logger.info(tool)
if self.tools_tc[repo][tool].value:
t = tool
if t.startswith('/:'):
t = " " + t[1:]
t = t.split(":")
status = api_action.prep_start(name=t[0],
branch=t[1],
version=t[2])
if status[0]:
tool_dict.update(status[1])
thr = threading.Thread(target=api_action.start,
args=(),
kwargs={'tool_dict': tool_dict})
popup(original_containers, "containers", thr,
'Please wait, starting containers...')
npyscreen.notify_confirm("Done starting containers.",
title='Started containers')
self.quit()
def get_nodes(self):
nodes = [{
'renderer': 'region',
'name': 'VENT',
'class': 'normal',
'maxVolume': 1000,
'updated': int(round(time.time() * 1000)),
'nodes': [],
'connections': []
}]
containers = Containers(exclude_labels=['monitoring'])
for container in containers:
node = {'renderer': 'focusedChild',
'name': container[0],
'class': 'normal'}
nodes[0]['nodes'].append(node)
return nodes
def popup(original_containers, thr, title):
"""
Start the thread and display a popup of the running containers
until the thread is finished
"""
thr.start()
container_str = ""
while thr.is_alive():
containers = diff(Containers(), original_containers)
if containers:
container_str = ""
for container in containers:
# TODO limit length of container_str to fit box
container_str += container[0] + ": " + container[1] + "\n"
npyscreen.notify_wait(container_str, title=title)
time.sleep(1)
return
def popup(original, orig_type, thr, title):
"""
Start the thread and display a popup of info
until the thread is finished
"""
thr.start()
info_str = ""
while thr.is_alive():
if orig_type == 'containers':
info = diff(Containers(), original)
elif orig_type == 'images':
info = diff(Images(), original)
if info:
info_str = ""
for entry in info:
# TODO limit length of info_str to fit box
info_str += entry[0] + ": " + entry[1] + "\n"
npyscreen.notify_wait(info_str, title=title)
time.sleep(1)
return
def test_containers():
""" Test the containers function """
containers = Containers()
containers = Containers(vent=False)
assert isinstance(containers, list)
def while_waiting(self):
""" Update fields periodically if nothing is happening """
# give a little extra time for file descriptors to close
time.sleep(0.1)
self.addfield.value = Timestamp()
self.addfield.display()
self.addfield2.value = Uptime()
self.addfield2.display()
self.addfield3.value = str(len(Containers())) + ' running'
if len(Containers()) > 0:
self.addfield3.labelColor = 'GOOD'
else:
self.addfield3.labelColor = 'DEFAULT'
self.addfield3.display()
# update core tool status
self.addfield5.value, values = MainForm.t_status(True)
if values[0] + values[1] == 0:
color = 'DANGER'
self.addfield4.labelColor = 'CAUTION'
self.addfield4.value = 'Idle'
elif values[0] >= int(values[2]):
color = 'GOOD'
self.addfield4.labelColor = color
self.addfield4.value = 'Ready to start jobs'
else:
color = 'CAUTION'
self.addfield4.labelColor = color
self.addfield4.value = 'Ready to start jobs'
self.addfield5.labelColor = color
# update plugin tool status
plugin_str, values = MainForm.t_status(False)
plugin_str += ', ' + str(values[3]) + ' plugin(s) installed'
self.addfield6.value = plugin_str
# get jobs
jobs = Jobs()
# number of jobs, number of tool containers
self.addfield7.value = str(jobs[0]) + ' jobs running (' + str(jobs[1])
self.addfield7.value += ' tool containers), ' + str(jobs[2])
self.addfield7.value += ' completed jobs'
if jobs[0] > 0:
self.addfield4.labelColor = 'GOOD'
self.addfield4.value = 'Processing jobs'
self.addfield7.labelColor = 'GOOD'
else:
self.addfield7.labelColor = 'DEFAULT'
self.addfield4.display()
self.addfield5.display()
self.addfield6.display()
self.addfield7.display()
# if file drop location changes deal with it
logger = Logger(__name__)
status = (False, None)
if self.file_drop.value != DropLocation()[1]:
logger.info('Starting: file drop restart')
try:
self.file_drop.value = DropLocation()[1]
logger.info('Path given: ' + str(self.file_drop.value))
# restart if the path is valid
if DropLocation()[0]:
status = self.api_action.clean(name='file_drop')
status = self.api_action.prep_start(name='file_drop')
else:
logger.error('file drop path name invalid' +
DropLocation()[1])
if status[0]:
tool_d = status[1]
status = self.api_action.start(tool_d)
logger.info('Status of file drop restart: ' +
str(status[0]))
except Exception as e: # pragma no cover
logger.error('file drop restart failed with error: ' + str(e))
logger.info('Finished: file drop restart')
self.file_drop.display()
return
def on_ok(self):
"""
Take the tool selections and perform the provided action on them
"""
def diff(first, second):
"""
Get the elements that exist in the first list and not in the second
"""
second = set(second)
return [item for item in first if item not in second]
def popup(original, orig_type, thr, title):
"""
Start the thread and display a popup of info
until the thread is finished
"""
thr.start()
info_str = ""
while thr.is_alive():
if orig_type == 'containers':
info = diff(Containers(), original)
elif orig_type == 'images':
info = diff(Images(), original)
if info:
info_str = ""
for entry in info:
info_str = entry[0] + ": " + entry[1] + "\n" + info_str
if self.action['action_name'] != 'configure':
npyscreen.notify_wait(info_str, title=title)
time.sleep(1)
thr.join()
try:
result = self.api_action.queue.get(False)
if isinstance(result, tuple) and isinstance(result[1], tuple):
running, failed = result[1]
r_str = ''
for container in running:
r_str += container + ": successful\n"
for container in failed:
r_str += container + ": failed\n"
npyscreen.notify_confirm(r_str)
except Exception as e: # pragma: no cover
pass
return
if self.action['type'] == 'images':
originals = Images()
else:
originals = Containers()
tool_d = {}
if self.action['action_name'] in ['clean', 'remove', 'stop', 'update']:
reconfirmation_str = ""
if self.action['cores']:
reconfirmation_str = "Are you sure you want to "
reconfirmation_str += self.action['action_name']
reconfirmation_str += " core containers?"
else:
reconfirmation_str = "Are you sure you want to "
reconfirmation_str += self.action['action_name']
reconfirmation_str += " plugin containers?"
perform = npyscreen.notify_ok_cancel(reconfirmation_str,
title="Confirm command")
if not perform:
return
tools_to_configure = []
for repo in self.tools_tc:
for tool in self.tools_tc[repo]:
self.logger.info(tool)
if self.tools_tc[repo][tool].value:
t = tool.split(":", 2)[2].split("/")[-1]
if t.startswith('/:'):
t = " " + t[1:]
t = t.split(":")
if self.action['action_name'] == 'start':
status = self.action['action_object2'](name=t[0],
branch=t[1],
version=t[2])
if status[0]:
tool_d.update(status[1])
elif self.action['action_name'] == 'configure':
constraints = {
'name': t[0],
'branch': t[1],
'version': t[2],
'repo': repo
}
options = ['type']
action = self.action['api_action']
tool = action.p_helper.constraint_options(
constraints, options)[0]
# only one tool should be returned
name = tool.keys()[0]
if tool[name]['type'] == 'registry':
registry_image = True
else:
registry_image = False
kargs = {
'name': 'Configure ' + t[0],
'tool_name': t[0],
'branch': t[1],
'version': t[2],
'repo': repo,
'next_tool': None,
'get_configure': action.get_configure,
'save_configure': action.save_configure,
'restart_tools': action.restart_tools,
'clean': action.clean,
'prep_start': action.prep_start,
'start_tools': action.start,
'from_registry': registry_image
}
if tools_to_configure:
kargs['next_tool'] = tools_to_configure[-1]
self.parentApp.addForm("EDITOR" + t[0], EditorForm,
**kargs)
tools_to_configure.append("EDITOR" + t[0])
else:
kargs = {'name': t[0], 'branch': t[1], 'version': t[2]}
# add core recognition
if self.action['cores']:
kargs.update({'groups': 'core'})
# use latest version for update, not necessarily
# version in manifest
if self.action['action_name'] == 'update':
if t[2] != 'HEAD':
repo_commits = self.m_helper.repo_commits(
repo)[1]
for branch in repo_commits:
if branch[0] == t[1]:
kargs.update(
{'new_version': branch[1][0]})
else:
kargs.update({'new_version': 'HEAD'})
thr = Thread(target=self.action['action_object1'],
args=(),
kwargs=kargs)
popup(
originals, self.action['type'], thr,
'Please wait, ' + self.action['present_t'] + '...')
if self.action['action_name'] == 'start':
thr = Thread(target=self.action['action_object1'],
args=(),
kwargs={'tool_d': tool_d})
popup(originals, self.action['type'], thr,
'Please wait, ' + self.action['present_t'] + '...')
if self.action['action_name'] != 'configure':
npyscreen.notify_confirm('Done ' + self.action['present_t'] + '.',
title=self.action['past_t'])
self.quit()
else:
if len(tools_to_configure) > 0:
self.parentApp.change_form(tools_to_configure[-1])
else:
npyscreen.notify_confirm(
"No tools selected, returning to"
" main menu",
title="No action taken")
self.quit()
def test_containers():
""" Test the containers function """
containers = Containers()
containers = Containers(vent=False)
assert type(containers) == list
def inventory(self, choices=None):
""" Return a dictionary of the inventory items and status """
self.logger.info("Starting: inventory")
status = (True, None)
self.logger.info("choices specified: " + str(choices))
try:
# choices: repos, core, tools, images, built, running, enabled
items = {
'repos': [],
'core': [],
'tools': [],
'images': [],
'built': [],
'running': [],
'enabled': []
}
tools = self.plugin.tools()
self.logger.info("found tools: " + str(tools))
for choice in choices:
for tool in tools:
try:
if choice == 'repos':
if 'repo' in tool:
if tool['repo'] and tool['repo'] not in items[
choice]:
items[choice].append(tool['repo'])
elif choice == 'core':
if 'groups' in tool:
if 'core' in tool['groups']:
items[choice].append(
(tool['section'], tool['name']))
elif choice == 'tools':
items[choice].append(
(tool['section'], tool['name']))
elif choice == 'images':
# TODO also check against docker
images = Images()
items[choice].append(
(tool['section'], tool['name'],
tool['image_name']))
elif choice == 'built':
items[choice].append(
(tool['section'], tool['name'], tool['built']))
elif choice == 'running':
containers = Containers()
status = 'not running'
for container in containers:
image_name = tool['image_name'].rsplit(
":" + tool['version'], 1)[0]
image_name = image_name.replace(':', '-')
image_name = image_name.replace('/', '-')
if container[0] == image_name:
status = container[1]
items[choice].append(
(tool['section'], tool['name'], status))
elif choice == 'enabled':
items[choice].append(
(tool['section'], tool['name'],
tool['enabled']))
else:
# unknown choice
pass
except Exception as e: # pragma: no cover
self.logger.error(
"unable to grab information about tool: " +
str(tool) + " because: " + str(e))
pass
status = (True, items)
except Exception as e:
self.logger.error("inventory failed with error: " + str(e))
status = (False, e)
self.logger.info("Status of inventory: " + str(status))
self.logger.info("Finished: inventory")
return status
def update(self,
repo=None,
name=None,
groups=None,
enabled="yes",
branch="master",
version="HEAD"):
"""
Update a set of tools that match the parameters given, if no parameters
are given, updated all installed tools on the master branch at verison
HEAD that are enabled
"""
args = locals()
self.logger.info("Starting: update")
self.logger.info(args)
status = (True, None)
try:
options = ['path', 'image_name', 'image_id']
sections, template = self.plugin.constraint_options(args, options)
# get existing containers and images and states
running_containers = Containers()
built_images = Images()
self.logger.info("running docker containers: " +
str(running_containers))
self.logger.info("built docker images: " + str(built_images))
# if repo, pull and build
# if registry image, pull
for section in sections:
try:
cwd = os.getcwd()
self.logger.info("current working directory: " + str(cwd))
os.chdir(sections[section]['path'])
self.plugin.version = version
self.plugin.branch = branch
c_status = self.plugin.checkout()
self.logger.info(c_status)
try:
os.chdir(cwd)
except Exception as e: # pragma: no cover
self.logger.error("unable to change directory: " +
str(e))
pass
template = self.plugin.builder(
template,
sections[section]['path'],
sections[section]['image_name'],
section,
build=True,
branch=branch,
version=version)
self.logger.info(template)
# stop and remove old containers and images if image_id updated
# !! TODO
# start containers if they were running
# !! TODO
# TODO logging
except Exception as e: # pragma: no cover
self.logger.error("unable to update: " + str(section) +
" because: " + str(e))
template.write_config()
except Exception as e:
self.logger.error("update failed with error: " + str(e))
status = (False, e)
self.logger.info("Status of update: " + str(status))
self.logger.info("Finished: update")
return status
def inventory(self, choices=None):
""" Return a dictionary of the inventory items and status """
status = (True, None)
if not choices:
return (False, 'No choices made')
try:
# choices: repos, tools, images, built, running, enabled
items = {
'repos': [],
'tools': {},
'images': {},
'built': {},
'running': {},
'enabled': {}
}
tools = Template(self.manifest).list_tools()
for choice in choices:
for tool in tools:
try:
if choice == 'repos':
if 'repo' in tool:
if (tool['repo']
and tool['repo'] not in items[choice]):
items[choice].append(tool['repo'])
elif choice == 'tools':
items[choice][tool['section']] = tool['name']
elif choice == 'images':
# TODO also check against docker
items[choice][tool['section']] = tool['image_name']
elif choice == 'built':
items[choice][tool['section']] = tool['built']
elif choice == 'running':
containers = Containers()
status = 'not running'
for container in containers:
image_name = tool['image_name'] \
.rsplit(':' +
tool['version'], 1)[0]
image_name = image_name.replace(':', '-')
image_name = image_name.replace('/', '-')
self.logger.info('image_name: ' + image_name)
if container[0] == image_name:
status = container[1]
elif container[0] == image_name + \
'-' + tool['version']:
status = container[1]
items[choice][tool['section']] = status
elif choice == 'enabled':
items[choice][tool['section']] = tool['enabled']
else:
# unknown choice
pass
except Exception as e: # pragma: no cover
self.logger.error('Unable to grab info about tool: ' +
str(tool) + ' because: ' + str(e))
status = (True, items)
except Exception as e: # pragma: no cover
self.logger.error('Inventory failed with error: {0}'.format(
str(e)))
status = (False, str(e))
return status
def while_waiting(self):
""" Update fields periodically if nothing is happening """
def popup(message):
npyscreen.notify_confirm(str(message),
title="Docker Error",
form_color='DANGER',
wrap=True)
self.exit()
# clean up forms with dynamic data
self.parentApp.remove_forms()
self.parentApp.add_forms()
if not self.triggered:
self.triggered = True
try:
self.api_action = Action()
except DockerException as de: # pragma: no cover
popup(de)
# give a little extra time for file descriptors to close
time.sleep(0.1)
try:
current_path = os.getcwd()
except Exception as e: # pragma: no cover
self.exit()
self.addfield.value = Timestamp()
self.addfield.display()
self.addfield2.value = Uptime()
self.addfield2.display()
self.addfield3.value = str(len(Containers())) + " running"
if len(Containers()) > 0:
self.addfield3.labelColor = "GOOD"
else:
self.addfield3.labelColor = "DEFAULT"
self.addfield3.display()
# set core value string
core = Core()
installed = 0
custom_installed = 0
built = 0
custom_built = 0
running = 0
custom_running = 0
normal = str(len(core['normal']))
for tool in core['running']:
if tool in core['normal']:
running += 1
else:
custom_running += 1
for tool in core['built']:
if tool in core['normal']:
built += 1
else:
custom_built += 1
for tool in core['installed']:
if tool in core['normal']:
installed += 1
else:
custom_installed += 1
core_str = str(running + custom_running) + "/" + normal + " running"
if custom_running > 0:
core_str += " (" + str(custom_running) + " custom)"
core_str += ", " + str(built + custom_built) + "/" + normal + " built"
if custom_built > 0:
core_str += " (" + str(custom_built) + " custom)"
core_str += ", " + str(installed +
custom_installed) + "/" + normal + " installed"
if custom_built > 0:
core_str += " (" + str(custom_installed) + " custom)"
self.addfield5.value = core_str
if running + custom_running == 0:
color = "DANGER"
self.addfield4.labelColor = "CAUTION"
self.addfield4.value = "Idle"
elif running >= int(normal):
color = "GOOD"
self.addfield4.labelColor = color
self.addfield4.value = "Ready to start jobs"
else:
color = "CAUTION"
self.addfield4.labelColor = color
self.addfield4.value = "Ready to start jobs"
self.addfield5.labelColor = color
# get jobs
jobs = Jobs()
# number of jobs, number of tool containers
self.addfield6.value = str(jobs[0]) + " jobs running (" + str(
jobs[1]) + " tool containers), " + str(jobs[2]) + " completed jobs"
# TODO check if there are jobs running and update addfield4
if jobs[0] > 0:
self.addfield4.labelColor = "GOOD"
self.addfield4.value = "Processing jobs"
self.addfield6.labelColor = "GOOD"
else:
self.addfield6.labelColor = "DEFAULT"
self.addfield4.display()
self.addfield5.display()
self.addfield6.display()
os.chdir(current_path)
return
def perform_action(self, action):
""" Perform actions in the api from the CLI """
def diff(first, second):
"""
Get the elements that exist in the first list and not in the second
"""
second = set(second)
return [item for item in first if item not in second]
def popup(original_containers, thr, title):
"""
Start the thread and display a popup of the running containers
until the thread is finished
"""
thr.start()
container_str = ""
while thr.is_alive():
containers = diff(Containers(), original_containers)
if containers:
container_str = ""
for container in containers:
# TODO limit length of container_str to fit box
container_str += container[0] + ": " + container[1] + "\n"
npyscreen.notify_wait(container_str, title=title)
time.sleep(1)
return
original_containers = Containers()
if action == 'add':
self.parentApp.change_form('ADD')
elif action == "build":
self.parentApp.change_form('BUILDTOOLS')
elif action == 'start':
self.parentApp.change_form('STARTTOOLS')
elif action == 'stop':
self.parentApp.change_form('STOPTOOLS')
elif action == 'clean':
self.parentApp.change_form('CLEANTOOLS')
elif action == "inventory":
self.parentApp.change_form('INVENTORY')
elif action == "update":
self.parentApp.change_form('UPDATETOOLS')
elif action == "remove":
self.parentApp.change_form('REMOVETOOLS')
# tutorial forms
elif action == "background":
self.parentApp.change_form('TUTORIALBACKGROUND')
elif action == "terminology":
self.parentApp.change_form('TUTORIALTERMINOLOGY')
elif action == "setup":
self.parentApp.change_form('TUTORIALGETTINGSETUP')
elif action == "building_cores":
self.parentApp.change_form('TUTORIALBUILDINGCORES')
elif action == "starting_cores":
self.parentApp.change_form('TUTORIALSTARTINGCORES')
elif action == "adding_plugins":
self.parentApp.change_form('TUTORIALADDINGPLUGINS')
elif action == "adding_files":
self.parentApp.change_form('TUTORIALADDINGFILES')
elif action == "setting_up_services":
self.parentApp.change_form('TUTORIALSETTINGUPSERVICES')
return