def setup(self, name='', command='', input_directory=None, output_directory=None, wait_for=None):
"""
Sets up the thread and builds the command structure.
@param name string
@param command string
@param input_directory string or None
@param output_directory string or None
@param wait_for string, None or ThreadableCommand
If this method is overridden, the overriding method must call back up to parent
or implement this functionality directly. Failure to do this will prevent the thread
from executing.
The flag `wait_for` will prevent the thread from executing until the wait_for thread
has completed. This is useful in the event a command needs the complete output from
another thread before it can work, for example, downloading data then parsing it.
If `wait_for` is None, the thread will execute as soon as there is room in the pool.
(Default pool size = ThreadManager.POOL_SIZE).
"""
# pylint: disable=arguments-differ
# This method provides specific implementations of *args and **kwargs
# pylint: disable=too-many-arguments
# This method requires a larger number of arguments than the standard
self.thread_name = name
self._input_directory = Replacements().replace(input_directory)
self._output_directory = Replacements().replace(
output_directory,
additional=ThreadableCommand.replacements(output_directory)
)
self._build_command(command)
self._block = wait_for
def replacements(string_to_search):
"""
Compiles a list of optional string replacements for command thread strings
"""
replacements = {}
function_regex = re.compile(
r'.*?[-_.]{1}\{(?P<what>.*?)\.(?P<command>.*?)\}',
re.DOTALL
)
matches = [match.groupdict() for match in function_regex.finditer(string_to_search)]
if len(matches) == 0:
return None
for match in matches:
string = '{0}.{1}'.format(match['what'], match['command'])
what = match['what']
if Replacements().find(what.upper()):
what = Replacements().replace('{' + what.upper() + '}')
command = '{0}_helper'.format(match['command'])
try:
command = include(command, 'pyccata', 'helpers')
except InvalidModuleError:
Logger().error(
'Invalid helper method {0} specified for {1}'.format(
match['command'],
string
)
)
raise
replacements[string] = command(what)
return replacements
def setup(self, name, input_name, output_name, recreate=False):
"""
Set up the Move command
:param string: name Name for the thread
:param string: input_name Name of the file or directory to move
:param string: output_name Name of the location to move the file or directory to
:param bool: recreate If true will recreate an empty copy of the file or directory
"""
# pylint: disable=arguments-differ
self.thread_name = name
self._input_name = Replacements().replace(input_name, additional=self.replacements(input_name))
self._output_name = Replacements().replace(output_name, additional=self.replacements(output_name))
self._recreate = recreate
def _write(self, document, text):
""" Write text into the document """
if isinstance(text, list):
for item in text:
value = getattr(item, 'value') if isinstance(
item, object) and hasattr(item, 'value') else item
if self._prepend is not None and self._prepend != '':
value = '{0} {1}'.format(self._prepend, value)
document.add_list(Replacements().replace(value),
style=List.STYLE_MAPPINGS[self._style])
elif text is not None:
if self._prepend is not None and self._prepend != '':
text = '{0} {1}'.format(self._prepend, text)
document.add_list(Replacements().replace(text),
style=List.STYLE_MAPPINGS[self._style])
def setup(self,
query,
max_results=0,
fields=None,
collate=None,
distinct=False,
namespace=None,
group_by=None):
"""
Initialise the filter
@param query string The query to search with
@param max_results [bool|int] If False will retrieve all matching issues
in batches of 50
@param fields list An optional list of fields to retrieve.
If empty, retrieves all fields
"""
# pylint: disable=arguments-differ
# It is understood that the arguments will always differ
# from the super() class on this method due to the use of
# *args and **kwargs in the call from __init__
# pylint: disable=too-many-arguments
# This class requires a large number of arguments to come from the config
self._query = Replacements().replace(query)
self._fields = fields
self._observers = []
self._max_results = max_results
collation = Collation.get(
collate, namespace=namespace) if collate is not None else collate
self._results = ResultList(collate=collation,
distinct=distinct,
namespace=namespace)
self._group_by = group_by
def _load(self):
"""
Attempts to load the configuration file from JSON
"""
for location in self._get_locations():
Logger().debug(
'Checking for config file in \'{0}\''.format(location))
path = os.path.join(str(location), self._filename)
try:
with open(path) as configuration_file:
self._configuration = json.load(
configuration_file,
object_hook=lambda x: namedtuple('Config', x.keys())
(*x.values()))
Logger().debug(
'Using configuration from \'{0}\'.'.format(path))
break
except IOError:
pass
if not self._configuration:
raise IOError(
'Invalid configuration file or path not provided (got \'{0}\')'
.format(self._filename))
self.validate_config(self._required_root_elements)
if hasattr(self._configuration, 'replacements'):
self._replacements = Replacements(
configuration=self._configuration.replacements)
if not self._dont_parse:
self._parse_flags()
self._is_loaded = True
def setup(self, query, fields=None, collate=None, output_path='/tmp'):
""" Set up the attachments object """
# pylint: disable=arguments-differ
self._content = Filter(query, max_results=50, fields=fields)
self.threadmanager.append(self._content)
self.projectmanager = Configuration().manager
self._collate = collate.split(',')
self._output_path = Replacements().replace(output_path)
create_directory(self._output_path)
def render(self, document):
"""
Render the text of the abstract into a series of paragraphs
@param document ReportManager
"""
Logger().info('Adding section abstract')
for paragraph in self._content:
document.add_paragraph(Replacements().replace(paragraph))
def _write_heading(self, alternate_title=''):
"""
Writes a heading for the table
@param alternate_title string An optional title to use in place
"""
level = 3 if alternate_title == '' else 4
title = self.title if alternate_title == '' else alternate_title
self._report.add_heading(Replacements().replace(title), level)
def test_replacements_find_returns_none_if_replacement_is_not_cofigured(
self):
self.tearDown()
Config = namedtuple('Config', 'name type value overridable')
config = Config(name='test',
type='string',
value='this is a test',
overridable=False)
class Namespace():
test = None
def __init__(self, test=None):
self.test = test
namespace = Namespace(test=None)
replacements = Replacements(configuration=[config])
self.assertEquals(None, replacements.find('helloworld'))
def build_document(self):
""" Creates the document structure """
try:
self._document_controller.build()
final = Replacements().find('FINAL').value
if final in ['1', 'True', 'TRUE', 'true']:
pipelines = self._document_controller.threadmanager.find('Deployment Pipelines')
if pipelines is not None or len(pipelines) > 0:
for pipeline in pipelines:
self.pipeline_trigger(pipeline)
self._document_ready = True
except:
self._document_ready = False
raise
def pipeline_trigger(self, pipeline):
"""
Triggers a pipeline on Jenkins
FUTURE - move out to Jenkins API wrapper
"""
replacements = Replacements()
pipeline = replacements.replace(pipeline)
matches = re.search(r'(http.*\/).*', pipeline)
if matches:
pipeline = matches.groups(0)[0].rstrip('/')
Logger().info('Triggering pipeline {0}'.format(pipeline))
params = {}
auth = (self._configuration.jenkins.user, self._configuration.jenkins.password)
pipeline = '{0}/{1}'.format(pipeline, replacements.replace('{TRIGGER_URI}'))
response = requests.post(pipeline, auth=auth, params=params, verify=False)
if response.status_code != 200:
Logger().error(
'Error whilst triggering pipeline - server returned status {0}'.format(
response.status_code
)
)
Logger().info('Done {0}'.format(pipeline))
def render(self, document):
""" Render the paragraph text """
# pylint: disable=arguments-differ
Logger().info('Writing paragraph')
if isinstance(self._content, str):
document.add_paragraph(Replacements().replace(self._content))
else:
for i, run in enumerate(self._content):
if i == 0:
document.add_paragraph(run)
elif isinstance(run, str):
document.add_run(run)
else:
document.add_run(run.text, style=run.style)
def setup(self,
name,
command='',
input_directory='',
output_directory='',
input_pattern='*',
strip='',
output_extension='',
maxthreads=1,
wait_for=None):
""" Set up the fileloop object """
# pylint: disable=arguments-differ,too-many-arguments
self.thread_name = name
self._command = command
self._input_directory = Replacements().replace(input_directory)
self._output_directory = Replacements().replace(
output_directory, additional=self.replacements(output_directory))
self._strip = strip
self._input_pattern = input_pattern
self._output_extension = output_extension
self._maxthreads = maxthreads
self._wait_for = None
if wait_for is not None:
self._wait_for = self.threadmanager.find(wait_for)
def _build_command_list(self):
"""
Builds the current command set, one for each file
"""
files = [
os.path.join(self._input_directory, filename)
for filename in os.listdir(self._input_directory)
if os.path.isfile(os.path.join(self._input_directory, filename))
and fnmatch.fnmatch(filename, self._input_pattern)
]
config = namedtuple(
'config', 'name command input_directory output_directory wait_for')
for index, filename in enumerate(files):
additional = {}
additional['filename'] = filename
output_file, current_extension = os.path.splitext(filename)
if self._strip != '' and self._output_extension == '':
self._output_extension = current_extension
if self._strip == '' and self._output_extension != '':
self._strip = '.' + current_extension
if self._strip != '':
output_file = os.path.basename(filename)
output_file = re.sub(r'{0}'.format(self._strip), '',
output_file)
if self._output_extension != '':
output_file = output_file + '.' + self._output_extension
additional['output'] = os.path.join(self._output_directory,
output_file)
additional['logfile'] = os.path.join(
ThreadableCommand.logdir(), '{0}.log'.format(self.thread_name))
command_string = Replacements().replace(self._command,
additional=additional)
self._commands.append(
ThreadableCommand(
self._thread_manager,
config(name='{0}-{1}'.format(self._name, index),
command=command_string,
input_directory=self._input_directory,
output_directory=self._output_directory,
wait_for=None)))
def render(self, report):
""" render the current object """
Logger().info('\033[1mWriting section {0}\033[0m'.format(
self.title if self.title is not None else ''))
if len(self._structure) > 0:
using_tables = len(self._structure)
for item in self._structure:
if isinstance(item, Table) and isinstance(item.rows, Filter):
using_tables = using_tables - 1 if len(
item.rows.results) == 0 else using_tables
if using_tables == 0:
# we probably have nothing to render
Logger().info('Empty section. Skipping...')
return
report.add_heading(str(Replacements().replace(self._title)),
self.level)
self._abstract.render(report)
for part in self._structure:
part.render(report)
def file_collator(path, attachments):
"""
This method unpacks all zip files and copies the resulting SQL into a temporary location
Once complete, it repackages the temporary location and moves it into the Workspace for
attachment to the release instruction email
"""
Logger().info('Collating SQL files from attachments')
package_name = Replacements().replace('mss-platform-release-{FIX_VERSION}').replace('/', '-')
destination = os.path.join(path, package_name)
up_dir = os.path.join(destination, 'up')
down_dir = os.path.join(destination, 'down')
package_dir = os.path.join(os.getcwd(), 'packages')
if not os.path.exists(package_dir):
create_directory(package_dir)
create_directory(destination)
create_directory(up_dir)
create_directory(down_dir)
for attachment in attachments:
if hasattr(ReleaseInstructions, attachment.extension.lower() + '_handler'):
filename = os.path.join(path, attachment.filename)
Logger().debug(
'Calling handler \'' + attachment.extension + '_handler\' for file ' + filename
)
try:
getattr(ReleaseInstructions, attachment.extension.lower() + '_handler')(
destination,
os.path.join(path, attachment.filename)
)
except Exception as exception:
Logger().error('Failed to add file \'' + filename + '\' to archive')
Logger().error('reason: ' + str(exception))
filename = mkzip(destination, os.path.join(path, package_name + '.zip'))
shutil.move(os.path.join(destination, filename), os.path.join(package_dir, os.path.basename(filename)))
return os.path.basename(filename)
def test_replacements_validator_updates_replacements(self):
self.tearDown()
Config = namedtuple('Config', 'name type value overridable')
config = Config(name='TEST',
type='string',
value='this is a test',
overridable=False)
class Namespace():
test = None
def __init__(self, test=None):
self.test = test
namespace = Namespace(test=None)
replacements = Replacements(configuration=[config])
replacement_validator = ReplacementsValidator(None, 'test')
replacement_validator.__call__(None, namespace, 'hello world')
self.assertEquals('hello world', replacements[0].value)
def _write_table(self, results):
"""
Write out a set of results as a table
"""
data = results if isinstance(results, list) else results.results
if hasattr(data, 'dataframe') and data.dataframe is None:
for index, row in enumerate(data):
data[index] = [
Replacements().replace(cell)
if isinstance(cell, str) else cell for cell in row
]
if len(data) > Table.MAX_ROWS and (hasattr(data, 'dataframe')
and data.dataframe is not None):
data.dataframe.to_csv(results.name + '.csv', index=False)
self._report.add_paragraph(
'Table results written to file \'{0}\''.format(results.name +
'.csv'))
else:
self._report.add_table(headings=self._columns,
data=data,
style=self._style)
def _parse_content(self, rows):
"""
Parses a list of row/cell data, forming filters where necessary
@param rows list
@return list
"""
for row_index, row in enumerate(rows):
for cell_index, cell in enumerate(row):
if isinstance(cell, tuple):
try:
rows[row_index][cell_index] = Table._parse_filter(cell)
self.threadmanager.append(rows[row_index][cell_index])
#pylint: disable=broad-except
except Exception as exception:
Logger().warning(
'Failed to create filter from config object')
Logger().warning('Exception was:')
Logger().warning(exception)
elif isinstance(cell, str):
rows[row_index][cell_index] = Replacements().replace(cell)
return rows
def render(self, document):
""" Render the paragraph text """
# pylint: disable=arguments-differ
Logger().info('Writing list {0}'.format(
self.title if self.title is not None else ''))
if self.title is not None:
document.add_heading(Replacements().replace(self.title), 3)
# Lists should be unique
for item in self:
# Only append the first result, discard any extra
text = None
if isinstance(item, str):
text = item
elif isinstance(item, ResultListItemAbstract):
text = getattr(item, self._field) if hasattr(
item, self._field) else ''
elif (isinstance(item.results, list) and len(item.results) == 1
and isinstance(item.results[0], ResultListItemAbstract)):
text = getattr(item.results[0], self._field) if hasattr(
item.results[0],
self._field) else item.results[0].description
self._write(document, text)
def _cleanup(self):
string = Replacements().replace('{BASE_PATH}') + '*'
pattern = glob.glob(string)
for path in pattern:
shutil.rmtree(path)
os.makedirs(Replacements().replace('{BASE_PATH}'))
def create_title_page(self):
""" Creates a title page for the report using settings from configuration """
self.add_heading(Replacements().replace(self.configuration.report.title), 0)
self.add_heading(Replacements().replace(self.configuration.report.subtitle), 1)
for paragraph in read_file(self.configuration.report.abstract).split("\n"):
self.add_paragraph(Replacements().replace(paragraph))
def logdir():
""" Get a log directory for the command output """
path = os.path.join(Replacements().replace('{BASE_PATH}'), 'log')
if not os.path.exists(path):
os.makedirs(path)
return path