def _has_diff_expression(self):
diffs = self._get_diffs_from_current()
exp_dict = {}
for target_diff in self.target_diffs:
if type(target_diff) is not dict:
raise ConfigError(
'Invalid diff configuration specified:\n{}'.format(
yaml.dump(target_diff)))
id = get_or_default(target_diff, ID, None, str)
if id is not None:
path = get_or_default(target_diff, PATH, None, str)
if path is not None:
for detected_diff in diffs:
if re.match(target_diff[PATH],
detected_diff.a_rawpath.decode(UTF8)):
LOG.info('[{}] path pattern {} matches {}'.format(
self.id, target_diff[PATH],
detected_diff.a_rawpath.decode(UTF8)))
content = get_or_default(target_diff, CONTENT,
None, str)
if content is not None:
if id not in exp_dict or not exp_dict[id]:
exp_dict[id] = self._has_content_diff(
content, detected_diff)
else:
exp_dict[id] = True
if id not in exp_dict:
exp_dict[id] = False
return evaluate_expression(self.expression, exp_dict)
def run_prehooks(self):
if self.prehooks is not None:
LOG.log(
PLUMBER_LOGS,
wrap_in_dividers("Running prehooks for {}".format(
self.config[ID]),
divider_char='-'))
super(PlumberPipe, self).run_prehooks()
def get_data(self):
try:
with open(self.path) as file:
return self.parser.full_load(file)
except FileNotFoundError:
LOG.warning(
'File {} not found, will be created upon persistence'.format(
self.path))
return {}
def run_posthooks(self, last_result):
if self.posthooks_execute and (
self.posthooks is not None or
(last_result == SUCCESS and self.posthooks_success is not None) or
(last_result == FAILURE and self.posthooks_failure is not None)):
LOG.log(
PLUMBER_LOGS,
wrap_in_dividers("Running global posthooks", divider_char='-'))
super(PlumberPlanner, self).run_posthooks(last_result)
def save_data(self, content, info=None):
super().save_data(content, info)
self.repo.git.add(self.path)
if info is not None:
self.repo.index.commit(
':wrench::construction_worker: [Plumber]\n{}'.format(info))
origin = self.repo.remote(name='origin')
origin.push()
else:
LOG.error('Commit content not provided')
def substitute_env_var(string):
match = self.pattern.match(string)
if match is None:
return string
starting, env_var, remaining = match.groups()
LOG.debug(
'Found environment variable {}, will be substituted if found'.
format(env_var))
return substitute_env_var(starting) + os.getenv(
env_var, '{env.' + env_var + '}') + remaining
def run_posthooks(self, last_result):
if self.posthooks is not None or (
last_result == SUCCESS and self.posthooks_success
is not None) or (last_result == FAILURE
and self.posthooks_failure is not None):
LOG.log(
PLUMBER_LOGS,
wrap_in_dividers("Running posthooks for {}".format(
self.config[ID]),
divider_char='-'))
super(PlumberPipe, self).run_posthooks(last_result)
def main_execution_logic():
if self.pipes is None:
raise ExecutionFailure('No pipes configured')
self.results = [{
ID: pipe.config[ID],
STATUS: UNKNOWN,
PIPE: pipe
} for pipe in self.pipes]
for item in self.results:
LOG.log(
PLUMBER_LOGS,
wrap_in_dividers('Pipe evaluation for [{}]'.format(
item[ID])))
def pipe_execution_logic():
if item[PIPE].evaluate():
self.posthooks_execute = True
item[STATUS] = DETECTED
LOG.log(
PLUMBER_LOGS,
'Detected change on pipe {}, starting execution'.
format(item[ID]))
item[PIPE].execute()
LOG.log(PLUMBER_LOGS,
'Steps for pipe {} executed'.format(item[ID]))
item[STATUS] = EXECUTED
else:
LOG.log(
PLUMBER_LOGS,
'No change detected on pipe {}. Moving on'.format(
item[ID]))
item[STATUS] = NOT_DETECTED
if item[STATUS] != NOT_DETECTED:
checkpoint = item[PIPE].get_new_checkpoint()
if checkpoint is not None:
self.current_checkpoint[
item[PIPE].config[ID]] = checkpoint
try:
item[PIPE].wrap_in_hooks(pipe_execution_logic)()
except Exception as e:
item[STATUS] = FAILED
raise e
return self.results
def _get_diffs_from_current(self):
if COMMIT not in self.checkpoint:
return None
found_diffs = list()
commit_found = False
for commit in self.repo.iter_commits():
diffs = self.repo.head.commit.diff(commit)
for diff in diffs:
found_diffs.append(diff)
commit_found = str(commit) == self.checkpoint[COMMIT]
if commit_found:
break
if not commit_found:
LOG.warn('[{}] traversed all git log, checkpoint commit not found'.
format(self.id))
LOG.info('[{}] detected diffs since last run:\n{}\n'.format(
self.id,
''.join(f'\n\t {l.a_rawpath.decode(UTF8)}' for l in found_diffs)))
return found_diffs
def _has_diff_all(self):
diffs = self._get_diffs_from_current()
for target_diff in self.target_diffs:
if type(target_diff) is not dict:
raise ConfigError(
'Invalid diff configuration specified:\n{}'.format(
yaml.dump(target_diff)))
for detected_diff in diffs:
path = get_or_default(target_diff, PATH, None, str)
if path is not None:
if re.match(path, detected_diff.a_rawpath.decode(UTF8)):
LOG.info('[{}] path pattern {} matches {}'.format(
self.id, path,
detected_diff.a_rawpath.decode(UTF8)))
content = get_or_default(target_diff, CONTENT, None,
str)
if content is not None:
if self._has_content_diff(content, detected_diff):
return True
else:
return True
return False
def evaluate(self):
if self.result is None:
if self.active_branch is not None and str(
self.repo.active_branch) != self.active_branch:
LOG.info(
'[{}] Not on active branch, conditional disabled'.format(
self.id))
self.result = False
return self.result
if self.target_branch is not None and str(
self.repo.active_branch) != self.target_branch:
previous_branch = str(self.repo.active_branch)
try:
LOG.info('[{}] checking out target branch {}'.format(
self.id, self.target_branch))
self.repo.git.checkout(self.target_branch)
self.result = self._has_diff()
finally:
self.repo.git.checkout(previous_branch)
else:
self.result = self._has_diff()
return self.result
def _has_diff(self):
if COMMIT not in self.checkpoint:
LOG.warning('[{}] no checkpoint found, pipe will be executed')
return True
if self.expression is not None:
LOG.info('[{}] detecting through expression evaluation: {}'.format(
self.id, self.expression))
return self._has_diff_expression()
else:
LOG.info('[{}] detecting any of the diffs'.format(self.id))
return self._has_diff_all()
def save_new_checkpoint(current_result):
LOG.log(PLUMBER_LOGS, wrap_in_dividers('Checkpointing'))
if checkpoint and contains_activity(
self.results) and current_result == SUCCESS or (
current_result == FAILURE
and self.checkpoint_unit == PIPE):
LOG.log(PLUMBER_LOGS,
'Changes performed, persisting a new checkpoint...')
self.checkpoint_store.save_data(
self.current_checkpoint,
create_execution_report(self.results, gitmojis=True))
else:
LOG.log(
PLUMBER_LOGS,
'Skip checkpointing due to inactivity, error or disabling')
def execute(self):
if self.steps is not None:
if self.batch:
script = ''.join(f'\n {l}' for l in self.steps)
result = self._run_script(script=script)
self.results.append(result)
if result[RETURN_CODE] != 0:
LOG.error(create_execution_log(result))
raise ExecutionFailure(
'Step \n{} exited with code {}'.format(
script, result[RETURN_CODE]))
else:
LOG.log(PLUMBER_LOGS, create_execution_log(result))
else:
for step in self.steps:
result = self._run_script(script=step)
self.results.append(result)
if result[RETURN_CODE] != 0:
LOG.error(create_execution_log(result))
raise ExecutionFailure(
'Step {} exited with code {}'.format(
step, result[RETURN_CODE]))
else:
LOG.log(PLUMBER_LOGS, create_execution_log(result))
def pipe_execution_logic():
if item[PIPE].evaluate():
self.posthooks_execute = True
item[STATUS] = DETECTED
LOG.log(
PLUMBER_LOGS,
'Detected change on pipe {}, starting execution'.
format(item[ID]))
item[PIPE].execute()
LOG.log(PLUMBER_LOGS,
'Steps for pipe {} executed'.format(item[ID]))
item[STATUS] = EXECUTED
else:
LOG.log(
PLUMBER_LOGS,
'No change detected on pipe {}. Moving on'.format(
item[ID]))
item[STATUS] = NOT_DETECTED
if item[STATUS] != NOT_DETECTED:
checkpoint = item[PIPE].get_new_checkpoint()
if checkpoint is not None:
self.current_checkpoint[
item[PIPE].config[ID]] = checkpoint
def initialize_default_checkpoint_store():
LOG.debug('Initialized with default YAML checkpoint store')
checkpoint_store = YamlFileStore()
checkpoint_store.configure({PATH: DEFAULT_CHECKPOINT_FILENAME})
return checkpoint_store
def run_prehooks(self):
if self.prehooks is not None:
LOG.log(
PLUMBER_LOGS,
wrap_in_dividers("Running global prehooks", divider_char='-'))
super(PlumberPlanner, self).run_prehooks()
def create_checkpoint(self):
LOG.info('[{}] New checkpoint {}'.format(self.id, self.new_checkpoint))
return {COMMIT: self.new_checkpoint}