def decrypt_file_task(self, password: str, source: Path, target: Path):
if password is None:
return TaskFailed("No passphrase specified; please use doit passphrase=**** decrypt")
target.parent.mkdir(exist_ok=True)
key = get_key(self.salt, password)
try:
decrypt_file(key, source, target)
except InvalidToken:
return TaskFailed("Incorrect password, could not decrypt files")
def check_hash(target_fn, expected):
print(' * Checking hash of {0}'.format(target_fn), file=sys.stderr)
if expected == hashfile(target_fn):
return True
else:
os.remove(target_fn)
return TaskFailed('{0} has non-matching hash; download error?'.format(target_fn))
def do_merge(task, pos_arg_val):
"""Merge current branch with given branch (default master) and push it."""
if task.error is not None:
return TaskFailed(task.error)
branch = pos_arg_val[0] if len(pos_arg_val) > 0 else "master"
with checkout(branch) as current_branch:
check_call(["git", "merge", "--no-ff", current_branch])
check_call(["git", "push", "origin", branch])
def make_k8s_def():
import jinja2
config = get_k8s_config()
if 'error' in config:
raise ValueError(config['error'])
context = {}
registry = config.pop('image_registry', None)
if registry:
context['image_registry'] = '%s/' % registry
for key in ['postgres', 'minio']:
value = config.pop('%s_volume' % key, None)
if isinstance(value, str):
pass
elif value:
value = 'reproserver-{key}-{tier}'.format(key=key,
tier=config['tier'])
else:
value = ''
context['%s_volume' % key] = value
context['use_minio'] = use_minio = config.pop('use_minio', False)
if use_minio:
context['s3_url'] = 'http://reproserver-minio-{tier}:9000'.format(
tier=config['tier'])
context['s3_bucket_prefix'] = 'minio'
else:
context['s3_url'] = ''
context['s3_bucket_prefix'] = config.pop('s3_bucket_prefix')
context['s3_client_url'] = config.pop('s3_client_url', '')
if 'tag' not in config:
return TaskFailed("You must set a tag explicitly, either in "
"config.yml or on the command-line")
context['tag'] = config.pop('tag')
if context['tag'].startswith(':'):
context['version'] = context['tag'][1:]
else:
context['version'] = context['tag']
context['tier'] = config.pop('tier')
context['postgres_db'] = config.pop('postgres_database', 'reproserver')
context['storage_driver'] = config.pop('storage_driver')
context['liveness_probe_period_seconds'] = config.pop(
'liveness_probe_period_seconds', 30)
if config:
sys.stderr.write("Warning: unrecognized config options:\n")
for k in config:
sys.stderr.write(" %s\n" % k)
env = jinja2.Environment(loader=jinja2.FileSystemLoader('.'))
template = env.get_template('k8s.tpl.yml')
with open('k8s.yml', 'w') as out:
out.write(template.render(context))
def check_sum(path, type='md5', digest=''):
if not os.path.isfile(path):
return TaskFailed("'{}' is not a file. Cannot checksum.".format(path))
if type == 'md5':
m = hashlib.md5()
with open(path, 'rb') as f:
for chunk in lazy_read(f):
m.update(chunk)
md5_sum = m.hexdigest()
if digest == md5_sum:
print("'{}' ({}) matches the md5sum. ".format(path, digest))
return True
else:
return TaskFailed("'{}' ({}) do not match the md5sum.".format(
path, digest))
else:
return TaskFailed("'{}' is not a suppported sum type.".format(type))
def do_release(task, pos_arg_val):
"""Bump version and push to master."""
if task.error is not None:
return TaskFailed(task.error)
choices = ("major", "minor", "patch")
msg = "{} PART argument. Availlable choices are: {}."
if len(pos_arg_val) == 0:
return TaskFailed(msg.format("Missing", str(choices)))
part = pos_arg_val[0]
if part not in choices:
return TaskFailed(msg.format("Wrong", str(choices)))
with checkout("master"):
print("Commits since", task.last_version)
print(task.unreleased_commits)
check_call(["poetry", "run", "bump2version", "-n", "--verbose", part])
proceed = input("Do you agree with the changes? (y/n): ")
if proceed.lower().strip().startswith("y"):
check_call(["poetry", "run", "bump2version", part])
check_call(["git", "push", "--tags", "origin", "master"])
else:
return TaskFailed("Cancelled by user.")
check_call(["git", "merge", "--no-ff", "master"])
current_branch = get_stdout(GIT_CURRENT_BRANCH_CMD).strip("\n\r ")
check_call(["git", "push", "origin", current_branch])
def execute(self, out=None, err=None):
"""Execute command action
both stdout and stderr from the command are captured and saved
on self.out/err. Real time output is controlled by parameters
:param out: None - no real time output a file like object (has
write method)
:param err: idem
:return failure: see CmdAction.execute
"""
# set std stream
old_stdout = sys.stdout
output = StringIO()
out_writer = Writer()
# capture output but preserve isatty() from original stream
out_writer.add_writer(output, old_stdout.isatty())
if out:
out_writer.add_writer(out)
sys.stdout = out_writer
old_stderr = sys.stderr
errput = StringIO()
err_writer = Writer()
err_writer.add_writer(errput, old_stderr.isatty())
if err:
err_writer.add_writer(err)
sys.stderr = err_writer
kwargs = self._prepare_kwargs()
# execute action / callable
try:
returned_value = self.py_callable(*self.args, **kwargs)
except Exception as exception:
return TaskError("PythonAction Error", exception)
finally:
# restore std streams /log captured streams
sys.stdout = old_stdout
sys.stderr = old_stderr
self.out = output.getvalue()
self.err = errput.getvalue()
# if callable returns false. Task failed
if returned_value is False:
return TaskFailed("Python Task failed: '%s' returned %s" %
(self.py_callable, returned_value))
elif returned_value is True or returned_value is None:
pass
elif isinstance(returned_value, six.string_types):
self.result = returned_value
elif isinstance(returned_value, dict):
self.values = returned_value
self.result = returned_value
elif isinstance(returned_value, TaskFailed) \
or isinstance(returned_value, TaskError):
return returned_value
else:
return TaskError(
"Python Task error: '%s'. It must return:\n"
"False for failed task.\n"
"True, None, string or dict for successful task\n"
"returned %s (%s)" %
(self.py_callable, returned_value, type(returned_value)))
def error_sample(): return TaskFailed("too bad")
ye_olde_action = action.PythonAction(error_sample)
def wrapper(*args, **kwargs):
try:
return fn(*args, **kwargs)
except subprocess.CalledProcessError as e:
return TaskFailed('Error while executing {}'.format(' '.join(
e.cmd)))
def check_towncrier_fragments(changed):
for path in map(Path, changed):
if not (is_towncrier_fragment(path) or path == CHANGELOG_TEMPLATE):
return TaskFailed(
f"{CHANGELOG_PATH.name}/{path.name} is not a valid towncrier "
f"fragment name.")
def error_sample():
return TaskFailed("too bad")
def check_module(*modules):
for module in modules:
try:
importlib.import_module(module)
except ImportError:
return TaskFailed("'{}' module not found.".format(module))
def check_cmd(*commands):
for command in commands:
if not shutil.which(command):
return TaskFailed("'{}' not found.".format(command))
def do_release(task, pos_arg_val):
"""Bump version and push to master."""
if task.error is not None:
return TaskFailed(task.error)
def do_merge(task, pos_arg_val):
"""Merge current branch with given branch (default master) and push it."""
if task.error is not None:
return TaskFailed(task.error)
return False
task.last_version = get_stdout(GIT_LAST_VERSION_CMD).strip("\n\r ")
task.unreleased_commits = get_stdout(
GIT_BRIEF_LOG_CMD + [task.last_version + ".."]
)
return len(task.unreleased_commits) == 0
def do_release(task, pos_arg_val):
"""Bump version and push to master."""
if task.error is not None:
return TaskFailed(task.error)
choices = ("major", "minor", "patch")
msg = "{} PART argument. Availlable choices are: {}."
if len(pos_arg_val) == 0:
return TaskFailed(msg.format("Missing", str(choices)))
part = pos_arg_val[0]
if part not in choices:
return TaskFailed(msg.format("Wrong", str(choices)))
with checkout("master"):
print("Commits since", task.last_version)
print(task.unreleased_commits)
check_call(["poetry", "run", "bump2version", "-n", "--verbose", part])
proceed = input("Do you agree with the changes? (y/n): ")
if proceed.lower().strip().startswith("y"):
check_call(["poetry", "run", "bump2version", part])
check_call(["git", "push", "--tags", "origin", "master"])
else:
return TaskFailed("Cancelled by user.")
check_call(["git", "merge", "--no-ff", "master"])
current_branch = get_stdout(GIT_CURRENT_BRANCH_CMD).strip("\n\r ")