class MavenRepository(object):
"""Maven repository.
A Maven repository consists in:
- a main local (writable) repository,
- a set of "remote" (read-only) repositories.
A remote repository is specified via a URL (file:// or http[s]://).
Any artifact from a remote repository may be brought locally.
Snapshot artifacts are always checked for newer versions agaist the remote
repositories.
"""
def __init__(
self,
local=DEFAULT,
remotes=(),
exclusions=frozenset(),
remote_snapshots=False,
):
"""Initializes a Maven repository.
Args:
local: Path of the directory for the local repository.
remotes: Ordered list of remote repository URLs (file:// or http[s]://).
exclusions: Optional set of excluded Maven repositories.
remote_snapshots: Whether to allow fetching snapshot artifacts remotely.
"""
if local is DEFAULT:
local = os.path.join(os.environ["HOME"], ".m2", "repository")
if len(urllib.request.urlparse(local).scheme) == 0:
local = "file://%s" % local
assert (urllib.request.urlparse(local).scheme == "file"), (
"Invalid local repository path: %r" % local)
self._local_path = local
self._local = RemoteRepository(path=self._local_path,
use_checksum=True)
self._remote_paths = tuple(remotes)
self._remotes = tuple(map(RemoteRepository, self._remote_paths))
self._exclusions = frozenset(exclusions)
self._remote_snapshots = remote_snapshots
logging.debug(
"Initialized Maven repository with local=%s and remotes=%r",
self.local, self.remotes)
def __str__(self):
return "MavenRepository(local=%s, remotes=%s, exclusions=%s)" % (
self.local,
",".join(self.remotes),
",".join(self.exclusions),
)
def __repr__(self):
return str(self)
@property
def local(self):
"""Local writable Maven repository."""
return self._local
@property
def remotes(self):
"""Ordered list of remote Maven repositories."""
return self._remotes
@property
def exclusions(self):
"""Set of excluded Maven remote repositories."""
return self._exclusions
def add_remote(self, remote):
"""Adds a new remote repository, unless it belongs to the exclusion list.
Args:
remote: URL of the Maven remote repository.
Returns:
Whether the remote repository was added or not.
"""
if remote in self._remote_paths:
# Remote repository already registered.
pass
elif remote in self.exclusions:
logging.debug("Ignoring excluded Maven remote repository: %r",
remote)
else:
logging.debug("Adding Maven remote repository: %r", remote)
self._remote_paths += (remote, )
self._remotes += (RemoteRepository(remote), )
def get(self, artf):
"""Retrieves an artifact locally.
If the artifact is a snapshot (version ends with '-SNAPSHOT'),
all remotes are checked for a newer version.
Args:
group: Artifact group ID.
artifact: Artifact ID.
version: Artifact version.
type: Artifact type, aka. packaging ('jar', 'pom', etc).
classifier: Optional classifier (eg. 'test' or 'release').
Returns:
The path of the artifact in the local repository.
None if the artifact cannot be found.
"""
return self.Get(
group=artf.group_id,
artifact=artf.artifact_id,
version=artf.version,
type=artf.packaging,
classifier=artf.classifier,
)
def Get(self, group, artifact, version, type, classifier=None):
"""Retrieves an artifact locally.
Deprecated, this method is going away: use get(artifact) instead.
If the artifact is a snapshot (version ends with '-SNAPSHOT'),
all remotes are checked for a newer version.
Args:
group: Artifact group ID.
artifact: Artifact ID.
version: Artifact version.
type: Artifact type, aka. packaging ('jar', 'pom', etc).
classifier: Optional classifier (eg. 'test' or 'release').
Returns:
The path of the artifact in the local repository.
None if the artifact cannot be found.
"""
coordinate = dict(
group=group,
artifact=artifact,
version=version,
type=type,
classifier=classifier,
)
logging.log(LOG_LEVEL.DEBUG_VERBOSE,
"Getting artifact with coordinate %r", coordinate)
path = self.local.GetPath(**coordinate)
parsed = urllib.request.urlparse(self.local.get_url(**coordinate))
assert (parsed.scheme == "file")
local_path = parsed.path
md5_path = "%s.md5" % local_path
sha1_path = "%s.sha1" % local_path
# Artifact is a snapshot, resolve it first if allowed:
if version.endswith("-SNAPSHOT") and self._remote_snapshots:
# Find the most recent snapshot version from all the remote repositories:
def ScanRemotes():
for remote in self.remotes:
resolved = remote.resolve(**coordinate)
if resolved is None: continue
# Allow for snapshots resolved in a local repository:
# if resolved["snapshot_version"] is None: continue
yield (resolved, remote)
# Snapshot on remote repositories are expected to have a snapshot_version:
best_remote = None
best_version = dict(snapshot_version="", **coordinate)
# Snapshot built locally into a local repository has no snapshot_version.
# This lists the local repositories where unversioned snapshots are found:
local_repos = list()
for (resolved, remote) in ScanRemotes():
if resolved["snapshot_version"] is None:
local_repos.append(remote)
elif best_version["snapshot_version"] < resolved[
"snapshot_version"]:
best_remote = remote
best_version = resolved
if (best_remote is None) and (len(local_repos) == 0):
raise ArtifactNotFoundError(
"Artifact %s:%s:%s:%s:%s not found in remote repositories"
% (coordinate["group"], coordinate["artifact"],
coordinate["classifier"], coordinate["type"],
coordinate["version"]))
elif len(local_repos) == 0:
logging.debug("Artifact resolved to %s in remote %s",
best_version, best_remote)
elif best_remote is None:
assert (len(local_repos) == 1), \
("Multiple snapshot local copies of %r" % coordinate)
local_repo = local_repos[0]
parsed = urllib.request.urlparse(
local_repo.get_url(**coordinate))
assert (parsed.scheme == "file")
local_path = parsed.path
return local_path
else:
raise Error(
"Multiple snapshot copies in local repositories and remote repositories: %r"
% coordinate)
(http_reply, md5, sha1) = best_remote.open(**best_version)
try:
# Do we have this snapshot artifact locally already:
if (os.path.exists(local_path) and os.path.exists(md5_path)
and os.path.exists(sha1_path)
and md5 == self.local.read_md5_file(path)
and sha1 == self.local.read_sha1_file(path)
and (get_file_fingerprints(local_path)
== (self.local.read_md5_file(path),
self.local.read_sha1_file(path)))):
logging.log(LOG_LEVEL.DEBUG_VERBOSE,
"Snapshot artifact found locally: %r",
local_path)
return local_path
finally:
http_reply.close()
logging.log(LOG_LEVEL.DEBUG_VERBOSE,
"Snapshot artifact not found locally: %r", coordinate)
remotes = (best_remote, )
# Artifact is a snapshot but we do not allow remote snapshots:
elif version.endswith("-SNAPSHOT") and not self._remote_snapshots:
logging.log(LOG_LEVEL.DEBUG_VERBOSE,
"Restricting snapshot artifact to local FS: %r",
local_path)
if os.path.exists(local_path):
logging.log(LOG_LEVEL.DEBUG_VERBOSE,
"Local snapshot artifact found: %r", local_path)
return local_path
else:
logging.debug("Local snapshot artifact not found: %r",
local_path)
return None
else:
# Do we have this non-snapshot artifact locally already?
if (os.path.exists(local_path) and
(os.path.getsize(local_path) > 0) # FIXME quick workaround
and os.path.exists(md5_path) and os.path.exists(sha1_path)
and (get_file_fingerprints(local_path)
== (self.local.read_md5_file(path),
self.local.read_sha1_file(path)))):
# Yes, artifact found locally, with matching checksums:
logging.log(
LOG_LEVEL.DEBUG_VERBOSE,
"Artifact found locally with matching checksums: %r",
local_path)
return local_path
else:
# Look for the artifact in the configured remotes:
logging.debug("Artifact not found locally: %r", local_path)
remotes = self.remotes
# Artifact does not exist locally.
# Try each remote repository on after another,
# pick the first that contains the artifact we are looking for:
for remote in remotes:
try:
open_result = remote.open(**coordinate)
if open_result is None:
continue
(http_reply, md5, sha1) = open_result
try:
base.make_dir(os.path.dirname(local_path))
(actual_md5, actual_sha1) = RemoteRepository.read_to_file(
http_reply=http_reply,
output_path=local_path,
)
if md5 is None:
logging.debug("No MD5 sum for %r from %s", local_path,
remote.path)
md5 = actual_md5
elif md5 != actual_md5:
logging.error(
"MD5 mismatch for %r from %r: expected %r, got %r",
local_path, remote.path, md5, actual_md5)
if sha1 is None:
logging.debug("No SHA1 sum for %r from %s", local_path,
remote.path)
sha1 = actual_sha1
elif sha1 != actual_sha1:
logging.error(
"SHA1 mismatch for %r from %r: expected %r, got %r",
local_path, remote.path, sha1, actual_sha1)
if (md5 == actual_md5) and (sha1 == actual_sha1):
logging.debug("Writing MD5 sum for %r", local_path)
with open(md5_path, "w") as f:
f.write(md5)
logging.debug("Writing SHA1 sum for %r", local_path)
with open(sha1_path, "w") as f:
f.write(sha1)
return local_path
else:
logging.warning("Checksum invalid for %r", local_path)
os.remove(local_path)
finally:
http_reply.close()
except urllib.error.HTTPError as err:
logging.error("Error on remote %r: %r", remote,
err.readall().decode())
except urllib.error.URLError as err:
if isinstance(err.reason, FileNotFoundError):
logging.debug("File not found: %r", err.reason.filename)
else:
logging.error("Error on remote %r: %r", remote, err)
# Artifact is nowhere to be found:
return None
AddRemote = base.deprecated(add_remote)
# long-running or scheduled process. Not receiving a notification indicates a process
# is still running which is undesirable, especially in the event of failure. If mail
# couldn't be sent, try to send a truncated message so at least some notification is
# sent.
err_msg = email.mime.text.MIMEText(
"{smtp_err!r} raised trying to send mail.\n"
"Logs at {localhost}:{log_dir}.\n".format(
smtp_err=smtp_err,
localhost=socket.gethostname(),
log_dir=FLAGS.log_dir,
))
err_msg['Subject'] = "SMTP Error on {}".format(subject[:80])
err_msg['From'] = sender
err_msg['To'] = ','.join(recipients)
server.sendmail(from_addr=sender,
to_addrs=recipients,
msg=err_msg.as_string())
except Exception:
raise smtp_err
finally:
server.quit()
SendMail = base.deprecated(send_mail)
# ------------------------------------------------------------------------------
if __name__ == '__main__':
raise Error('Not a standalone program!')
Command = command
Commit = commit
CreateBranch = create_branch
CurrentBranch = current_branch
DeleteBranch = delete_branch
Fetch = fetch
GetCommitDetails = get_commit_details
GetCommitHash = get_commit_hash
GetCommitMessage = get_commit_message
GithubCommitWebLink = github_commit_web_link
ListBranches = list_branches
ListFiles = list_files
ListHeadRefNames = list_head_ref_names
ListRefs = list_refs
ListRemoteAddresses = list_remote_addresses
ListRevisions = list_revisions
ListSubmoduleStatus = list_submodule_status
ListTags = list_tags
Merge = merge
Push = push
SetRemote = set_remote
Status = status
Tag = tag
UpdateCommitMessage = update_commit_message
UpdateCommitTag = update_commit_tag
UpdateSubmodule = update_submodule
FormatRef = base.deprecated(format_ref)
FormatRefs = base.deprecated(format_refs)
class Task(object, metaclass=abc.ABCMeta):
"""Base class for a task."""
FAILURE = TaskState.FAILURE
SUCCESS = TaskState.SUCCESS
@classmethod
def task_name(cls):
"""Returns: the name of this task."""
if cls.__module__ == '__main__':
return cls.__name__
else:
return '%s.%s' % (cls.__module__, cls.__name__)
def __init__(
self,
workflow,
task_id,
runs_after=frozenset(),
runs_before=frozenset(),
):
"""Initializes a new task with the specified ID and dependencies.
Args:
workflow: Workflow this task belongs to.
task_id: Task unique ID. Must be hashable and immutable.
Most often, a string, a tuple or named tuple.
runs_after: Tasks (or task IDs) this task must run after.
runs_before: Tasks (or task IDs) this task must run before.
"""
self._state = TaskState.INIT
self._task_id = task_id
self._workflow = workflow
self._workflow._add_task(self)
# Set of task IDs, frozen after call to Task._build():
self._runs_after = set()
self._runs_before = set()
# Initialize dependencies from existing workflow state:
for dep in self._workflow._deps:
if dep.before == self._task_id:
self._runs_before.add(dep.after)
if dep.after == self._task_id:
self._runs_after.add(dep.before)
# Initialize dependencies from constructor parameters:
for dep in runs_after:
self.must_run_after(dep)
for dep in runs_before:
self.must_run_before(dep)
# While workflow runs, lists task IDs this task is waiting for:
self._pending_deps = None
# datetime instances set when the task runs:
self._start_time = None
self._end_time = None
@property
def workflow(self):
"""Returns: the workflow this task belongs to."""
return self._workflow
@property
def task_id(self):
"""Returns: the unique ID for this task."""
return self._task_id
@property
def state(self):
"""Returns: the task state."""
return self._state
@property
def is_runnable(self):
"""Returns: whether this task is runnable."""
return (self._state == TaskState.PENDING) \
and (len(self._pending_deps) == 0)
@property
def pending_deps(self):
return frozenset(self._pending_deps)
_COMPLETION_STATES = frozenset({TaskState.SUCCESS, TaskState.FAILURE})
@property
def completed(self):
"""Returns: whether this task has completed."""
return (self._state in self._COMPLETION_STATES)
@property
def runs_after(self):
"""Returns: IDs of the tasks this task depends on, ie. runs before."""
return self._runs_after
@property
def runs_before(self):
"""Returns: IDs of the tasks that depend on, ie. run after this task."""
return self._runs_before
def must_run_after(self, task):
"""Declares a dependency from this task to a given task.
Args:
task: Task or task ID to add a dependency upon.
Returns:
This task.
"""
assert (self._state == TaskState.INIT)
task_id = get_task_id(task)
if task_id not in self._runs_after:
self._runs_after.add(task_id)
self._workflow._AddDep(Dependency(before=task_id, after=self._task_id))
return self
def must_run_before(self, task):
"""Declares a dependency from a given task to this task.
Args:
task: Task or task ID to add a dependency upon.
Returns:
This task.
"""
assert (self._state == TaskState.INIT)
task_id = get_task_id(task)
if task_id not in self._runs_before:
self._runs_before.add(task_id)
self._workflow._AddDep(Dependency(before=self._task_id, after=task_id))
return self
RunsAfter = base.deprecated(must_run_after)
RunsBefore = base.deprecated(must_run_before)
def _build(self):
"""Completes the definition of this task.
Called internally by Workflow.Build().
"""
assert (self._state == TaskState.INIT)
self._state = TaskState.PENDING
self._runs_after = frozenset(self._runs_after)
self._runs_before = frozenset(self._runs_before)
self._pending_deps = set(self._runs_after)
def __str__(self):
"""Returns: a debug representation of this task."""
return ('Task(id=%s, runs_after=%s, runs_before=%s)'
% (self.task_id, self._runs_after, self._runs_before))
def __repr__(self):
return str(self)
def run(self):
"""Subclasses must override this method with the task's logic.
Returns:
Task completion status (TaskState.SUCCESS or TaskState.FAILURE).
"""
logging.warning("Task.Run() is deprecated in %r, use Task.run()", self.__class__)
return self.Run()
#@abc.abstractmethod
def Run(self):
raise Error('AbstractMethod')
def _run(self):
"""Wraps Run() to update and validate the task's status.
Uncaught exceptions are fatal.
Returns:
Task completion status.
"""
try:
self._start_time = datetime.datetime.now()
try:
self._state = self.run()
finally:
self._end_time = datetime.datetime.now()
if not self.completed:
logging.error(
'%r returned invalid task completion code: %r',
type(self).run, self._state)
# A task status that is neither SUCCESS nor FAILURE is a programming
# error: program should not continue in such a scenario.
base.shutdown()
return self._state
except:
logging.error('Unhandled exception from Task.run().')
traceback.print_exc()
# It is arguable whether an uncaught exception should cause the program
# to stop. We current assume an uncaught exception is a programming error.
# This could be made configurable via a flag.
base.shutdown()
def _task_success(self, task):
"""Processes the success of a task.
Args:
task: Task whose success is reported.
"""
assert (task.task_id in self._runs_after), task.task_id
assert (task.task_id in self._pending_deps), task.task_id
self._pending_deps.remove(task.task_id)
def _task_failure(self, task):
"""Processes the failure of a task.
Args:
task: Task whose failure is reported.
"""
assert (task.task_id in self._runs_after), \
('%s depending on %s' % (self.task_id, task.task_id))
assert (task.task_id in self._pending_deps), \
('%s depending on %s' % (self.task_id, task.task_id))
self._state = TaskState.FAILURE
def _set_completion_state(self, state):
"""Forcibly sets the completion state of this task.
Args:
New state of the task.
"""
self._state = state
assert self.completed
@property
def start_time(self):
"""Returns: the start time of the task run. or None if not started yet."""
return self._start_time
@property
def end_time(self):
"""Returns: the end time of the task run, or None if not completed yet."""
return self._end_time
@property
def graphviz_label(self):
"""Returns: a label for the Graphiz node representing this task."""
return self.task_id
"{smtp_err!r} raised trying to send mail.\n"
"Logs at {localhost}:{log_dir}.\n".format(
smtp_err=smtp_err,
localhost=socket.gethostname(),
log_dir=FLAGS.log_dir,
)
)
err_msg['Subject'] = "SMTP Error on {}".format(subject[:80])
err_msg['From'] = sender
err_msg['To'] = ','.join(recipients)
server.sendmail(
from_addr=sender,
to_addrs=recipients,
msg=err_msg.as_string()
)
except Exception:
raise smtp_err
finally:
server.quit()
SendMail = base.deprecated(send_mail)
# ------------------------------------------------------------------------------
if __name__ == '__main__':
raise Error('Not a standalone program!')