async def async_start(self):
"""Async start the task."""
self.reclaim_fut = self.event_loop.create_task(self.reclaim_task())
self.task_fut = self.event_loop.create_task(self.run_task())
try:
await self.task_fut
except Download404:
self.status = STATUSES["resource-unavailable"]
self.task_log(traceback.format_exc(), level=logging.CRITICAL)
except (DownloadError, RetryError):
self.status = STATUSES["intermittent-task"]
self.task_log(traceback.format_exc(), level=logging.CRITICAL)
except TaskError:
self.status = STATUSES["malformed-payload"]
self.task_log(traceback.format_exc(), level=logging.CRITICAL)
except asyncio.CancelledError:
# We already dealt with self.status in reclaim_task
self.task_log(traceback.format_exc(), level=logging.CRITICAL)
log.info("Stopping task %s %s with status %s", self.task_id,
self.run_id, self.status)
self.reclaim_fut.cancel()
await self.upload_task()
await self.complete_task()
rm(self.task_dir)
self.complete = True
async def run_task(self):
"""Run the task, creating a task-specific log file."""
self.status = 0
username = self.config["notarization_username"]
password = self.config["notarization_password"]
await self.download_uuids()
self.pending_uuids = list(self.uuids)
while True:
self.task_log("pending uuids: %s", self.pending_uuids)
for uuid in sorted(self.pending_uuids):
self.task_log("Polling %s", uuid)
base_cmd = list(self.config["xcrun_cmd"]) + [
"altool", "--notarization-info", uuid, "-u", username,
"--password"
]
log_cmd = base_cmd + ["********"]
rm(self.poll_log_path)
status = await retry_async(
run_command,
args=[base_cmd + [password]],
kwargs={
"log_path": self.poll_log_path,
"log_cmd": log_cmd,
"exception": RetryError
},
retry_exceptions=(RetryError, ),
attempts=10,
)
with open(self.poll_log_path, "r") as fh:
contents = fh.read()
self.task_log("Polling response (status %d)",
status,
worker_log=False)
for line in contents.splitlines():
self.task_log(" %s", line, worker_log=False)
if status == STATUSES["success"]:
m = NOTARIZATION_POLL_REGEX.search(contents)
if m is not None:
if m["status"] == "invalid":
self.status = STATUSES["failure"]
self.task_log("Apple believes UUID %s is invalid!",
uuid,
level=logging.CRITICAL)
raise TaskError(
"Apple believes UUID %s is invalid!" % uuid)
# There are only two possible matches with the regex
# Adding `pragma: no branch` to be explicit in our
# checks, but still avoid testing an unreachable code
# branch
if m["status"] == "success": # pragma: no branch
self.task_log("UUID %s is successful", uuid)
self.pending_uuids.remove(uuid)
if len(self.pending_uuids) == 0:
self.task_log("All UUIDs are successfully notarized: %s",
self.uuids)
break
else:
await asyncio.sleep(self.config["poll_sleep_time"])
async def extract_all_apps(config, all_paths):
"""Extract all the apps into their own directories.
Args:
work_dir (str): the ``work_dir`` path
all_paths (list): a list of ``App`` objects with their ``orig_path`` set
Raises:
IScriptError: on failure
"""
log.info("Extracting all apps")
futures = []
work_dir = config["work_dir"]
unpack_dmg = os.path.join(os.path.dirname(__file__), "data",
"unpack-diskimage")
for counter, app in enumerate(all_paths):
app.check_required_attrs(["orig_path"])
app.parent_dir = os.path.join(work_dir, str(counter))
rm(app.parent_dir)
makedirs(app.parent_dir)
if app.orig_path.endswith((".tar.bz2", ".tar.gz", ".tgz")):
futures.append(
asyncio.ensure_future(
run_command(
["tar", "xf", app.orig_path],
cwd=app.parent_dir,
exception=IScriptError,
)))
elif app.orig_path.endswith(".dmg"):
unpack_mountpoint = os.path.join(
"/tmp", f"{config.get('dmg_prefix', 'dmg')}-{counter}-unpack")
futures.append(
asyncio.ensure_future(
run_command(
[
unpack_dmg, app.orig_path, unpack_mountpoint,
app.parent_dir
],
cwd=app.parent_dir,
exception=IScriptError,
log_level=logging.DEBUG,
)))
else:
raise IScriptError(f"unknown file type {app.orig_path}")
await raise_future_exceptions(futures)
if app.orig_path.endswith(".dmg"):
# nuke the softlink to /Applications
for counter, app in enumerate(all_paths):
rm(os.path.join(app.parent_dir, " "))
def main(event_loop=None):
"""Notarization poller entry point: get everything set up, then enter the main loop.
Args:
event_loop (asyncio.BaseEventLoop, optional): the event loop to use.
If None, use ``asyncio.get_event_loop()``. Defaults to None.
"""
event_loop = event_loop or asyncio.get_event_loop()
config = get_config_from_cmdln(sys.argv[1:])
update_logging_config(config)
log.info("Notarization poller starting up at {} UTC".format(
arrow.utcnow().format()))
log.info("Worker FQDN: {}".format(socket.getfqdn()))
rm(config["work_dir"])
makedirs(config["work_dir"])
running_tasks = RunTasks(config)
async def _handle_sigterm():
log.info("SIGTERM received; shutting down")
await running_tasks.cancel()
def _handle_sigusr1():
"""Stop accepting new tasks."""
log.info("SIGUSR1 received; no more tasks will be taken")
running_tasks.is_stopped = True
event_loop.add_signal_handler(
signal.SIGTERM, lambda: asyncio.ensure_future(_handle_sigterm()))
event_loop.add_signal_handler(signal.SIGUSR1, _handle_sigusr1)
try:
event_loop.run_until_complete(running_tasks.invoke())
except Exception:
log.critical("Fatal exception", exc_info=1)
raise
finally:
log.info("Notarization poller stopped at {} UTC".format(
arrow.utcnow().format()))
log.info("Worker FQDN: {}".format(socket.getfqdn()))
def remove_extra_files(top_dir, file_list):
"""Find any extra files in `top_dir`, given an expected `file_list`.
Args:
top_dir (str): the dir to walk
file_list (list): the list of expected files
Returns:
list: the list of extra files
"""
all_files = [
os.path.realpath(f)
for f in glob.glob(os.path.join(top_dir, "**", "*"), recursive=True)
]
good_files = [os.path.realpath(f) for f in file_list]
extra_files = list(set(all_files) - set(good_files))
for f in extra_files:
if os.path.isfile(f):
log.warning("Extra file to clean up: {}".format(f))
rm(f)
return extra_files
def start(self):
"""Start the task."""
rm(self.task_dir)
makedirs(self.task_dir)
self._reclaim_task = {}
self.main_fut = self.event_loop.create_task(self.async_start())
def test_rm_dir(tmpdir):
assert os.path.exists(tmpdir)
utils.rm(tmpdir)
assert not os.path.exists(tmpdir)
def test_rm_file():
_, tmp = tempfile.mkstemp()
assert os.path.exists(tmp)
utils.rm(tmp)
assert not os.path.exists(tmp)
def test_rm_empty():
utils.rm(None)
async def lockfile(paths, name=None, attempts=10, sleep=30):
"""Acquire a lockfile from among ``paths`` and yield the path.
If we want inter-process semaphores, we can use lock files rather than
``asyncio.Semaphore``.
The reason we're using ``open(path, "x")`` rather than purely relying
on ``fcntl.lockf`` is that fcntl file locking doesn't lock the file
inside the current process, only for outside processes. If we don't
keep track of which lockfiles we've used in our async script, we can
easily blow away our own lock if we, say, used ``open(path, "w")`` or
``open(path, "a") and then closed any of the open filehandles.
(See http://0pointer.de/blog/projects/locking.html for more details.)
Args:
paths (list): a list of path strings to use as lockfiles.
name (str, optional): a descriptive name for the process that needs
the lockfile, for logging purposes. Defaults to ``None``.
attempts (int, optional): the number of attempts to get a lockfile.
This means we attempt to get a lockfile from every path in ``paths``, ``attempts`` times. Defaults to 20.
sleep (int, optional): the number of seconds to sleep between attempts.
We sleep after attempting every path in ``paths``. Defaults to 30.
Yields:
str: the lockfile path acquired.
Raises:
LockfileError: if we've exhausted our attempts.
"""
if name is not None:
acquired_msg = "Lockfile acquired for {} at %s".format(name)
wait_msg = "Couldn't get lock for {}; sleeping %s".format(name)
failed_msg = "Can't get lock for {} from paths %s after %s attempts".format(
name
)
else:
acquired_msg = "Lockfile acquired at %s"
wait_msg = "Couldn't get lock; sleeping %s"
failed_msg = "Can't get lock from paths %s after %s attempts"
for attempt in range(0, attempts):
for path in [item for item in random.sample(paths, len(paths))]:
try:
# Ensure the file doesn't exist, so we don't blow away
# our own lockfiles.
with open(path, "x") as fh:
# Acquire an fcntl lock, in case other processes
# use something other than ``lockfile`` to acquire
# locks
try:
fcntl.lockf(fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
log.debug(acquired_msg, path)
yield path
# We'll clean up `path` in the `finally` block below
return
finally:
rm(path)
except (FileExistsError, OSError):
continue
log.debug(wait_msg, sleep)
if attempt < attempts - 1:
await asyncio.sleep(sleep)
raise LockfileError(failed_msg, paths, attempts)
