async def _drop_publish(self, args, _): base_mirror_name = args[0] base_mirror_version = args[1] projectname = args[2] projectversion = args[3] dist = args[4] logger.info("aptly worker: got drop publish task: %s/%s %s %s %s", base_mirror_name, base_mirror_version, projectname, projectversion, dist) apt = get_aptly_connection() await apt.publish_drop(base_mirror_name, base_mirror_version, projectname, projectversion, dist)
async def update_mirror(task_queue, build_id, base_mirror, base_mirror_version, mirror, version, components): """ Creates an update task in the asyncio event loop. Args: mirror (str): The mirror's name (project name). version (str): The mirror's version. base_distribution (str): The mirror's base dist (mirror_distribution). components (list): The mirror's components """ apt = get_aptly_connection() task_id = await apt.mirror_update(base_mirror, base_mirror_version, mirror, version) logger.info("start update progress: aptly task %s", task_id) loop = asyncio.get_event_loop() loop.create_task( finalize_mirror(task_queue, build_id, base_mirror, base_mirror_version, mirror, version, components, task_id))
async def _create_mirror(self, args, session): ( mirror, url, mirror_distribution, components, keys, keyserver, is_basemirror, architectures, version, key_url, basemirror_id, download_sources, download_installer, ) = args build = Build( version=version, git_ref=None, ci_branch=None, is_ci=False, versiontimestamp=None, sourcename=mirror, buildstate="new", buildtype="mirror", buildconfiguration=None, sourcerepository=None, maintainer=None, ) build.log_state("created") session.add(build) await build_added(build) session.commit() write_log_title(build.id, "Create Mirror") mirror_project = ( session.query(Project) # pylint: disable=no-member .filter(Project.name == mirror, Project.is_mirror.is_(True)).first()) if not mirror_project: mirror_project = Project(name=mirror, is_mirror=True, is_basemirror=is_basemirror) session.add(mirror_project) # pylint: disable=no-member project_version = ( session.query(ProjectVersion).join(Project).filter( # pylint: disable=no-member Project.name == mirror, Project.is_mirror.is_(True)).filter( ProjectVersion.name == version).first()) if project_version: write_log( build.id, "W: mirror with name '%s' and version '%s' already exists\n" % (mirror, version)) logger.error( "mirror with name '%s' and version '%s' already exists", mirror, version) await build.set_successful() session.commit() return True base_mirror = None base_mirror_version = None db_buildvariant = None if not is_basemirror: db_basemirror = ( session.query(ProjectVersion) # pylint: disable=no-member .filter(ProjectVersion.id == basemirror_id).first()) if not db_basemirror: write_log( build.id, "E: could not find a basemirror with id '%d'\n" % basemirror_id) logger.error("could not find a basemirror with id '%d'", basemirror_id) await build.set_failed() session.commit() return False base_mirror = db_basemirror.project.name base_mirror_version = db_basemirror.name db_buildvariant = ( session.query(BuildVariant) # pylint: disable=no-member .filter(BuildVariant.base_mirror_id == basemirror_id).first()) if not db_buildvariant: write_log( build.id, "E: could not find a buildvariant for basemirror with id '%d'\n" % db_basemirror.id) logger.error( "could not find a buildvariant for basemirror with id '%d'", db_basemirror.id) await build.set_failed() session.commit() return False mirror_project_version = ProjectVersion( name=version, project=mirror_project, mirror_url=url, mirror_distribution=mirror_distribution, mirror_components=",".join(components), mirror_architectures="{" + ",".join(architectures) + "}", mirror_with_sources=download_sources, mirror_with_installer=download_installer, ) if db_buildvariant: mirror_project_version.buildvariants.append(db_buildvariant) session.add(mirror_project_version) session.commit() build.projectversion_id = mirror_project_version.id session.commit() write_log(build.id, "I: adding GPG keys\n") apt = get_aptly_connection() if key_url: try: await apt.gpg_add_key(key_url=key_url) except AptlyError as exc: write_log(build.id, "E: Error adding keys from '%s'\n" % key_url) logger.error("key error: %s", exc) await build.set_failed() return False elif keyserver and keys: try: await apt.gpg_add_key(key_server=keyserver, keys=keys) except AptlyError as exc: write_log(build.id, "E: Error adding keys %s\n" % str(keys)) logger.error("key error: %s", exc) await build.set_failed() return False write_log(build.id, "I: creating mirror\n") try: await apt.mirror_create( mirror, version, base_mirror, base_mirror_version, url, mirror_distribution, components, architectures, download_sources=download_sources, download_udebs=download_installer, download_installer=download_installer, ) except NotFoundError as exc: write_log(build.id, "E: aptly seems to be not available: %s\n" % str(exc)) logger.error("aptly seems to be not available: %s", str(exc)) await build.set_failed() return False except AptlyError as exc: write_log( build.id, "E: failed to create mirror %s on aptly: %s\n" % (mirror, str(exc))) logger.error("failed to create mirror %s on aptly: %s", mirror, str(exc)) await build.set_failed() return False args = { "update_mirror": [ build.id, mirror_project_version.id, base_mirror, base_mirror_version, mirror, version, components ] } await self.aptly_queue.put(args)
async def startup_mirror(task_queue): """ Starts a finalize_mirror task in the asyncio event loop for all mirrors which have the state 'updating' """ loop = asyncio.get_event_loop() apt = get_aptly_connection() with Session() as session: # get mirrors in updating state query = session.query(ProjectVersion) # pylint: disable=no-member query = query.join(Project, Project.id == ProjectVersion.project_id) query = query.filter(Project.is_mirror.is_(True)) query = query.filter( or_(ProjectVersion.mirror_state == "updating", ProjectVersion.mirror_state == "publishing")) if not query.count(): return mirrors = query.all() tasks = await apt.get_tasks() for mirror in mirrors: # FIXME: only one buildvariant supported base_mirror = "" base_mirror_version = "" taskname = "Update mirror" buildstate = "building" if mirror.mirror_state == "publishing": taskname = "Publish snapshot:" buildstate = "publishing" if not mirror.project.is_basemirror: base_mirror = mirror.buildvariants[0].base_mirror.project.name base_mirror_version = mirror.buildvariants[0].base_mirror.name task_name = "{} {}-{}-{}-{}".format(taskname, base_mirror, base_mirror_version, mirror.project.name, mirror.name) else: task_name = "{} {}-{}".format(taskname, mirror.project.name, mirror.name) m_tasks = None if tasks: m_tasks = [task for task in tasks if task["Name"] == task_name] if not m_tasks: # No task on aptly found mirror.mirror_state = "error" session.commit() # pylint: disable=no-member continue m_task = max(m_tasks, key=operator.itemgetter("ID")) build = (session.query(Build).filter( Build.buildtype == "mirror", Build.buildstate == buildstate, Build.projectversion_id == mirror.id, ).first()) if not build: # No task on aptly found mirror.mirror_state = "error" session.commit() # pylint: disable=no-member continue components = mirror.mirror_components.split(",") loop.create_task( finalize_mirror( task_queue, build.id, base_mirror, base_mirror_version, mirror.project.name, mirror.name, components, m_task.get("ID"), ))
async def finalize_mirror(task_queue, build_id, base_mirror, base_mirror_version, mirror, version, components, task_id): """ """ try: mirrorname = "{}-{}".format(mirror, version) logger.info("finalizing mirror %s task %d, build_%d", mirrorname, task_id, build_id) with Session() as session: # FIXME: get entry from build.projectversion_id query = session.query(ProjectVersion) # pylint: disable=no-member query = query.join(Project, Project.id == ProjectVersion.project_id) query = query.filter(Project.is_mirror.is_(True)) query = query.filter(ProjectVersion.name == version) entry = query.filter(Project.name == mirror).first() if not entry: logger.error("finalize mirror: mirror '%s' not found", mirrorname) return build = session.query(Build).filter(Build.id == build_id).first() if not build: logger.error("aptly worker: mirror build with id %d not found", build_id) return apt = get_aptly_connection() if entry.mirror_state == "updating": while True: try: upd_progress = await apt.mirror_get_progress(task_id) except Exception as exc: logger.error( "update mirror %s get progress exception: %s", mirrorname, exc) entry.mirror_state = "error" await build.set_failed() session.commit() # pylint: disable=no-member return # 0: init, 1: running, 2: success, 3: failed if upd_progress["State"] == 2: break if upd_progress["State"] == 3: logger.error("update mirror %s progress error", mirrorname) entry.mirror_state = "error" await build.set_failed() session.commit() # pylint: disable=no-member return logger.info( "mirrored %d/%d files (%.02f%%), %.02f/%.02fGB (%.02f%%)", upd_progress["TotalNumberOfPackages"] - upd_progress["RemainingNumberOfPackages"], upd_progress["TotalNumberOfPackages"], upd_progress["PercentPackages"], (upd_progress["TotalDownloadSize"] - upd_progress["RemainingDownloadSize"]) / 1024.0 / 1024.0 / 1024.0, upd_progress["TotalDownloadSize"] / 1024.0 / 1024.0 / 1024.0, upd_progress["PercentSize"], ) await asyncio.sleep(2) await apt.delete_task(task_id) write_log(build.id, "I: creating snapshot\n") await build.set_publishing() session.commit() # snapshot after initial download logger.info("creating snapshot for: %s", mirrorname) try: task_id = await apt.mirror_snapshot( base_mirror, base_mirror_version, mirror, version) except AptlyError as exc: logger.error("error creating mirror %s snapshot: %s", mirrorname, exc) entry.mirror_state = "error" await build.set_publish_failed() session.commit() # pylint: disable=no-member return while True: try: task_state = await apt.get_task_state(task_id) except Exception: logger.exception("error getting mirror %s state", mirrorname) entry.mirror_state = "error" await build.set_publish_failed() session.commit() # pylint: disable=no-member return # States: # 0: init, 1: running, 2: success, 3: failed if task_state["State"] == 2: break if task_state["State"] == 3: logger.error("creating mirror %s snapshot failed", mirrorname) entry.mirror_state = "error" await build.set_publish_failed() session.commit() # pylint: disable=no-member return # FIMXE: why sleep ? await asyncio.sleep(2) await apt.delete_task(task_id) entry.mirror_state = "publishing" session.commit() # pylint: disable=no-member # publish new snapshot write_log(build.id, "I: publishing mirror\n") logger.info("publishing snapshot: %s", mirrorname) try: task_id = await apt.mirror_publish( base_mirror, base_mirror_version, mirror, version, entry.mirror_distribution, components) except Exception as exc: logger.error("error publishing mirror %s snapshot: %s", mirrorname, str(exc)) entry.mirror_state = "error" await build.set_publish_failed() session.commit() # pylint: disable=no-member await apt.mirror_snapshot_delete(base_mirror, base_mirror_version, mirror, version) return if entry.mirror_state == "publishing": while True: try: upd_progress = await apt.mirror_get_progress(task_id) except Exception as exc: logger.error("error publishing mirror %s: %s", mirrorname, str(exc)) entry.mirror_state = "error" await build.set_publish_failed() session.commit() # pylint: disable=no-member await apt.mirror_snapshot_delete( base_mirror, base_mirror_version, mirror, version) return # States: # 0: init, 1: running, 2: success, 3: failed if upd_progress["State"] == 2: break if upd_progress["State"] == 3: logger.error("error publishing mirror %s snapshot", mirrorname) entry.mirror_state = "error" await build.set_publish_failed() session.commit() # pylint: disable=no-member await apt.mirror_snapshot_delete( base_mirror, base_mirror_version, mirror, version) return logger.info( "published %d/%d packages (%.02f%%)", upd_progress["TotalNumberOfPackages"] - upd_progress["RemainingNumberOfPackages"], upd_progress["TotalNumberOfPackages"], upd_progress["PercentPackages"], ) await asyncio.sleep(2) if entry.project.is_basemirror: for arch_name in entry.mirror_architectures[1:-1].split(","): arch = (session.query(Architecture).filter( Architecture.name == arch_name).first()) # pylint: disable=no-member if not arch: await build.set_publish_failed() logger.error( "finalize mirror: architecture '%s' not found", arch_name) return buildvariant = BuildVariant(base_mirror=entry, architecture=arch) session.add(buildvariant) # pylint: disable=no-member write_log(build.id, "I: starting chroot environments build\n") chroot_build = Build( version=version, git_ref=None, ci_branch=None, is_ci=None, versiontimestamp=None, sourcename=mirror, buildstate="new", buildtype="chroot", projectversion_id=build.projectversion_id, buildconfiguration=None, parent_id=build.id, sourcerepository=None, maintainer=None, ) session.add(chroot_build) session.commit() chroot_build.log_state("created") await build_added(chroot_build) await chroot_build.set_needs_build() session.commit() await chroot_build.set_scheduled() session.commit() chroot = Chroot(buildvariant=buildvariant, ready=False) session.add(chroot) session.commit() loop = asyncio.get_event_loop() loop.create_task( create_schroot( task_queue, chroot.id, chroot_build.id, buildvariant.base_mirror.mirror_distribution, buildvariant.base_mirror.project.name, buildvariant.base_mirror.name, buildvariant.architecture.name, )) entry.is_locked = True entry.mirror_state = "ready" session.commit() # pylint: disable=no-member await build.set_successful() session.commit() logger.info("mirror %s succesfully created", mirrorname) write_log_title(build.id, "Done") except Exception as exc: logger.exception(exc)
async def delete_mirror(request): """ Delete a single mirror on aptly and from database. --- description: Delete a single mirror on aptly and from database. tags: - Mirrors consumes: - application/x-www-form-urlencoded parameters: - name: id in: path required: true type: integer description: id of the mirror produces: - text/json responses: "200": description: successful "400": description: removal failed from aptly. "404": description: mirror not found on aptly. "500": description: internal server error. "503": description: removal failed from database. """ apt = get_aptly_connection() mirror_id = request.match_info["id"] query = request.cirrina.db_session.query(ProjectVersion) # pylint: disable=no-member query = query.join(Project, Project.id == ProjectVersion.project_id) query = query.filter(Project.is_mirror.is_(True)) entry = query.filter(ProjectVersion.id == mirror_id).first() if not entry: logger.warning("error deleting mirror '%s': mirror not found", mirror_id) return error(404, "Error deleting mirror '%d': mirror not found", mirror_id) # FIXME: check state, do not delete ready/updating/... mirrorname = "{}-{}".format(entry.project.name, entry.name) # check relations if entry.sourcerepositories: logger.warning( "error deleting mirror '%s': referenced by one or more source repositories", mirrorname) return error( 412, "Error deleting mirror {}: still referenced by one or more source repositories", mirrorname) if entry.buildconfiguration: logger.warning( "error deleting mirror '%s': referenced by one or more build configurations", mirrorname) return error( 412, "Error deleting mirror {}: still referenced by one or more build configurations", mirrorname) if entry.dependents: logger.warning( "error deleting mirror '%s': referenced by one or project versions", mirrorname) return error( 412, "Error deleting mirror {}: still referenced by one or more project versions", mirrorname) base_mirror = "" base_mirror_version = "" if not entry.project.is_basemirror: basemirror = entry.buildvariants[0].base_mirror base_mirror = basemirror.project.name base_mirror_version = basemirror.name # FIXME: cleanup chroot table, schroots, debootstrap, try: # FIXME: use altpy queue ! await apt.mirror_delete(base_mirror, base_mirror_version, entry.project.name, entry.name, entry.mirror_distribution) except Exception as exc: # mirror did not exist # FIXME: handle mirror has snapshots and cannot be deleted? logger.exception(exc) pass project = entry.project bvs = request.cirrina.db_session.query(BuildVariant).filter( BuildVariant.base_mirror_id == entry.id).all() # pylint: disable=no-member for bvariant in bvs: # FIXME: delete buildconfigurations request.cirrina.db_session.delete(bvariant) # pylint: disable=no-member builds = request.cirrina.db_session.query(Build).filter( Build.projectversion_id == entry.id).all() for build in builds: # FIXME: delete buildconfigurations # FIXME: remove buildout dir request.cirrina.db_session.delete(build) # pylint: disable=no-member request.cirrina.db_session.delete(entry) # pylint: disable=no-member request.cirrina.db_session.commit() # pylint: disable=no-member if not project.projectversions: request.cirrina.db_session.delete(project) # pylint: disable=no-member request.cirrina.db_session.commit() # pylint: disable=no-member return web.Response( status=200, text="Successfully deleted mirror: {}".format(mirrorname))