def build_pkg_now(): if request.method == 'POST': pkgname = request.form['pkgname'] dev = request.form['dev'] if not pkgname or pkgname is None or pkgname == '': abort(500) pexists = status.all_packages() pexists = pexists.ismember(pkgname) if not pexists: try: package.Package(name=pkgname) if os.path.exists('/var/tmp/antergos-packages/' + pkgname): pexists = True except Exception: pass if pexists: is_logged_in = user.is_authenticated() p, a, rev_pending = get_build_info(1, 'completed', is_logged_in) # logger.info(rev_pending) pending = False for bnum in rev_pending.keys(): bld_obj = build_obj.get_build_object(bnum=bnum) if pkgname == bld_obj.pkgname: pending = True break if pending: flash('Unable to build %s because it is in "pending review" status.' % pkgname, category='error') else: args = (True, True) if 'antergos-iso' in pkgname: if not status.iso_building: status.iso_flag = True args = (True, True) if 'openbox' in pkgname: status.iso_minimal = True else: logger.info('RATE LIMIT ON ANTERGOS ISO IN EFFECT') return redirect(redirect_url()) q = status.queue() q.rpush(pkgname) queue.enqueue_call(builder.handle_hook, args=args, timeout=84600) tl_event( msg='<strong>%s</strong> added <strong>%s</strong> to the build queue.' % (dev, pkgname), tl_type='0') else: flash('Package not found. Has the PKGBUILD been pushed to github?', category='error') return redirect(redirect_url())
def __init__(self, request=None): self.can_process = False self.is_monitor = False self.is_cnchi = False try: self.request = request.method except AttributeError: self.request = False self.is_monitor = True if self.request is None or db is None or queue is None: logger.error("@@-webhook.py-@@ 40 | Cant process new webhook because request or db is None.") elif self.request or self.is_monitor is True: self.can_process = True self.request = request self.is_manual = False self.is_numix = False self.is_github = False self.is_gitlab = False self.changes = [] self.phab_payload = False self.the_queue = status.queue() self.repo = "antergos-packages" self.payload = None self.full_name = None self.pusher = None self.commits = None self.result = None self.building = status.now_building self.result = None self.allpkgs = status.all_packages() self.is_authorized = self.is_from_authorized_sender() if self.is_authorized: # Process Webhook if self.is_manual: self.process_manual() if self.is_cnchi: self.process_cnchi() if self.is_github: self.process_github() if len(self.changes) > 0: self.process_changes() else: if self.result is None: self.result = "Nothing to see here, move along ..."
def scheduled(): try: queued = status.queue() except Exception: queued = None building = status.now_building the_queue = [] logger.debug(queued) if queued: for pak in queued: try: pkg_obj = package.Package(name=pak) name = pkg_obj.name version = pkg_obj.version_str all_info = (name, version) the_queue.append(all_info) except ValueError as err: logger.error(err) return render_template("scheduled.html", building=building, queue=the_queue, user=user)
def handle_worker_exception(job, exc_type, exc_value, traceback): # TODO: This needs some thought on how to recover instead of bailing on entire build queue doc = docker.Client(base_url='unix://var/run/docker.sock', timeout=10) if job['origin'] == 'build_queue': container = db.get('container') elif job['origin'] == 'repo_queue': container = db.get('repo_container') else: container = '' queue = status.queue() now_building = status.now_building try: doc.kill(container) doc.remove_container(container) except Exception: logger.error('Unable to kill container') if job['origin'] == 'build_queue': db.set('%s:result' % now_building['key'], 'failed') db.rpush('failed', now_building['build_id']) if not queue or len(queue) == 0 or queue == []: repo = os.path.join("/tmp", "staging") cache = os.path.join("/tmp", "pkg_cache") remove(repo) remove(cache) remove('/opt/antergos-packages') # db.set('idle', "True") status.idle = True # db.set('building', 'Idle') status.current_status = 'Idle' # db.hset('now_building', 'pkg', '') # db.set('container', '') # db.set('building_num', '') # db.set('building_start', '') logger.error('Caught Build Exception: %s', traceback) return True
def handle_hook(first=False, last=False): status.idle = False pull_from = 'antergos' packages = status.queue() if os.path.exists(REPO_DIR): remove(REPO_DIR) try: subprocess.check_call( ['git', 'clone', 'http://github.com/antergos/antergos-packages.git'], cwd='/opt') subprocess.check_call(['chmod', '-R', 'a+rw', REPO_DIR], cwd='/opt') except subprocess.CalledProcessError as err: logger.error(err) if status.iso_flag: status.iso_flag = False status.current_status = 'Building docker image.' status.iso_building = True image = docker_utils.maybe_build_mkarchiso() db.lrem('queue', 0, 'antergos-iso') db.lrem('queue', 0, 'antergos-iso.openbox') if image: archs = ['x86_64', 'i686'] if db.get('isoMinimal') == 'True': iso_name = 'antergos-iso-minimal-' else: iso_name = 'antergos-iso-' for arch in archs: db.rpush('queue', iso_name + arch) version = datetime.datetime.now().strftime('%Y.%m.%d') pkgobj = package.get_pkg_object(iso_name + arch) pkgobj.save_to_db('version', version) build_iso() db.set('isoBuilding', 'False') db.set('isoMinimal', 'False') db.set('idle', "True") return True elif first and not status.iso_flag: status.current_status = 'Building docker image.' image = docker_utils.maybe_build_base_devel() if not image: return False logger.info('Checking database for packages.') status.current_status = 'Checking database for queued packages' all_deps = process_package_queue(packages) logger.info('All queued packages are in the database, checking deps to determine build order.') status.current_status = 'Determining build order by sorting package depends' if len(all_deps) > 1: topsort = check_deps(all_deps) check = [] packages.delete() for p in topsort: # TODO: What if there is already a group of packages in queue prior to the current group? packages.append(p) logger.info('Check deps complete. Starting build_pkgs') logger.debug((packages, status.iso_flag)) status.current_status = 'Check deps complete. Starting build container.' if not status.iso_flag and len(packages) > 0: pack = status.queue().lpop() if pack and pack is not None and pack != '': pkgobj = package.get_pkg_object(name=pack) else: return False rqjob = get_current_job(db) rqjob.meta['is_first'] = first rqjob.meta['is_last'] = last rqjob.meta['package'] = pkgobj.name rqjob.save() status.now_building = pkgobj.name built = build_pkgs(last, pkgobj) # TODO: Move this into its own method if built: completed = status.completed() failed = status.failed() blds = pkgobj.builds() total = len(blds) if total > 0: success = len([x for x in pkgobj.blds if x in completed]) failure = len([x for x in pkgobj.blds if x in failed]) if success > 0: success = 100 * success / total else: success = 0 if failure > 0: failure = 100 * failure / total else: failure = 0 pkgobj.success_rate = success pkgobj.failure_rate = failure if last: remove('/opt/antergos-packages') status.idle = True status.building = 'Idle' status.now_building = 'Idle' status.container = '' status.building_num = '' status.building_start = '' logger.info('All builds completed.')