def maybe_check_for_new_items(): """ :return: """ return db.exists("FEED_CHECKED")
def cache_buster(): if db.exists('antbs:misc:cache_buster:flag'): db.delete('antbs:misc:cache_buster:flag') return True elif user.is_authenticated(): return True return False
def get_repo_info(repo=None, logged_in=False): if repo is None: abort(500) rinfo_key = 'cache:repo_info:%s' % repo repo_info_cache = db.exists(rinfo_key) pkg_list = {} p, a, rev_pending = get_build_info(1, repo, logged_in) # logger.info('@@-antbs.py-@@ 293 | GET_REPO_INFO - FIRED') if not repo_info_cache: logger.info('@@-antbs.py-@@ 295 | GET_REPO_INFO - CACHE CHECK FAILED. WE ARE NOT USING CACHED INFO') all_packages = glob.glob('/srv/antergos.info/repo/%s/x86_64/***.pkg.tar.xz' % repo) if all_packages is not None: for item in all_packages: # logger.info(item) item = item.split('/')[-1] item = re.search('^([a-z]|[0-9]|-|_)+(?=-\d|r|v)', item) item = item.group(0) or '' if not item or item == '': continue logger.info(item) pkg = package.Package(item) builds = pkg.builds try: bnum = builds[0] except Exception: bnum = '' review_stat = db.get('build_log:%s:review_stat' % bnum) or 'n/a' review_stat = db.get('review_stat:%s:string' % review_stat) or 'n/a' review_dev = db.get('build_log:%s:review_dev' % bnum) or 'n/a' review_date = db.get('build_log:%s:review_date' % bnum) or 'n/a' all_info = dict(bnum=bnum, name=pkg.name, version=pkg.version, review_dev=review_dev, review_stat=review_stat, review_date=review_date, pkgid=pkg.pkgid) db.hmset('%s:%s' % (rinfo_key, pkg.pkgid), all_info) db.expire('%s:%s' % (rinfo_key, pkg.pkgid), 901) db.rpush(rinfo_key, pkg.pkgid) db.expire(rinfo_key, 900) pkg_list[pkg.pkgid] = all_info else: logger.info('@@-antbs.py-@@ 318 | GET_REPO_INFO - CACHE CHECK PASSED. WE ARE USING CACHED INFO') rindex = db.lrange(rinfo_key, 0, -1) for i in rindex: h = db.hgetall('%s:%s' % (rinfo_key, i)) pkg_list[i] = h # logger.info('@@-antbs.py-@@ 320 | GET_REPO_INFO - pkg_list hash is %s' % str(pkg_list)) return pkg_list, rev_pending
def maybe_update_pkgbuild_repo(): """ """ if not db.exists('PKGBUILD_REPO_UPDATED'): if db.setnx('PKGBUILD_REPO_LOCK', True): db.expire('PKGBUILD_REPO_LOCK', 300) try: if os.path.exists('/var/tmp/antergos-packages'): shutil.rmtree('/var/tmp/antergos-packages') subprocess.check_call(['git', 'clone', 'http://github.com/antergos/antergos-packages'], cwd='/var/tmp') db.setex('PKGBUILD_REPO_UPDATED', 350, True) except subprocess.CalledProcessError as err: logger.error(err) db.delete('PKGBUILD_REPO_UPDATED') db.delete('PKGBUILD_REPO_LOCK') return else: while not db.exists('PKGBUILD_REPO_UPDATED') and db.exists('PKGBUILD_REPO_LOCK'): time.sleep(2) return
def process_package_queue(): """ :param the_queue: :return: :raise ValueError: """ hook_queue = status.hook_queue logger.info(hook_queue) if hook_queue is None: raise ValueError('the_queue cannot be None') all_deps = [] if not db.exists('BUILD_REPO_UPDATED'): if db.setnx('BUILD_REPO_LOCK', True): db.expire('BUILD_REPO_LOCK', 300) try: subprocess.check_call( ['git', 'clone', 'http://github.com/antergos/antergos-packages.git'], cwd='/opt') subprocess.check_call(['chmod', '-R', 'a+rw', REPO_DIR], cwd='/opt') except subprocess.CalledProcessError: try: subprocess.check_call(['git', 'reset', '--soft', 'origin/master'], cwd='/opt/antergos-packages') subprocess.check_call(['git', 'pull'], cwd='/opt/antergos-packages') db.setex('BUILD_REPO_UPDATED', 350, True) except subprocess.CalledProcessError as err: logger.error(err) db.delete('BUILD_REPO_LOCK') else: while not db.exists('BUILD_REPO_UPDATED') and db.exists('BUILD_REPO_LOCK'): time.sleep(2) for pkg in hook_queue: logger.info(pkg) if pkg == '': continue pkg_obj = package.get_pkg_object(name=pkg) version = pkg_obj.get_version() if not version: status.hook_queue.remove(pkg_obj.name) if 'cnchi-dev' != pkg: logger.error('pkgbuild path is not valid for %s', pkg_obj.name) else: continue logger.info('Updating pkgver in database for %s to %s' % (pkg_obj.name, version)) status.current_status = 'Updating pkgver in database for %s to %s' % (pkg_obj.name, version) depends = pkg_obj.get_deps() if not pkg_obj.build_path or pkg_obj.build_path == '': paths = [os.path.join('/opt/antergos-packages/', pkg), os.path.join('/opt/antergos-packages/deepin_desktop', pkg), os.path.join('/opt/antergos-packages/cinnamon', pkg)] for p in paths: if os.path.exists(p): pkg_obj.build_path = p break if 'cnchi' in pkg: logger.info('cnchi package detected.') src = os.path.join('/var/tmp/antergos-packages/', pkg, 'cnchi') dest = os.path.join('/opt/antergos-packages/', pkg) remove(os.path.join(dest, 'cnchi')) shutil.move(src, dest) status.current_status = 'Fetching latest translations for %s from Transifex.' % pkg logger.info(status.current_status) cnchi_dir = '/opt/antergos-packages/%s' % pkg fetch_and_compile_translations(translations_for=["cnchi"], pkg_obj=pkg_obj) remove(os.path.join(cnchi_dir, 'cnchi/.git')) subprocess.check_output(['tar', '-cf', 'cnchi.tar', 'cnchi'], cwd='/opt/antergos-packages/%s' % pkg) elif 'numix-icon-theme-square' in pkg: src = os.path.join('/var/tmp/antergos-packages/', pkg, pkg) dest = os.path.join('/opt/antergos-packages/', pkg) shutil.move(src, dest) subprocess.check_output(['tar', '-cf', pkg + '.tar', pkg], cwd='/opt/antergos-packages/%s' % pkg) if depends and len(hook_queue) > 1: all_deps.append(depends) elif len(hook_queue) == 1: all_deps.append(1) return all_deps
def maybe_check_for_new_items(): return db.exists('FEED_CHECKED')
def process_github(self): if not self.is_manual: self.payload = json.loads(self.request.data) # Save payload in the database temporarily in case we need it later. dt = datetime.datetime.now().strftime("%m%d%Y-%I%M") key = "antbs:github:payloads:%s" % dt if db.exists(key): for i in range(1, 5): tmp = "%s:%s" % (key, i) if not db.exists(tmp): key = tmp break db.hmset(key, self.payload) db.rpush("antbs:github:payloads:index", key) db.expire(key, 172800) self.full_name = self.payload["repository"]["full_name"] self.repo = self.payload["repository"]["name"] self.pusher = self.payload["pusher"]["name"] self.commits = self.payload["commits"] if self.repo == "numix-icon-theme": rate_limit = True if "numix-icon-theme" not in self.the_queue and "numix-icon-theme" != self.building: if not db.exists("numix-commit-flag"): self.changes.append(["numix-icon-theme"]) self.is_numix = True db.setex("numix-commit-flag", 1200, "True") rate_limit = False if rate_limit: msg = "RATE LIMIT IN EFFECT FOR numix-icon-theme" logger.info(msg) self.result = json.dumps({"msg": msg}) else: self.repo = "antergos-packages" elif self.repo == "cnchi-dev": self.changes.append(["cnchi-dev"]) self.repo = "antergos-packages" self.is_cnchi = True # idle = db.get('idle') # working = db.exists('creating-cnchi-archive-from-dev') # check = 'cnchi-dev' != self.building or idle == "True" # if not working and 'cnchi-dev' not in self.the_queue and check: # db.set('creating-cnchi-archive-from-dev', 'True') # cnchi_git = 'https://github.com/lots0logs/cnchi-dev.git' # cnchi_clone = '/tmp/cnchi' # git = '/tmp/cnchi/.git' # cnchi_tar_tmp = '/tmp/cnchi.tar' # cnchi_tar = '/srv/antergos.org/cnchi.tar' # # for f in [cnchi_clone, cnchi_tar, cnchi_tar_tmp]: # if os.path.exists(f): # rm_file_or_dir(f) # try: # subprocess.check_call(['git', 'clone', cnchi_git, 'cnchi'], cwd='/tmp') # shutil.rmtree(git) # subprocess.check_call(['tar', '-cf', '/tmp/cnchi.tar', '-C', '/tmp', 'cnchi']) # shutil.copy('/tmp/cnchi.tar', '/srv/antergos.org/') # except subprocess.CalledProcessError as err: # logger.error(err.output) # # db.delete('creating-cnchi-archive-from-dev') elif self.pusher != "antbs": for commit in self.commits: self.changes.append(commit["modified"]) self.changes.append(commit["added"])
def build_iso(): iso_arch = ['x86_64', 'i686'] in_dir_last = len([name for name in os.listdir('/srv/antergos.info/repo/iso/testing')]) if in_dir_last is None: in_dir_last = "0" db.set('pkg_count_iso', in_dir_last) is_minimal = db.get('isoMinimal') if is_minimal == 'True': iso_name = 'antergos-iso-minimal-' else: iso_name = 'antergos-iso-' for arch in iso_arch: if db.exists('iso:one:arch') and arch == 'x86_64': continue pkgobj = package.get_pkg_object(iso_name + arch) failed = False db.incr('build_number') dt = datetime.datetime.now().strftime("%m/%d/%Y %I:%M%p") build_id = db.get('build_number') pkgobj.save_to_db('builds', build_id, 'list') this_log = 'build_log:%s' % build_id db.set('%s:start' % this_log, dt) db.set('building_num', build_id) db.hset('now_building', 'build_id', build_id) db.hset('now_building', 'key', this_log) db.hset('now_building', 'pkg', pkgobj.name) db.set(this_log, True) db.set('building_start', dt) logger.info('Building %s' % pkgobj.name) db.set('building', 'Building: %s' % pkgobj.name) db.lrem('queue', 0, pkgobj.name) db.set('%s:pkg' % this_log, pkgobj.name) db.set('%s:version' % this_log, pkgobj.version) flag = '/srv/antergos.info/repo/iso/testing/.ISO32' minimal = '/srv/antergos.info/repo/iso/testing/.MINIMAL' if arch is 'i686': if not os.path.exists(flag): open(flag, 'a').close() else: if os.path.exists(flag): os.remove(flag) if is_minimal == "True": out_dir = '/out' if not os.path.exists(minimal): open(minimal, 'a').close() else: out_dir = '/out' if os.path.exists(minimal): os.remove(minimal) # Get and compile translations for updater script # TODO: Move this into its own method. trans_dir = "/opt/antergos-iso-translations/" trans_files_dir = os.path.join(trans_dir, "translations/antergos.cnchi_updaterpot") dest_dir = '/srv/antergos.info/repo/iso/testing/trans' if not os.path.exists(dest_dir): os.mkdir(dest_dir) try: subprocess.check_call(['tx', 'pull', '-a', '-r', 'antergos.cnchi_updaterpot', '--minimum-perc=50'], cwd=trans_dir) for r, d, f in os.walk(trans_files_dir): for tfile in f: logger.info('tfile is %s' % tfile) logger.info('tfile cut is %s' % tfile[:-2]) mofile = tfile[:-2] + 'mo' logger.info('mofile is %s' % mofile) subprocess.check_call(['msgfmt', '-v', tfile, '-o', mofile], cwd=trans_files_dir) os.rename(os.path.join(trans_files_dir, mofile), os.path.join(dest_dir, mofile)) except subprocess.CalledProcessError as err: logger.error(err.output) except Exception as err: logger.error(err) nm = iso_name + arch # Initiate communication with docker daemon run_docker_clean(nm) hconfig = create_host_config(privileged=True, cap_add=['ALL'], binds={ '/opt/archlinux-mkarchiso': { 'bind': '/start', 'ro': False }, '/run/dbus': { 'bind': '/var/run/dbus', 'ro': False }, '/srv/antergos.info/repo/iso/testing': { 'bind': out_dir, 'ro': False }}, restart_policy={ "MaximumRetryCount": 2, "Name": "on-failure"}) try: iso_container = doc.create_container("antergos/mkarchiso", command='/start/run.sh', tty=True, name=nm, host_config=hconfig, cpuset='0-3') db.set('container', iso_container.get('Id')) except Exception as err: logger.error("Cant connect to Docker daemon. Error msg: %s", err) failed = True break try: doc.start(iso_container, privileged=True, cap_add=['ALL'], binds={ '/opt/archlinux-mkarchiso': { 'bind': '/start', 'ro': False }, '/run/dbus': { 'bind': '/var/run/dbus', 'ro': False }, '/srv/antergos.info/repo/iso/testing': { 'bind': out_dir, 'ro': False }, }) cont = db.get('container') stream_process = Process(target=publish_build_ouput, args=(cont, this_log)) stream_process.start() result = doc.wait(cont) result2 = None if result is not 0: doc.restart(cont) stream_process2 = Process(target=publish_build_ouput, args=(cont, this_log)) stream_process2.start() result2 = doc.wait(cont) if result2 is not 0: # failed = True # db.set('build_failed', "True") logger.error('[CONTAINER EXIT CODE] Container %s exited. Return code was %s' % (nm, result)) if result is 0 or (result2 and result2 is 0): logger.info('[CONTAINER EXIT CODE] Container %s exited. Return code was %s' % (nm, result)) db.set('build_failed', "False") except Exception as err: logger.error("Cant start container. Error msg: %s", err) break db.publish('build-output', 'ENDOFLOG') db.set('%s:end' % this_log, datetime.datetime.now().strftime("%m/%d/%Y %I:%M%p")) in_dir = len([name for name in os.listdir('/srv/antergos.info/repo/iso/testing')]) last_count = int(db.get('pkg_count_iso')) if in_dir > last_count: db.incr('pkg_count_iso', (in_dir - last_count)) db.rpush('completed', build_id) db.set('%s:result' % this_log, 'completed') # db.set('%s:review_stat' % this_log, '1') else: logger.error('%s not found after container exit.' % iso_name + arch) failed = True db.set('%s:result' % this_log, 'failed') db.rpush('failed', build_id) remove('/opt/archlinux-mkarchiso/antergos-iso') doc.remove_container(cont, v=True)