def get_repo(self): """ Return string of generated repository located LOCALLY It downloads all tagged packages and creates repo via createrepo :return: str """ dir_prefix = common.conf["nspawn"]["basedir"] process.run("{HOSTPACKAGER} install createrepo koji".format( **common.trans_dict), ignore_status=True) if common.is_recursive_download(): dirname = os.path.join(dir_prefix, "localrepo_recursive") else: dirname = os.path.join( dir_prefix, "localrepo_%s_%s_%s" % (self.name, self.stream, self.version)) absdir = os.path.abspath(dirname) # Test if directory contains repository, otherwise download everything again if os.path.exists(os.path.join(absdir, "repodata", "repomd.xml")): pass else: if not os.path.exists(absdir): os.mkdir(absdir) self.download_tagged(absdir) if common.is_recursive_download(): allmodules = self.generateDepModules() for mo in allmodules: localrepo = PDCParserKoji(mo, allmodules[mo]) localrepo.download_tagged(dirname) process.run("cd %s; createrepo -v %s" % (absdir, absdir), shell=True, verbose=core.is_debug()) return "file://%s" % absdir
def start(self, command="/bin/true"): """ start the RPM based module (like systemctl start service) :param command: Do not use it directly (It is defined in config.yaml) :return: None """ command = self.info.get('start') or command self.run(command, shell=True, ignore_bg_processes=True, verbose=core.is_debug()) self.status() trans_dict["GUESTPACKAGER"] = self.get_packager()
def main(): from core.license import License from core import is_debug if License().check() or is_debug(): from core.config import Config if Config().check() or is_debug(): app = BaseApp() widget = MainView() widget.Show() app.MainLoop() else: from views.config import ConfigView app = wx.App() widget = ConfigView() widget.Show() app.MainLoop() else: from views.license import LicenseView app = wx.App() widget = LicenseView() if widget.ShowModal() == wx.OK: main() else: app.Destroy()
def tmpfunc(): a = process.run( "cd %s; koji download-build %s -a %s -a noarch" % (dirname, pkgbouid, common.conf["generic"]["arch"]), shell=True, verbose=core.is_debug(), ignore_status=True) if a.exit_status == 1: if "packages available for" in a.stdout.strip(): core.print_debug( 'UNABLE TO DOWNLOAD package (intended for other architectures, GOOD):', a.command) else: raise mtfexceptions.KojiExc( 'UNABLE TO DOWNLOAD package (KOJI issue, BAD):', a.command)
def get_pos_process(self): if self.process is None: return None try: p = psutil.Process(pid=self.process.pid) if core.is_debug(): return p ps = p.children() if len(ps) != 1: return None p = ps[0] if p.name().lower() == 'proofofspace.exe': return p return None except Exception as e: return None
def resume(self): self.sub_task.suspend = False self.sub_task.suspend_time = 0 self.sub_task.suspend_remain_time = 0 self.updateTask() if self.process is None: return try: if is_debug(): p = psutil.Process(pid=self.process.pid) p.resume() return pos_process = self.get_pos_process() if pos_process: pos_process.resume() except: pass
def suspend(self, for_time=0): self.sub_task.suspend = True self.sub_task.suspend_time = for_time self.sub_task.suspend_remain_time = for_time self.updateTask() if self.process is None: return try: if is_debug(): p = psutil.Process(pid=self.process.pid) p.suspend() return pos_process = self.get_pos_process() if pos_process: pos_process.suspend() except: pass
def download_tagged(self, dirname): """ Downloads packages to directory, based on koji tags It downloads just ARCH and noarch packages :param dirname: string :return: None """ core.print_info("DOWNLOADING ALL packages for %s_%s_%s" % (self.name, self.stream, self.version)) for foo in process.run("koji list-tagged --quiet %s" % self.get_pdc_info()["koji_tag"], verbose=core.is_debug()).stdout.split("\n"): pkgbouid = foo.strip().split(" ")[0] if len(pkgbouid) > 4: core.print_debug("DOWNLOADING: %s" % foo) @timeoutlib.Retry( attempts=common.conf["generic"]["retrycount"] * 10, timeout=common.conf["generic"]["retrytimeout"] * 60, delay=common.conf["generic"]["retrytimeout"], error=mtfexceptions.KojiExc( "RETRY: Unbale to fetch package from koji after %d attempts" % (common.conf["generic"]["retrycount"] * 10))) def tmpfunc(): a = process.run( "cd %s; koji download-build %s -a %s -a noarch" % (dirname, pkgbouid, common.conf["generic"]["arch"]), shell=True, verbose=core.is_debug(), ignore_status=True) if a.exit_status == 1: if "packages available for" in a.stdout.strip(): core.print_debug( 'UNABLE TO DOWNLOAD package (intended for other architectures, GOOD):', a.command) else: raise mtfexceptions.KojiExc( 'UNABLE TO DOWNLOAD package (KOJI issue, BAD):', a.command) tmpfunc() core.print_info("DOWNLOADING finished")
def installTestDependencies(self, packages=None): """ Install packages on a host machine to prepare a test environment. :param (list): packages to install. If not specified, rpms from config.yaml will be installed. :return: None """ if not packages: packages = self.get_test_dependencies() if packages: core.print_info("Installs test dependencies: ", packages) # you have to have root permission to install packages: try: self.runHost("{HOSTPACKAGER} install " + " ".join(packages), ignore_status=False, verbose=core.is_debug()) except avocado.utils.process.CmdError as e: raise mtfexceptions.CmdExc( "Installation failed; Do you have permission to do that?", e)
def run(self): t = self.task plat = platform.system() if plat == 'Windows': folder = 'windows' bin_file = 'chia-plotter-windows-amd64.exe' elif plat == 'Darwin': folder = 'macos' bin_file = 'chia-plotter-darwin-amd64' elif plat == 'Linux': folder = 'linux' bin_file = 'chia-plotter-linux-amd64' else: return False if plat == 'Windows' and 'DEBUG' in os.environ and os.environ[ 'DEBUG'] == '1': bin_file = 'test.exe' exe_cwd = os.path.join(BASE_DIR, 'bin', folder, 'plotter') exe = os.path.join(exe_cwd, bin_file) args = [ exe, '-action', 'plotting', '-plotting-fpk', t.fpk, '-plotting-ppk', t.ppk, '-plotting-n', '1', '-r', f'{t.number_of_thread}', '-b', f'{t.memory_size}', '-d', t.hdd_folder, '-t', t.temporary_folder, '-2', t.temporary_folder, ] while True: delay_remain = self.task.delay_remain() if self.stopping: self.stopping = False self.sub_task.status = '已取消' self.sub_task.finish = True self.sub_task.success = False self.sub_task.end_time = datetime.now() for i in range(self.task.current_task_index + 1, self.task.count): rest_sub_task = self.task.sub_tasks[i] rest_sub_task.success = False rest_sub_task.status = '已手动停止' rest_sub_task.finish = True self.updateTask(sub_task=rest_sub_task) else: self.updateTask() break if delay_remain: time.sleep(1) continue self.task.running = False if not PlotTaskManager.assign_task(self.task): self.sub_task.status = '排队中' time.sleep(1) continue self.sub_task.begin_time = datetime.now() self.sub_task.status = '正在执行' self.sub_task.progress = 0 self.updateTask() self.process = Popen(args, stdout=PIPE, stderr=PIPE, cwd=exe_cwd, creationflags=CREATE_NO_WINDOW) success = True finished = False while True: line = self.process.stdout.readline() text = line.decode('utf-8', errors='replace') text = text.rstrip() if not text and self.process.poll() is not None: break if text: self.sub_task.log.append(text) _failed, _finished = self.handleLog(text) if _failed: success = False if _finished: finished = True self.signalTaskOutput.emit(self.task, self.sub_task, text) self.updateTask() self.process = None self.task.running = False stop = False failed = False plot_path = os.path.join(self.sub_task.hdd_folder, self.plot_filename) if self.stopping: self.stopping = False stop = failed = True self.sub_task.status = '已手动停止' elif not self.plot_filename: stop = failed = True self.sub_task.status = '没有plot文件名' elif not success or not finished: stop = failed = True self.sub_task.status = '失败' elif not os.path.exists(plot_path) and not is_debug(): stop = failed = True self.sub_task.status = 'plot文件不存在' else: self.sub_task.status = '完成' self.sub_task.finish = True self.sub_task.success = True self.sub_task.progress = 100.0 self.sub_task.suspended_seconds = 0 self.sub_task.end_time = datetime.now() self.sub_task.plot_file = plot_path self.task.signalNewPlot.emit(self.task, self.sub_task) self.updateTask() break self.updateTask() if failed: self.sub_task.success = False self.sub_task.finish = True self.sub_task.end_time = datetime.now() break if stop: for i in range(self.task.current_task_index + 1, self.task.count): rest_sub_task = self.task.sub_tasks[i] rest_sub_task.success = False rest_sub_task.status = '已手动停止' rest_sub_task.finish = True self.updateTask(sub_task=rest_sub_task) else: self.updateTask() break
def handleProgress(self, text): if text.startswith('Starting phase 1/4'): self.phase = 1 self.task.phase = 1 elif text.startswith('Computing table'): self.table = text elif text.startswith('Starting phase 2/4'): self.phase = 2 self.task.phase = 2 elif text.startswith('Backpropagating on table'): self.table = text elif text.startswith('Starting phase 3/4'): self.phase = 3 self.task.phase = 3 elif text.startswith('Compressing tables'): self.phase3_first_computation = True self.table = text elif text.startswith('First computation'): self.phase3_first_computation = False elif text.startswith('Starting phase 4/4'): self.phase = 4 self.task.phase = 4 elif text.startswith('Wrote left entries'): if self.table == 'Backpropagating on table 7': self.sub_task.progress = 29.167 self.updateTask() elif text.startswith('Bucket'): r = re.compile(r'Bucket (\d*) ') found = re.findall(r, text) if not found: return self.bucket = int(found[0]) if self.phase == 1: if self.table == 'Computing table 2': base_progress = 0.0 max_progress = 4.167 total_bucket = 127 elif self.table == 'Computing table 3': base_progress = 4.167 max_progress = 8.333 total_bucket = 127 elif self.table == 'Computing table 4': base_progress = 8.333 max_progress = 12.500 total_bucket = 127 elif self.table == 'Computing table 5': base_progress = 12.500 max_progress = 16.667 total_bucket = 127 elif self.table == 'Computing table 6': base_progress = 16.667 max_progress = 20.833 total_bucket = 127 elif self.table == 'Computing table 7': base_progress = 20.833 max_progress = 25.000 total_bucket = 127 else: return elif self.phase == 2: if self.table == 'Backpropagating on table 6': base_progress = 29.167 max_progress = 33.333 total_bucket = 110 elif self.table == 'Backpropagating on table 5': base_progress = 33.333 max_progress = 37.500 total_bucket = 110 elif self.table == 'Backpropagating on table 4': base_progress = 37.500 max_progress = 41.667 total_bucket = 110 elif self.table == 'Backpropagating on table 3': base_progress = 41.667 max_progress = 45.833 total_bucket = 110 elif self.table == 'Backpropagating on table 2': base_progress = 45.833 max_progress = 50.000 total_bucket = 110 else: return elif self.phase == 3: if self.table == 'Compressing tables 1 and 2': base_progress = 50.000 max_progress = 54.167 total_bucket = 127 elif self.table == 'Compressing tables 2 and 3': base_progress = 54.167 max_progress = 58.333 total_bucket = 102 + 81 + 1 if not self.phase3_first_computation: self.bucket = 102 + 1 + self.bucket elif self.table == 'Compressing tables 3 and 4': base_progress = 58.333 max_progress = 62.500 total_bucket = 102 + 82 + 1 if not self.phase3_first_computation: self.bucket = 102 + 1 + self.bucket elif self.table == 'Compressing tables 4 and 5': base_progress = 62.500 max_progress = 66.667 total_bucket = 103 + 83 + 1 if not self.phase3_first_computation: self.bucket = 103 + 1 + self.bucket elif self.table == 'Compressing tables 5 and 6': base_progress = 66.667 max_progress = 70.833 total_bucket = 105 + 86 + 1 if not self.phase3_first_computation: self.bucket = 105 + 1 + self.bucket elif self.table == 'Compressing tables 6 and 7': base_progress = 70.833 max_progress = 75.000 total_bucket = 110 + 95 + 1 if not self.phase3_first_computation: self.bucket = 110 + 1 + self.bucket else: return elif self.phase == 4: base_progress = 75.000 max_progress = 99.000 total_bucket = 127 else: return # bucket_progress = 100 * self.bucket / total_bucket # progress = bucket_progress * max_progress / 100 self.sub_task.progress = (100 * self.bucket / total_bucket) * ( max_progress - base_progress) / 100 + base_progress self.updateTask() elif text.startswith('Final File size'): self.copying = True self.sub_task.status = '生成文件' self.task.signalMakingPlot.emit(self.task, self.sub_task) self.updateTask() if core.is_debug(): time.sleep(10) elif text.startswith('Copied final file'): self.sub_task.progress = 100.0 self.updateTask()
def accept(self) -> None: if self.modify: thread_num = self.spinThreadNum.value() memory_size = self.spinMemory.value() self.task.hdd_folder = self.comboHDD.currentData(Qt.UserRole) self.task.number_of_thread = thread_num self.task.memory_size = memory_size super().accept() return fpk = self.editFpk.toPlainText() ppk = self.editPpk.toPlainText() ssd_folder = self.comboSSD.currentData() hdd_folder = self.comboHDD.currentData() CreatePlotDialog.last_ssd_folder = ssd_folder CreatePlotDialog.last_hdd_folder = hdd_folder delayTime = self.timeEditDelay.time() delay = delayTime.hour() * 60 * 60 + delayTime.minute( ) * 60 + delayTime.second() specify_count = self.checkBoxSpecifyCount.isChecked() number = self.spinNumber.value() thread_num = self.spinThreadNum.value() memory_size = self.spinMemory.value() if not os.path.exists(ssd_folder): QMessageBox.information(self, '提示', '临时目录不存在') return if not os.path.exists(hdd_folder): QMessageBox.information(self, '提示', '最终目录不存在') return if not fpk: QMessageBox.information(self, '提示', '请输入fpk') return if not ppk: QMessageBox.information(self, '提示', '请输入ppk') return if not specify_count: number = 1 config = get_config() config['fpk'] = fpk config['ppk'] = ppk config['specify_count'] = specify_count config['num'] = number config['thread_num'] = thread_num config['memory_size'] = memory_size save_config() temporary_folder = os.path.join(ssd_folder, make_name(12)) temporary_folder = temporary_folder.replace('\\', '/') try: os.mkdir(temporary_folder) except: QMessageBox.information(self, '提示', '创建临时目录失败 %s' % temporary_folder) return ssd_usage = psutil.disk_usage(ssd_folder) hdd_usage = psutil.disk_usage(hdd_folder) if not is_debug() and ssd_usage.free < 2**30 * 332: if QMessageBox.information( self, '提示', '临时目录的空间不够332G,确定要继续吗?', QMessageBox.Ok | QMessageBox.Cancel) == QMessageBox.Cancel: return if not is_debug() and hdd_usage.free < 2**30 * 102: if QMessageBox.information( self, '提示', '最终目录的空间不够101G,确定要继续吗?', QMessageBox.Ok | QMessageBox.Cancel) == QMessageBox.Cancel: return self.task.create_time = datetime.now() self.task.fpk = fpk self.task.ppk = ppk self.task.ssd_folder = ssd_folder self.task.hdd_folder = hdd_folder self.task.temporary_folder = temporary_folder self.task.specify_count = specify_count self.task.count = number self.task.number_of_thread = thread_num self.task.memory_size = memory_size self.task.delay_seconds = delay if specify_count: for i in range(number): self.task.sub_tasks.append(PlotSubTask(self.task, i)) else: self.task.sub_tasks.append(PlotSubTask(self.task, 0)) super().accept()