def get_repo(self):
"""
Return string of generated repository located LOCALLY
It downloads all tagged packages and creates repo via createrepo
:return: str
"""
dir_prefix = common.conf["nspawn"]["basedir"]
process.run("{HOSTPACKAGER} install createrepo koji".format(
**common.trans_dict),
ignore_status=True)
if common.is_recursive_download():
dirname = os.path.join(dir_prefix, "localrepo_recursive")
else:
dirname = os.path.join(
dir_prefix,
"localrepo_%s_%s_%s" % (self.name, self.stream, self.version))
absdir = os.path.abspath(dirname)
# Test if directory contains repository, otherwise download everything again
if os.path.exists(os.path.join(absdir, "repodata", "repomd.xml")):
pass
else:
if not os.path.exists(absdir):
os.mkdir(absdir)
self.download_tagged(absdir)
if common.is_recursive_download():
allmodules = self.generateDepModules()
for mo in allmodules:
localrepo = PDCParserKoji(mo, allmodules[mo])
localrepo.download_tagged(dirname)
process.run("cd %s; createrepo -v %s" % (absdir, absdir),
shell=True,
verbose=core.is_debug())
return "file://%s" % absdir
def start(self, command="/bin/true"):
"""
start the RPM based module (like systemctl start service)
:param command: Do not use it directly (It is defined in config.yaml)
:return: None
"""
command = self.info.get('start') or command
self.run(command,
shell=True,
ignore_bg_processes=True,
verbose=core.is_debug())
self.status()
trans_dict["GUESTPACKAGER"] = self.get_packager()
def main():
from core.license import License
from core import is_debug
if License().check() or is_debug():
from core.config import Config
if Config().check() or is_debug():
app = BaseApp()
widget = MainView()
widget.Show()
app.MainLoop()
else:
from views.config import ConfigView
app = wx.App()
widget = ConfigView()
widget.Show()
app.MainLoop()
else:
from views.license import LicenseView
app = wx.App()
widget = LicenseView()
if widget.ShowModal() == wx.OK:
main()
else:
app.Destroy()
def tmpfunc():
a = process.run(
"cd %s; koji download-build %s -a %s -a noarch" %
(dirname, pkgbouid, common.conf["generic"]["arch"]),
shell=True,
verbose=core.is_debug(),
ignore_status=True)
if a.exit_status == 1:
if "packages available for" in a.stdout.strip():
core.print_debug(
'UNABLE TO DOWNLOAD package (intended for other architectures, GOOD):',
a.command)
else:
raise mtfexceptions.KojiExc(
'UNABLE TO DOWNLOAD package (KOJI issue, BAD):',
a.command)
def get_pos_process(self):
if self.process is None:
return None
try:
p = psutil.Process(pid=self.process.pid)
if core.is_debug():
return p
ps = p.children()
if len(ps) != 1:
return None
p = ps[0]
if p.name().lower() == 'proofofspace.exe':
return p
return None
except Exception as e:
return None
def resume(self):
self.sub_task.suspend = False
self.sub_task.suspend_time = 0
self.sub_task.suspend_remain_time = 0
self.updateTask()
if self.process is None:
return
try:
if is_debug():
p = psutil.Process(pid=self.process.pid)
p.resume()
return
pos_process = self.get_pos_process()
if pos_process:
pos_process.resume()
except:
pass
def suspend(self, for_time=0):
self.sub_task.suspend = True
self.sub_task.suspend_time = for_time
self.sub_task.suspend_remain_time = for_time
self.updateTask()
if self.process is None:
return
try:
if is_debug():
p = psutil.Process(pid=self.process.pid)
p.suspend()
return
pos_process = self.get_pos_process()
if pos_process:
pos_process.suspend()
except:
pass
def download_tagged(self, dirname):
"""
Downloads packages to directory, based on koji tags
It downloads just ARCH and noarch packages
:param dirname: string
:return: None
"""
core.print_info("DOWNLOADING ALL packages for %s_%s_%s" %
(self.name, self.stream, self.version))
for foo in process.run("koji list-tagged --quiet %s" %
self.get_pdc_info()["koji_tag"],
verbose=core.is_debug()).stdout.split("\n"):
pkgbouid = foo.strip().split(" ")[0]
if len(pkgbouid) > 4:
core.print_debug("DOWNLOADING: %s" % foo)
@timeoutlib.Retry(
attempts=common.conf["generic"]["retrycount"] * 10,
timeout=common.conf["generic"]["retrytimeout"] * 60,
delay=common.conf["generic"]["retrytimeout"],
error=mtfexceptions.KojiExc(
"RETRY: Unbale to fetch package from koji after %d attempts"
% (common.conf["generic"]["retrycount"] * 10)))
def tmpfunc():
a = process.run(
"cd %s; koji download-build %s -a %s -a noarch" %
(dirname, pkgbouid, common.conf["generic"]["arch"]),
shell=True,
verbose=core.is_debug(),
ignore_status=True)
if a.exit_status == 1:
if "packages available for" in a.stdout.strip():
core.print_debug(
'UNABLE TO DOWNLOAD package (intended for other architectures, GOOD):',
a.command)
else:
raise mtfexceptions.KojiExc(
'UNABLE TO DOWNLOAD package (KOJI issue, BAD):',
a.command)
tmpfunc()
core.print_info("DOWNLOADING finished")
def installTestDependencies(self, packages=None):
"""
Install packages on a host machine to prepare a test environment.
:param (list): packages to install. If not specified, rpms from config.yaml
will be installed.
:return: None
"""
if not packages:
packages = self.get_test_dependencies()
if packages:
core.print_info("Installs test dependencies: ", packages)
# you have to have root permission to install packages:
try:
self.runHost("{HOSTPACKAGER} install " + " ".join(packages),
ignore_status=False,
verbose=core.is_debug())
except avocado.utils.process.CmdError as e:
raise mtfexceptions.CmdExc(
"Installation failed; Do you have permission to do that?",
e)
def run(self):
t = self.task
plat = platform.system()
if plat == 'Windows':
folder = 'windows'
bin_file = 'chia-plotter-windows-amd64.exe'
elif plat == 'Darwin':
folder = 'macos'
bin_file = 'chia-plotter-darwin-amd64'
elif plat == 'Linux':
folder = 'linux'
bin_file = 'chia-plotter-linux-amd64'
else:
return False
if plat == 'Windows' and 'DEBUG' in os.environ and os.environ[
'DEBUG'] == '1':
bin_file = 'test.exe'
exe_cwd = os.path.join(BASE_DIR, 'bin', folder, 'plotter')
exe = os.path.join(exe_cwd, bin_file)
args = [
exe,
'-action',
'plotting',
'-plotting-fpk',
t.fpk,
'-plotting-ppk',
t.ppk,
'-plotting-n',
'1',
'-r',
f'{t.number_of_thread}',
'-b',
f'{t.memory_size}',
'-d',
t.hdd_folder,
'-t',
t.temporary_folder,
'-2',
t.temporary_folder,
]
while True:
delay_remain = self.task.delay_remain()
if self.stopping:
self.stopping = False
self.sub_task.status = '已取消'
self.sub_task.finish = True
self.sub_task.success = False
self.sub_task.end_time = datetime.now()
for i in range(self.task.current_task_index + 1,
self.task.count):
rest_sub_task = self.task.sub_tasks[i]
rest_sub_task.success = False
rest_sub_task.status = '已手动停止'
rest_sub_task.finish = True
self.updateTask(sub_task=rest_sub_task)
else:
self.updateTask()
break
if delay_remain:
time.sleep(1)
continue
self.task.running = False
if not PlotTaskManager.assign_task(self.task):
self.sub_task.status = '排队中'
time.sleep(1)
continue
self.sub_task.begin_time = datetime.now()
self.sub_task.status = '正在执行'
self.sub_task.progress = 0
self.updateTask()
self.process = Popen(args,
stdout=PIPE,
stderr=PIPE,
cwd=exe_cwd,
creationflags=CREATE_NO_WINDOW)
success = True
finished = False
while True:
line = self.process.stdout.readline()
text = line.decode('utf-8', errors='replace')
text = text.rstrip()
if not text and self.process.poll() is not None:
break
if text:
self.sub_task.log.append(text)
_failed, _finished = self.handleLog(text)
if _failed:
success = False
if _finished:
finished = True
self.signalTaskOutput.emit(self.task, self.sub_task, text)
self.updateTask()
self.process = None
self.task.running = False
stop = False
failed = False
plot_path = os.path.join(self.sub_task.hdd_folder,
self.plot_filename)
if self.stopping:
self.stopping = False
stop = failed = True
self.sub_task.status = '已手动停止'
elif not self.plot_filename:
stop = failed = True
self.sub_task.status = '没有plot文件名'
elif not success or not finished:
stop = failed = True
self.sub_task.status = '失败'
elif not os.path.exists(plot_path) and not is_debug():
stop = failed = True
self.sub_task.status = 'plot文件不存在'
else:
self.sub_task.status = '完成'
self.sub_task.finish = True
self.sub_task.success = True
self.sub_task.progress = 100.0
self.sub_task.suspended_seconds = 0
self.sub_task.end_time = datetime.now()
self.sub_task.plot_file = plot_path
self.task.signalNewPlot.emit(self.task, self.sub_task)
self.updateTask()
break
self.updateTask()
if failed:
self.sub_task.success = False
self.sub_task.finish = True
self.sub_task.end_time = datetime.now()
break
if stop:
for i in range(self.task.current_task_index + 1,
self.task.count):
rest_sub_task = self.task.sub_tasks[i]
rest_sub_task.success = False
rest_sub_task.status = '已手动停止'
rest_sub_task.finish = True
self.updateTask(sub_task=rest_sub_task)
else:
self.updateTask()
break
def handleProgress(self, text):
if text.startswith('Starting phase 1/4'):
self.phase = 1
self.task.phase = 1
elif text.startswith('Computing table'):
self.table = text
elif text.startswith('Starting phase 2/4'):
self.phase = 2
self.task.phase = 2
elif text.startswith('Backpropagating on table'):
self.table = text
elif text.startswith('Starting phase 3/4'):
self.phase = 3
self.task.phase = 3
elif text.startswith('Compressing tables'):
self.phase3_first_computation = True
self.table = text
elif text.startswith('First computation'):
self.phase3_first_computation = False
elif text.startswith('Starting phase 4/4'):
self.phase = 4
self.task.phase = 4
elif text.startswith('Wrote left entries'):
if self.table == 'Backpropagating on table 7':
self.sub_task.progress = 29.167
self.updateTask()
elif text.startswith('Bucket'):
r = re.compile(r'Bucket (\d*) ')
found = re.findall(r, text)
if not found:
return
self.bucket = int(found[0])
if self.phase == 1:
if self.table == 'Computing table 2':
base_progress = 0.0
max_progress = 4.167
total_bucket = 127
elif self.table == 'Computing table 3':
base_progress = 4.167
max_progress = 8.333
total_bucket = 127
elif self.table == 'Computing table 4':
base_progress = 8.333
max_progress = 12.500
total_bucket = 127
elif self.table == 'Computing table 5':
base_progress = 12.500
max_progress = 16.667
total_bucket = 127
elif self.table == 'Computing table 6':
base_progress = 16.667
max_progress = 20.833
total_bucket = 127
elif self.table == 'Computing table 7':
base_progress = 20.833
max_progress = 25.000
total_bucket = 127
else:
return
elif self.phase == 2:
if self.table == 'Backpropagating on table 6':
base_progress = 29.167
max_progress = 33.333
total_bucket = 110
elif self.table == 'Backpropagating on table 5':
base_progress = 33.333
max_progress = 37.500
total_bucket = 110
elif self.table == 'Backpropagating on table 4':
base_progress = 37.500
max_progress = 41.667
total_bucket = 110
elif self.table == 'Backpropagating on table 3':
base_progress = 41.667
max_progress = 45.833
total_bucket = 110
elif self.table == 'Backpropagating on table 2':
base_progress = 45.833
max_progress = 50.000
total_bucket = 110
else:
return
elif self.phase == 3:
if self.table == 'Compressing tables 1 and 2':
base_progress = 50.000
max_progress = 54.167
total_bucket = 127
elif self.table == 'Compressing tables 2 and 3':
base_progress = 54.167
max_progress = 58.333
total_bucket = 102 + 81 + 1
if not self.phase3_first_computation:
self.bucket = 102 + 1 + self.bucket
elif self.table == 'Compressing tables 3 and 4':
base_progress = 58.333
max_progress = 62.500
total_bucket = 102 + 82 + 1
if not self.phase3_first_computation:
self.bucket = 102 + 1 + self.bucket
elif self.table == 'Compressing tables 4 and 5':
base_progress = 62.500
max_progress = 66.667
total_bucket = 103 + 83 + 1
if not self.phase3_first_computation:
self.bucket = 103 + 1 + self.bucket
elif self.table == 'Compressing tables 5 and 6':
base_progress = 66.667
max_progress = 70.833
total_bucket = 105 + 86 + 1
if not self.phase3_first_computation:
self.bucket = 105 + 1 + self.bucket
elif self.table == 'Compressing tables 6 and 7':
base_progress = 70.833
max_progress = 75.000
total_bucket = 110 + 95 + 1
if not self.phase3_first_computation:
self.bucket = 110 + 1 + self.bucket
else:
return
elif self.phase == 4:
base_progress = 75.000
max_progress = 99.000
total_bucket = 127
else:
return
# bucket_progress = 100 * self.bucket / total_bucket
# progress = bucket_progress * max_progress / 100
self.sub_task.progress = (100 * self.bucket / total_bucket) * (
max_progress - base_progress) / 100 + base_progress
self.updateTask()
elif text.startswith('Final File size'):
self.copying = True
self.sub_task.status = '生成文件'
self.task.signalMakingPlot.emit(self.task, self.sub_task)
self.updateTask()
if core.is_debug():
time.sleep(10)
elif text.startswith('Copied final file'):
self.sub_task.progress = 100.0
self.updateTask()
def accept(self) -> None:
if self.modify:
thread_num = self.spinThreadNum.value()
memory_size = self.spinMemory.value()
self.task.hdd_folder = self.comboHDD.currentData(Qt.UserRole)
self.task.number_of_thread = thread_num
self.task.memory_size = memory_size
super().accept()
return
fpk = self.editFpk.toPlainText()
ppk = self.editPpk.toPlainText()
ssd_folder = self.comboSSD.currentData()
hdd_folder = self.comboHDD.currentData()
CreatePlotDialog.last_ssd_folder = ssd_folder
CreatePlotDialog.last_hdd_folder = hdd_folder
delayTime = self.timeEditDelay.time()
delay = delayTime.hour() * 60 * 60 + delayTime.minute(
) * 60 + delayTime.second()
specify_count = self.checkBoxSpecifyCount.isChecked()
number = self.spinNumber.value()
thread_num = self.spinThreadNum.value()
memory_size = self.spinMemory.value()
if not os.path.exists(ssd_folder):
QMessageBox.information(self, '提示', '临时目录不存在')
return
if not os.path.exists(hdd_folder):
QMessageBox.information(self, '提示', '最终目录不存在')
return
if not fpk:
QMessageBox.information(self, '提示', '请输入fpk')
return
if not ppk:
QMessageBox.information(self, '提示', '请输入ppk')
return
if not specify_count:
number = 1
config = get_config()
config['fpk'] = fpk
config['ppk'] = ppk
config['specify_count'] = specify_count
config['num'] = number
config['thread_num'] = thread_num
config['memory_size'] = memory_size
save_config()
temporary_folder = os.path.join(ssd_folder, make_name(12))
temporary_folder = temporary_folder.replace('\\', '/')
try:
os.mkdir(temporary_folder)
except:
QMessageBox.information(self, '提示',
'创建临时目录失败 %s' % temporary_folder)
return
ssd_usage = psutil.disk_usage(ssd_folder)
hdd_usage = psutil.disk_usage(hdd_folder)
if not is_debug() and ssd_usage.free < 2**30 * 332:
if QMessageBox.information(
self, '提示', '临时目录的空间不够332G，确定要继续吗？',
QMessageBox.Ok | QMessageBox.Cancel) == QMessageBox.Cancel:
return
if not is_debug() and hdd_usage.free < 2**30 * 102:
if QMessageBox.information(
self, '提示', '最终目录的空间不够101G，确定要继续吗？',
QMessageBox.Ok | QMessageBox.Cancel) == QMessageBox.Cancel:
return
self.task.create_time = datetime.now()
self.task.fpk = fpk
self.task.ppk = ppk
self.task.ssd_folder = ssd_folder
self.task.hdd_folder = hdd_folder
self.task.temporary_folder = temporary_folder
self.task.specify_count = specify_count
self.task.count = number
self.task.number_of_thread = thread_num
self.task.memory_size = memory_size
self.task.delay_seconds = delay
if specify_count:
for i in range(number):
self.task.sub_tasks.append(PlotSubTask(self.task, i))
else:
self.task.sub_tasks.append(PlotSubTask(self.task, 0))
super().accept()