def tmp_dir_report(jobs,
dir_cfg,
sched_cfg,
width,
start_row=None,
end_row=None,
prefix=''):
'''start_row, end_row let you split the table up if you want'''
tab = tt.Texttable()
headings = ['tmp', 'ready', 'phases']
tab.header(headings)
tab.set_cols_dtype('t' * len(headings))
tab.set_cols_align('r' * (len(headings) - 1) + 'l')
for i, d in enumerate(sorted(dir_cfg.tmp)):
if (start_row and i < start_row) or (end_row and i >= end_row):
continue
phases = sorted(job.job_phases_for_tmpdir(d, jobs))
ready = manager.phases_permit_new_job(phases, d, sched_cfg, dir_cfg)
row = [
abbr_path(d, prefix), 'OK' if ready else '--',
phases_str(phases, 5)
]
tab.add_row(row)
tab.set_max_width(width)
tab.set_deco(tt.Texttable.BORDER | tt.Texttable.HEADER)
tab.set_deco(0) # No borders
return tab.draw()
def tmp_dir_report(jobs: typing.List[job.Job],
dir_cfg: configuration.Directories,
sched_cfg: configuration.Scheduling,
width: int,
start_row: typing.Optional[int] = None,
end_row: typing.Optional[int] = None,
prefix: str = '') -> str:
'''start_row, end_row let you split the table up if you want'''
tab = tt.Texttable()
headings = ['tmp', 'ready', 'phases']
tab.header(headings)
tab.set_cols_dtype('t' * len(headings))
tab.set_cols_align('r' * (len(headings) - 1) + 'l')
for i, d in enumerate(sorted(dir_cfg.tmp)):
if (start_row and i < start_row) or (end_row and i >= end_row):
continue
phases = sorted(job.job_phases_for_tmpdir(d, jobs))
ready = manager.phases_permit_new_job(phases, d, sched_cfg, dir_cfg)
row = [
abbr_path(d, prefix), 'OK' if ready else '--',
phases_str(phases, 5)
]
tab.add_row(row)
tab.set_max_width(width)
tab.set_deco(tt.Texttable.BORDER | tt.Texttable.HEADER)
tab.set_deco(0) # No borders
return tab.draw() # type: ignore[no-any-return]
def maybe_start_new_plot(dir_cfg, sched_cfg, plotting_cfg):
if psutil.cpu_percent(interval=10) > 90:
return (False, 'cpu busy')
jobs = job.Job.get_running_jobs(dir_cfg.log)
wait_reason = None # If we don't start a job this iteration, this says why.
youngest_job_age = min(
jobs, key=job.Job.get_time_wall).get_time_wall() if jobs else MAX_AGE
global_stagger = int(sched_cfg.global_stagger_m * MIN)
if (youngest_job_age < global_stagger):
wait_reason = 'stagger (%ds/%ds)' % (youngest_job_age, global_stagger)
elif len(jobs) >= sched_cfg.global_max_jobs:
wait_reason = 'max jobs (%d)' % sched_cfg.global_max_jobs
else:
tmp_to_all_phases = [(d, job.job_phases_for_tmpdir(d, jobs))
for d in dir_cfg.tmp]
eligible = [(d, phases) for (d, phases) in tmp_to_all_phases
if phases_permit_new_job(phases, d, sched_cfg, dir_cfg)]
rankable = [(d, phases[0]) if phases else (d, (999, 999))
for (d, phases) in eligible]
if not eligible:
wait_reason = 'no eligible tempdirs'
else:
# Plot to oldest tmpdir.
tmpdir = max(rankable, key=operator.itemgetter(1))[0]
# Select the dst dir least recently selected
dir2ph = {
d: ph
for (d, ph) in dstdirs_to_youngest_phase(jobs).items()
if d in dir_cfg.dst
}
unused_dirs = [d for d in dir_cfg.dst if d not in dir2ph.keys()]
dstdir = ''
if unused_dirs:
dstdir = random.choice(unused_dirs)
else:
dstdir = max(dir2ph, key=dir2ph.get)
logfile = os.path.join(
dir_cfg.log,
datetime.now().strftime('%Y-%m-%d-%H:%M:%S.log'))
plot_args = [
'chia', 'plots', 'create', '-k',
str(plotting_cfg.k), '-r',
str(plotting_cfg.n_threads), '-u',
str(plotting_cfg.n_buckets), '-b',
str(plotting_cfg.job_buffer), '-t', tmpdir, '-d', tmpdir
]
if plotting_cfg.e:
plot_args.append('-e')
if plotting_cfg.farmer_pk is not None:
plot_args.append('-f')
plot_args.append(plotting_cfg.farmer_pk)
if plotting_cfg.pool_pk is not None:
plot_args.append('-p')
plot_args.append(plotting_cfg.pool_pk)
if dir_cfg.tmp2 is not None:
plot_args.append('-2')
plot_args.append(dir_cfg.tmp2)
logmsg = ('Starting plot job: %s ; logging to %s' %
(' '.join(plot_args), logfile))
# start_new_sessions to make the job independent of this controlling tty.
p = subprocess.Popen(plot_args,
stdout=open(logfile, 'w'),
stderr=subprocess.STDOUT,
start_new_session=True)
psutil.Process(p.pid).nice(15)
return (True, logmsg)
return (False, wait_reason)
def maybe_start_new_plot(
dir_cfg: plotman.configuration.Directories,
sched_cfg: plotman.configuration.Scheduling,
plotting_cfg: plotman.configuration.Plotting,
log_cfg: plotman.configuration.Logging) -> typing.Tuple[bool, str]:
jobs = job.Job.get_running_jobs(log_cfg.plots)
wait_reason = None # If we don't start a job this iteration, this says why.
youngest_job_age = min(
jobs, key=job.Job.get_time_wall).get_time_wall() if jobs else MAX_AGE
global_stagger = int(sched_cfg.global_stagger_m * MIN)
if (youngest_job_age < global_stagger):
wait_reason = 'stagger (%ds/%ds)' % (youngest_job_age, global_stagger)
elif len(jobs) >= sched_cfg.global_max_jobs:
wait_reason = 'max jobs (%d) - (%ds/%ds)' % (
sched_cfg.global_max_jobs, youngest_job_age, global_stagger)
else:
tmp_to_all_phases = [(d, job.job_phases_for_tmpdir(d, jobs))
for d in dir_cfg.tmp]
eligible = [(d, phases) for (d, phases) in tmp_to_all_phases
if phases_permit_new_job(phases, d, sched_cfg, dir_cfg)]
rankable = [(d, phases[0]) if phases else (d, job.Phase(known=False))
for (d, phases) in eligible]
if not eligible:
wait_reason = 'no eligible tempdirs (%ds/%ds)' % (youngest_job_age,
global_stagger)
else:
# Plot to oldest tmpdir.
tmpdir = max(rankable, key=operator.itemgetter(1))[0]
dst_dirs = dir_cfg.get_dst_directories()
dstdir: str
if dir_cfg.dst_is_tmp2():
dstdir = dir_cfg.tmp2 # type: ignore[assignment]
elif tmpdir in dst_dirs:
dstdir = tmpdir
elif dir_cfg.dst_is_tmp():
dstdir = tmpdir
else:
# Select the dst dir least recently selected
dir2ph = {
d: ph
for (d, ph) in dstdirs_to_youngest_phase(jobs).items()
if d in dst_dirs and ph is not None
}
unused_dirs = [d for d in dst_dirs if d not in dir2ph.keys()]
dstdir = ''
if unused_dirs:
dstdir = random.choice(unused_dirs)
else:
def key(key: str) -> job.Phase:
return dir2ph[key]
dstdir = max(dir2ph, key=key)
log_file_path = log_cfg.create_plot_log_path(time=pendulum.now())
plot_args: typing.List[str]
if plotting_cfg.type == "madmax":
if plotting_cfg.madmax is None:
raise Exception(
"madmax plotter selected but not configured, report this as a plotman bug",
)
plot_args = [
plotting_cfg.madmax.executable, '-n',
str(1), '-r',
str(plotting_cfg.madmax.n_threads), '-u',
str(plotting_cfg.madmax.n_buckets), '-t',
tmpdir if tmpdir.endswith('/') else (tmpdir + '/'), '-d',
dstdir if dstdir.endswith('/') else (dstdir + '/')
]
if dir_cfg.tmp2 is not None:
plot_args.append('-2')
plot_args.append(dir_cfg.tmp2 if dir_cfg.tmp2.
endswith('/') else (dir_cfg.tmp2 + '/'))
if plotting_cfg.madmax.n_buckets3 is not None:
plot_args.append('-v')
plot_args.append(str(plotting_cfg.madmax.n_buckets3))
if plotting_cfg.madmax.n_rmulti2 is not None:
plot_args.append('-K')
plot_args.append(str(plotting_cfg.madmax.n_rmulti2))
else:
if plotting_cfg.chia is None:
raise Exception(
"chia plotter selected but not configured, report this as a plotman bug",
)
plot_args = [
plotting_cfg.chia.executable, 'plots', 'create', '-k',
str(plotting_cfg.chia.k), '-r',
str(plotting_cfg.chia.n_threads), '-u',
str(plotting_cfg.chia.n_buckets), '-b',
str(plotting_cfg.chia.job_buffer), '-t', tmpdir, '-d',
dstdir
]
if plotting_cfg.chia.e:
plot_args.append('-e')
if plotting_cfg.chia.x:
plot_args.append('-x')
if dir_cfg.tmp2 is not None:
plot_args.append('-2')
plot_args.append(dir_cfg.tmp2)
if plotting_cfg.farmer_pk is not None:
plot_args.append('-f')
plot_args.append(plotting_cfg.farmer_pk)
if plotting_cfg.pool_pk is not None:
plot_args.append('-p')
plot_args.append(plotting_cfg.pool_pk)
if plotting_cfg.pool_contract_address is not None:
plot_args.append('-c')
plot_args.append(plotting_cfg.pool_contract_address)
logmsg = ('Starting plot job: %s ; logging to %s' %
(' '.join(plot_args), log_file_path))
# TODO: CAMPid 09840103109429840981397487498131
try:
open_log_file = open(log_file_path, 'x')
except FileExistsError:
# The desired log file name already exists. Most likely another
# plotman process already launched a new process in response to
# the same scenario that triggered us. Let's at least not
# confuse things further by having two plotting processes
# logging to the same file. If we really should launch another
# plotting process, we'll get it at the next check cycle anyways.
message = (
f'Plot log file already exists, skipping attempt to start a'
f' new plot: {log_file_path!r}')
return (False, logmsg)
except FileNotFoundError as e:
message = (
f'Unable to open log file. Verify that the directory exists'
f' and has proper write permissions: {log_file_path!r}')
raise Exception(message) from e
# Preferably, do not add any code between the try block above
# and the with block below. IOW, this space intentionally left
# blank... As is, this provides a good chance that our handle
# of the log file will get closed explicitly while still
# allowing handling of just the log file opening error.
if sys.platform == 'win32':
creationflags = subprocess.CREATE_NO_WINDOW
nice = psutil.BELOW_NORMAL_PRIORITY_CLASS
else:
creationflags = 0
nice = 15
with open_log_file:
# start_new_sessions to make the job independent of this controlling tty (POSIX only).
# subprocess.CREATE_NO_WINDOW to make the process independent of this controlling tty and have no console window on Windows.
p = subprocess.Popen(plot_args,
stdout=open_log_file,
stderr=subprocess.STDOUT,
start_new_session=True,
creationflags=creationflags)
psutil.Process(p.pid).nice(nice)
return (True, logmsg)
return (False, wait_reason)
def maybe_start_new_plot(dir_cfg, sched_cfg, plotting_cfg):
jobs = job.Job.get_running_jobs(dir_cfg.log)
wait_reason = None # If we don't start a job this iteration, this says why.
youngest_job_age = min(
jobs, key=job.Job.get_time_wall).get_time_wall() if jobs else MAX_AGE
global_stagger = int(sched_cfg.global_stagger_m * MIN)
if (youngest_job_age < global_stagger):
wait_reason = 'stagger (%ds/%ds)' % (youngest_job_age, global_stagger)
elif len(jobs) >= sched_cfg.global_max_jobs:
wait_reason = 'max jobs (%d) - (%ds/%ds)' % (
sched_cfg.global_max_jobs, youngest_job_age, global_stagger)
else:
tmp_to_all_phases = [(d, job.job_phases_for_tmpdir(d, jobs))
for d in dir_cfg.tmp]
eligible = [(d, phases) for (d, phases) in tmp_to_all_phases
if phases_permit_new_job(phases, d, sched_cfg, dir_cfg)]
rankable = [(d, phases[0]) if phases else (d, job.Phase(known=False))
for (d, phases) in eligible]
if not eligible:
wait_reason = 'no eligible tempdirs (%ds/%ds)' % (youngest_job_age,
global_stagger)
else:
# Plot to oldest tmpdir.
tmpdir = max(rankable, key=operator.itemgetter(1))[0]
# Select the dst dir least recently selected
dir2ph = {
d: ph
for (d, ph) in dstdirs_to_youngest_phase(jobs).items()
if d in dir_cfg.dst and ph is not None
}
unused_dirs = [d for d in dir_cfg.dst if d not in dir2ph.keys()]
dstdir = ''
if unused_dirs:
dstdir = random.choice(unused_dirs)
else:
dstdir = max(dir2ph, key=dir2ph.get)
logfile = os.path.join(
dir_cfg.log,
pendulum.now().isoformat(timespec='microseconds').replace(
':', '_') + '.log')
plot_args = [
'chia', 'plots', 'create', '-k',
str(plotting_cfg.k), '-r',
str(plotting_cfg.n_threads), '-u',
str(plotting_cfg.n_buckets), '-b',
str(plotting_cfg.job_buffer), '-t', tmpdir, '-d', dstdir
]
if plotting_cfg.e:
plot_args.append('-e')
if plotting_cfg.farmer_pk is not None:
plot_args.append('-f')
plot_args.append(plotting_cfg.farmer_pk)
if plotting_cfg.pool_pk is not None:
plot_args.append('-p')
plot_args.append(plotting_cfg.pool_pk)
if dir_cfg.tmp2 is not None:
plot_args.append('-2')
plot_args.append(dir_cfg.tmp2)
logmsg = ('Starting plot job: %s ; logging to %s' %
(' '.join(plot_args), logfile))
try:
open_log_file = open(logfile, 'x')
except FileExistsError:
# The desired log file name already exists. Most likely another
# plotman process already launched a new process in response to
# the same scenario that triggered us. Let's at least not
# confuse things further by having two plotting processes
# logging to the same file. If we really should launch another
# plotting process, we'll get it at the next check cycle anyways.
message = (
f'Plot log file already exists, skipping attempt to start a'
f' new plot: {logfile!r}')
return (False, logmsg)
except FileNotFoundError as e:
message = (
f'Unable to open log file. Verify that the directory exists'
f' and has proper write permissions: {logfile!r}')
raise Exception(message) from e
# Preferably, do not add any code between the try block above
# and the with block below. IOW, this space intentionally left
# blank... As is, this provides a good chance that our handle
# of the log file will get closed explicitly while still
# allowing handling of just the log file opening error.
with open_log_file:
# start_new_sessions to make the job independent of this controlling tty.
p = subprocess.Popen(plot_args,
stdout=open_log_file,
stderr=subprocess.STDOUT,
start_new_session=True)
psutil.Process(p.pid).nice(15)
return (True, logmsg)
return (False, wait_reason)
def maybe_start_new_plot(dir_cfg, sched_cfg, plotting_cfg):
jobs = job.Job.get_running_jobs(dir_cfg.log)
wait_reason = None # If we don't start a job this iteration, this says why.
youngest_job_age = min(
jobs, key=job.Job.get_time_wall).get_time_wall() if jobs else MAX_AGE
global_stagger = int(sched_cfg.global_stagger_m * MIN)
if (youngest_job_age < global_stagger):
wait_reason = 'stagger (%ds/%ds)' % (youngest_job_age, global_stagger)
elif len(jobs) >= sched_cfg.global_max_jobs:
wait_reason = 'max jobs (%d) - (%ds/%ds)' % (
sched_cfg.global_max_jobs, youngest_job_age, global_stagger)
else:
tmp_to_all_phases = [(d, job.job_phases_for_tmpdir(d, jobs))
for d in dir_cfg.tmp]
eligible = [(d, phases) for (d, phases) in tmp_to_all_phases
if phases_permit_new_job(phases, d, sched_cfg, dir_cfg)]
rankable = [(d, phases[0]) if phases else (d, job.Phase(known=False))
for (d, phases) in eligible]
if not eligible:
wait_reason = 'no eligible tempdirs (%ds/%ds)' % (youngest_job_age,
global_stagger)
else:
# Plot to oldest tmpdir.
tmpdir = max(rankable, key=operator.itemgetter(1))[0]
# 若 tmp 与 dst 目录项数一项，使用相同下标项
if len(dir_cfg.tmp) == len(dir_cfg.dst):
idx = dir_cfg.tmp.index(tmpdir)
dstdir = dir_cfg.dst[idx]
else:
# Select the dst dir least recently selected
dir2ph = {
d: ph
for (d, ph) in dstdirs_to_youngest_phase(jobs).items()
if d in dir_cfg.dst and ph is not None
}
unused_dirs = [
d for d in dir_cfg.dst if d not in dir2ph.keys()
]
dstdir = ''
if unused_dirs:
dstdir = random.choice(unused_dirs)
else:
dstdir = max(dir2ph, key=dir2ph.get)
logfile = os.path.join(
dir_cfg.log,
pendulum.now().isoformat(timespec='microseconds').replace(
':', '_') + '.log')
plot_args = [
'chia',
'plots',
'create',
# '--override-k',
'-k',
str(plotting_cfg.k),
'-r',
str(plotting_cfg.n_threads),
'-u',
str(plotting_cfg.n_buckets),
'-b',
str(plotting_cfg.job_buffer),
'-t',
tmpdir,
'-d',
dstdir
]
if plotting_cfg.e:
plot_args.append('-e')
if plotting_cfg.farmer_pk is not None:
plot_args.append('-f')
plot_args.append(plotting_cfg.farmer_pk)
if plotting_cfg.pool_pk is not None:
plot_args.append('-p')
plot_args.append(plotting_cfg.pool_pk)
if dir_cfg.tmp2 is not None:
plot_args.append('-2')
plot_args.append(dir_cfg.tmp2)
logmsg = ('Starting plot job: %s ; logging to %s' %
(' '.join(plot_args), logfile))
try:
open_log_file = open(logfile, 'x')
except FileExistsError:
# The desired log file name already exists. Most likely another
# plotman process already launched a new process in response to
# the same scenario that triggered us. Let's at least not
# confuse things further by having two plotting processes
# logging to the same file. If we really should launch another
# plotting process, we'll get it at the next check cycle anyways.
message = (
f'Plot log file already exists, skipping attempt to start a'
f' new plot: {logfile!r}')
return (False, logmsg)
except FileNotFoundError as e:
message = (
f'Unable to open log file. Verify that the directory exists'
f' and has proper write permissions: {logfile!r}')
raise Exception(message) from e
# Preferably, do not add any code between the try block above
# and the with block below. IOW, this space intentionally left
# blank... As is, this provides a good chance that our handle
# of the log file will get closed explicitly while still
# allowing handling of just the log file opening error.
with open_log_file:
# start_new_sessions to make the job independent of this controlling tty.
p = subprocess.Popen(plot_args,
stdout=open_log_file,
stderr=subprocess.STDOUT,
start_new_session=True)
psutil.Process(p.pid).nice(15)
cpu_count = psutil.cpu_count()
threads = plotting_cfg.n_threads
# 尝试绑定CPU，当线程数为2或4，且最大任务数不大于核数（1/threads）时
if (2 == threads or 4 == threads
) and cpu_count / threads >= sched_cfg.global_max_jobs:
# 计算CPU mask
cpu_mask = []
cpu_used = []
cpu_unused = []
while cpu_count > 0:
cpu_mask.append(len(cpu_mask))
cpu_used.append(-1)
cpu_count -= 1
cpu_unused = cpu_mask[:]
# 统计已绑定的CPU
for j in jobs:
if len(j.cpu_affinity) != len(cpu_mask):
for c in j.cpu_affinity:
cpu_used[c] = c
cpu_unused[c] = -1
logmsg = logmsg + ("\r\n {cpus:%d" %
len(cpu_mask)) + ', unused:[' + ','.join(
'%s' % item for item in cpu_unused)
# 尝试绑定到剩余CPU（两个核）
i = 0
while i < len(cpu_unused):
if cpu_unused[i] >= 0:
# 这两个核能用，绑定
cpu_used = cpu_unused[i:i + threads]
os.sched_setaffinity(p.pid, cpu_used)
logmsg = logmsg + ("], pid: %d, affinity:[" %
(p.pid)) + ','.join(
'%s' % item
for item in cpu_used)
break
i += threads
logmsg = logmsg + ']}'
return (True, logmsg)
return (False, wait_reason)