def run(self):
start_time = time.time()
self.__set_proxy_pool()
all_items = Downloader.pipeline(self.__get_all_coursewares, self.__get_real_coursewares,
self.__get_download_urls)
with ThreadPoolExecutor(max_workers=10) as executor:
all_tasks = {}
for subject in all_items:
print(f'开始处理 {subject} 下的所有课件')
for item in all_items[subject]:
all_tasks[executor.submit(self.download, subject, item.get('title'),
f'http://en.xiejiaxin.com/waiyanban/abShowSoftDown.asp?UrlID=1&SoftID={item.get("code")}')] = item.get(
'title')
for future in as_completed(all_tasks):
url = all_tasks[future]
try:
title = future.result()
if isinstance(title, str):
print('《{c.green}{title}{c.reset}》{c.green}下载成功{c.reset}'.format(title=title, c=cf))
except Exception as e:
print('在处理 {url} 的时候抛出了一个异常，内容为 {c.red}{e}{c.reset}'.format(url=url, e=str(e), c=cf))
print('耗时：', cf.magenta(f'{self.time_diff(start_time, time.time())}'), sep='')
def _(value: BatchTracer,
seen: Set[int],
show_values: bool = True,
indent: int = 0,
batch_dims: Optional[Tuple[Optional[int], ...]] = None) -> str:
return cf.magenta(f"BatchTracer {value.shape} {value.dtype} "
f"batched over {value.val.shape[value.batch_dim]}") + "\n"
def show():
"""
Show the modifiers and colors
"""
# modifiers
sys.stdout.write(colorful.bold('bold') + ' ')
sys.stdout.write(colorful.dimmed('dimmed') + ' ')
sys.stdout.write(colorful.italic('italic') + ' ')
sys.stdout.write(colorful.underlined('underlined') + ' ')
sys.stdout.write(colorful.inversed('inversed') + ' ')
sys.stdout.write(colorful.concealed('concealed') + ' ')
sys.stdout.write(colorful.struckthrough('struckthrough') + '\n')
# foreground colors
sys.stdout.write(colorful.red('red') + ' ')
sys.stdout.write(colorful.green('green') + ' ')
sys.stdout.write(colorful.yellow('yellow') + ' ')
sys.stdout.write(colorful.blue('blue') + ' ')
sys.stdout.write(colorful.magenta('magenta') + ' ')
sys.stdout.write(colorful.cyan('cyan') + ' ')
sys.stdout.write(colorful.white('white') + '\n')
# background colors
sys.stdout.write(colorful.on_red('red') + ' ')
sys.stdout.write(colorful.on_green('green') + ' ')
sys.stdout.write(colorful.on_yellow('yellow') + ' ')
sys.stdout.write(colorful.on_blue('blue') + ' ')
sys.stdout.write(colorful.on_magenta('magenta') + ' ')
sys.stdout.write(colorful.on_cyan('cyan') + ' ')
sys.stdout.write(colorful.on_white('white') + '\n')
def skip_step(step, resource=None, message=None):
if resource is None:
resource = 'any'
if message is None:
message = '{} {} {}'.format(
colorful.orange('Can not find'), colorful.magenta(resource),
colorful.orange('defined in target terraform plan.'))
else:
message = colorful.orange(message)
console_write("\t{}: {}".format(
colorful.bold_purple('SKIPPING'),
message.format(resource=colorful.magenta(resource))))
step.skip()
# Skip all steps in the scenario
for each in step.parent.all_steps:
each.runable = False
def run(self):
start_time = time.time()
subject, books_items = self.__get_books_items(self.page_url)
print('共发现 {c.bold_red_on_black}{count}{c.reset} 份课本，开始创建任务'.format(
count=len(books_items), c=cf))
with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
# 将任务添加到线程池
# 将 future 与 url 映射，便于后续捕获当前任务的 url
all_tasks = {
executor.submit(Downloader.download, subject,
item.get('title'), item.get('url')):
item.get('url')
for item in books_items
}
# 阻塞，让主线程等待所有任务执行完成
# wait(all_tasks, return_when=ALL_COMPLETED)
for future in as_completed(all_tasks):
url = all_tasks[future]
try:
data = future.result()
if isinstance(data, str):
print(
'《{c.green}{title}{c.reset}》{c.green}下载成功{c.reset}'
.format(title=data, c=cf))
except Exception as e:
print(
'在处理 {url} 的时候抛出了一个异常，内容为 {c.red}{e}{c.reset}'.format(
url=url, e=str(e), c=cf))
else:
pass
print('耗时：',
cf.magenta(f'{self.time_diff(start_time, time.time())}'),
sep='')
def test_step_matches_configs(match_config_files,
basedirs,
cover_min_percentage=None,
cover_show_missing=False):
"""
Test if the given match config files matches the actual
matched step implementations.
"""
if cover_min_percentage is not None and float(cover_min_percentage) > 100:
sys.stderr.write(
str(
colorful.magenta(
'You are a little cocky to think you can reach a minimum coverage of {0:.2f}%\n'
.format(float(cover_min_percentage)))))
return 3
# load user's custom python files
for basedir in basedirs:
load_modules(basedir)
steps = StepRegistry().steps
if not steps:
sys.stderr.write(
str(
colorful.magenta(
'No step implementations found in {0}, thus doesn\'t make sense to continue'
.format(basedirs))))
return 4
failed = 0
passed = 0
covered_steps = set()
for match_config_file in match_config_files:
# load the given match config file
with codecs.open(match_config_file, "r", "utf-8") as f:
match_config = yaml.safe_load(f)
if not match_config:
print(
colorful.magenta(
'No sentences found in {0} to test against'.format(
match_config_file)))
return 5
print(
colorful.yellow('Testing sentences from {0}:'.format(
colorful.bold_yellow(match_config_file))))
failed_sentences, passed_senteces = test_step_matches(
match_config, steps)
failed += failed_sentences
passed += passed_senteces
covered_steps = covered_steps.union(x['should_match']
for x in match_config
if 'should_match' in x)
# newline
sys.stdout.write('\n')
report = colorful.bold_white('{0} sentences ('.format(failed + passed))
if passed > 0:
report += colorful.bold_green('{0} passed'.format(passed))
if passed > 0 and failed > 0:
report += colorful.bold_white(', ')
if failed > 0:
report += colorful.bold_red('{0} failed'.format(failed))
report += colorful.bold_white(')')
print(report)
step_coverage = 100.0 / len(steps) * len(covered_steps)
coverage_report = colorful.bold_white(
'Covered {0} of {1} step implementations'.format(
len(covered_steps), len(steps)))
ret = 0 if failed == 0 else 1
if cover_min_percentage:
coverage_color = colorful.bold_green if step_coverage >= float(
cover_min_percentage) else colorful.bold_red
coverage_report += colorful.bold_white(' (coverage: ')
coverage_report += coverage_color('{0:.2f}%'.format(step_coverage))
if float(cover_min_percentage) > step_coverage:
coverage_report += colorful.bold_white(
', expected a minimum of {0}'.format(
colorful.bold_green(cover_min_percentage + '%')))
if failed == 0:
ret = 2
# if tests have passed and coverage is too low we fail with exit code 2
coverage_report += colorful.bold_white(')')
print(coverage_report)
if cover_show_missing:
missing_steps = get_missing_steps(steps, covered_steps)
if missing_steps:
missing_step_report = colorful.bold_yellow('Missing steps:\n')
for step in missing_steps:
missing_step_report += '- {0} at '.format(
colorful.cyan(step[0]))
missing_step_report += colorful.cyan(step[1]) + '\n'
sys.stdout.write(missing_step_report)
return ret
def secondary(string):
return cf.magenta(string)
def test_step_matches_configs(
match_config_files, basedirs, cover_min_percentage=None, cover_show_missing=False
):
"""
Test if the given match config files matches the actual
matched step implementations.
"""
if cover_min_percentage is not None and float(cover_min_percentage) > 100:
sys.stderr.write(
str(
colorful.magenta(
"You are a little cocky to think you can reach a minimum coverage of {0:.2f}%\n".format(
float(cover_min_percentage)
)
)
)
)
return 3
# load user's custom python files
for basedir in basedirs:
load_modules(basedir)
steps = StepRegistry().steps
if not steps:
sys.stderr.write(
str(
colorful.magenta(
"No step implementations found in {0}, thus doesn't make sense to continue".format(
basedirs
)
)
)
)
return 4
failed = 0
passed = 0
covered_steps = set()
for match_config_file in match_config_files:
# load the given match config file
with codecs.open(match_config_file, "r", "utf-8") as f:
match_config = yaml.safe_load(f)
if not match_config:
print(
colorful.magenta(
"No sentences found in {0} to test against".format(
match_config_file
)
)
)
return 5
print(
colorful.yellow(
"Testing sentences from {0}:".format(
colorful.bold_yellow(match_config_file)
)
)
)
failed_sentences, passed_senteces = test_step_matches(match_config, steps)
failed += failed_sentences
passed += passed_senteces
covered_steps = covered_steps.union(
x["should_match"] for x in match_config if "should_match" in x
)
# newline
sys.stdout.write("\n")
report = colorful.bold_white("{0} sentences (".format(failed + passed))
if passed > 0:
report += colorful.bold_green("{0} passed".format(passed))
if passed > 0 and failed > 0:
report += colorful.bold_white(", ")
if failed > 0:
report += colorful.bold_red("{0} failed".format(failed))
report += colorful.bold_white(")")
print(report)
step_coverage = 100.0 / len(steps) * len(covered_steps)
coverage_report = colorful.bold_white(
"Covered {0} of {1} step implementations".format(len(covered_steps), len(steps))
)
ret = 0 if failed == 0 else 1
if cover_min_percentage:
coverage_color = (
colorful.bold_green
if step_coverage >= float(cover_min_percentage)
else colorful.bold_red
)
coverage_report += colorful.bold_white(" (coverage: ")
coverage_report += coverage_color("{0:.2f}%".format(step_coverage))
if float(cover_min_percentage) > step_coverage:
coverage_report += colorful.bold_white(
", expected a minimum of {0}".format(
colorful.bold_green(cover_min_percentage + "%")
)
)
if failed == 0:
ret = 2
# if tests have passed and coverage is too low we fail with exit code 2
coverage_report += colorful.bold_white(")")
print(coverage_report)
if cover_show_missing:
missing_steps = get_missing_steps(steps, covered_steps)
if missing_steps:
missing_step_report = colorful.bold_yellow("Missing steps:\n")
for step in missing_steps:
missing_step_report += "- {0} at ".format(colorful.cyan(step[0]))
missing_step_report += colorful.cyan(step[1]) + "\n"
sys.stdout.write(missing_step_report)
return ret
def doActions(self):
try:
for i in range(0, len(self.config)):
inf_check = False
if len(self.config) == 1 and 'duration' in self.__dict__:
self.config[i]['duration'] = int(self.duration)
if len(self.config) == 1 and 'loops' in self.__dict__:
self.config[i]['loops'] = int(self.loops)
if self.config[i]['action'] == 'off':
printLog(colorful.bold_purple("Turning Pi LED's off..."))
loop_itt = -1
rem, loop_bool = self.loopLogic(loop_itt, self.config[i]['loops'])
while loop_bool == True:
proc = Process(proc_name='python{0}.{1}'.format(str(sys.version_info.major), str(sys.version_info.minor)),
proc_args=['--color', '--rainbow'])
proc.killProcesses()
led.all_off()
if self.config[i]['duration'] != 0:
printLog('Sleeping for ' + colorful.bold_blue(str(self.config[i]['duration'])) + ' seconds.')
time.sleep(self.config[i]['duration'])
if loop_itt == -1:
loop_itt += 2
elif self.config[i]['loops'] > -1:
loop_itt += 1
rem, loop_bool = self.loopLogic(loop_itt, self.config[i]['loops'])
if inf_check == False:
printLog(colorful.bold_orange(rem) + ' loops remaining.')
inf_check = True
elif self.config[i]['action'] == 'color':
loop_itt = -1
rem, loop_bool = self.loopLogic(loop_itt, self.config[i]['loops'])
while loop_bool == True:
if 'args' in self.__dict__:
self.config[i]['args']['color'] = self.args[i]
led.fill(color_hex(self.config[i]['args']['color']))
led.update()
if self.config[i]['duration'] != 0:
printLog('Sleeping for ' + colorful.bold_blue(str(self.config[i]['duration'])) + ' seconds.')
time.sleep(self.config[i]['duration'])
if loop_itt == -1:
loop_itt += 2
elif self.config[i]['loops'] > -1:
loop_itt += 1
rem, loop_bool = self.loopLogic(loop_itt, self.config[i]['loops'])
if inf_check == False:
printLog(colorful.bold_orange(rem) + ' loops remaining.')
inf_check = True
elif self.config[i]['action'] == 'rainbow':
loop_itt = -1
anim = Rainbow(led)
rem, loop_bool = self.loopLogic(loop_itt, self.config[i]['loops'])
while loop_bool == True:
anim.step()
led.update()
if self.config[i]['duration'] != 0:
printLog('Sleeping for ' + colorful.bold_blue(str(self.config[i]['duration'])) + ' seconds.')
time.sleep(self.config[i]['duration'])
if loop_itt == -1:
loop_itt += 2
elif self.config[i]['loops'] > -1:
loop_itt += 1
rem, loop_bool = self.loopLogic(loop_itt, self.config[i]['loops'])
if inf_check == False:
printLog(colorful.bold_orange(rem) + ' loops remaining.')
inf_check = True
if self.config[-1]['action'] != 'off':
led.all_off()
printSuccess('Actions completed.')
except KeyboardInterrupt:
led.all_off()
print(colorful.bold(' ------------->>> ') + colorful.magenta('Interrupt entered.'))
printExit('Actions interrupted.')
except Exception as e:
if str(e)[-1] != '.':
e_str = str(e) + '.'
else:
e_str = str(e)
printExit('Invalid config: ' + e_str)
def style_info(text):
return colorful.magenta(f'{text}', nested=True)
def err(self, text):
if not self.silent:
print(
colorful.magenta(self.name) + " " + colorful.red("err") + " " +
str(text))
def debug(self, text):
if not self.silent and self.isDebug:
print(
colorful.magenta(self.name) + " " + colorful.cyan("debug") +
" " + str(text))
def info(self, text):
if not self.silent:
print(
colorful.magenta(self.name) + " " + colorful.green("info") +
" " + str(text))
def _(value: DynamicJaxprTracer,
seen: Set[int],
show_values: bool = True,
indent: int = 0,
batch_dims: Optional[Tuple[Optional[int], ...]] = None) -> str:
return cf.magenta(f"DynamicJaxprTracer {value.shape} {value.dtype}") + "\n"
def warn(self, text):
if not self.silent:
print(
colorful.magenta(self.name) + " " + colorful.yellow("warn") +
" " + str(text))
# combine styles with strings
print(cf.bold & cf.red | 'Hello World')
# use true colors
cf.use_true_colors()
# extend default color palette
cf.update_palette({'mint': '#c5e8c8'})
print(cf.mint_on_snow('Wow, this is actually mint'))
# choose a predefined style
cf.use_style('solarized')
# print the official solarized colors
print(cf.yellow('yellow'), cf.orange('orange'),
cf.red('red'), cf.magenta('magenta'),
cf.violet('violet'), cf.blue('blue'),
cf.cyan('cyan'), cf.green('green'))
# directly print with colors
cf.print('{c.bold_blue}Hello World{c.reset}')
# choose specific color mode for one block
with cf.with_8_ansi_colors() as c:
print(c.bold_green('colorful is awesome!'))
# create and choose your own color palette
MY_COMPANY_PALETTE = {
'companyOrange': '#f4b942',
'companyBaige': '#e8dcc5'
}
def debug(self, text):
print(
colorful.magenta(self.name) + " " + colorful.cyan("debug") + " " +
str(text))
