def save(self, article=None):
""" Exceptions:
- AssertionError
"""
assert article is not None, "article is not defined."
# monitor_conn = pymongo.MongoClient("mongodb://mongo:27017/monitor")
# monitor_db = monitor_conn["monitor"]
monitor = Monitor()
conn = pymongo.MongoClient("mongodb://mongo:27017/blog_crawler")
db = conn["blog_crawler"]
# Ensuring index
db.data.create_index([("permalink", pymongo.ASCENDING)], unique=True, background=True)
db.data.create_index([("converted", pymongo.ASCENDING)], background=True)
db.data.create_index("TTL",expireAfterSeconds=2592000, background=True)
try:
db.data.insert_one(article)
monitor.capture_insert_document(article["_crawled_by"])
# monitor_db.status.update(
# {"crawler_name": re.compile(article["_crawled_by"], re.IGNORECASE)},
# {"$set":{
# "crawler_name": article["_crawled_by"].title(),
# "last_insert_time": arrow.utcnow().datetime
# }},
# upsert=True
# )
print(fmtstr("[BlogSaver][success] Inserted One Document!"))
except pymongo.errors.DuplicateKeyError:
print(fmtstr("[BlogSaver][error] Duplicate Document!","red"))
finally:
conn.close()
# monitor_conn.close()
def render_into_rect(self, w_fsa):
width = w_fsa.width
height = w_fsa.height
wrapped = wrap(u"Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.",
width, height, text_fmt=lambda x: fmtstr(x, fg='red'))
w_fsa[0:height, 0:width] = wrapped
def paint_history(rows, columns, display_lines):
lines = []
for r, line in zip(range(rows), display_lines[-rows:]):
lines.append(fmtstr(line[:columns]))
r = fsarray(lines, width=columns)
assert r.shape[0] <= rows, repr(r.shape) + ' ' + repr(rows)
assert r.shape[1] <= columns, repr(r.shape) + ' ' + repr(columns)
return r
def formatted_docstring(docstring, columns, config):
if isinstance(docstring, bytes):
docstring = docstring.decode('utf8')
elif isinstance(docstring, str if py3 else unicode):
pass
else:
# TODO: fail properly here and catch possible exceptions in callers.
return []
color = func_for_letter(config.color_scheme['comment'])
return sum(([color(x) for x in (display_linize(line, columns) if line else
fmtstr(''))]
for line in docstring.split('\n')), [])
def fullscreen_winch_with_input():
print('this should be just off-screen')
w = FullscreenWindow(sys.stdout)
def sigwinch_handler(signum, frame):
print('sigwinch! Changed from %r to %r' % ((rows, columns), (w.height, w.width)))
signal.signal(signal.SIGWINCH, sigwinch_handler)
with w:
with Cbreak(sys.stdin):
for e in input.Input():
rows, columns = w.height, w.width
a = [fmtstr((('.%sx%s.%r.' % (rows, columns, e)) * rows)[:columns]) for row in range(rows)]
w.render_to_terminal(a)
def save(self, article=None):
""" Exceptions:
- AssertionError
"""
assert article is not None, "article is not defined."
monitor = Monitor()
conn = pymongo.MongoClient("mongodb://mongo:27017/news_crawler")
db = conn["news_crawler"]
# Ensuring index
db.data.create_index([("permalink", pymongo.ASCENDING)], unique=True, background=True)
db.data.create_index([("converted", pymongo.ASCENDING)], background=True)
db.data.create_index("TTL",expireAfterSeconds=2592000, background=True)
try:
db.data.insert_one(article)
monitor.capture_insert_document(article["_crawled_by"])
print(fmtstr("[ArticleSaver][success] Inserted One Document!"))
except pymongo.errors.DuplicateKeyError:
print(fmtstr("[ArticleSaver][error] Duplicate Document!","red"))
finally:
conn.close()
def template_output(pkg):
""" Creates a output string from a package
:parmans pkg: dict like object describing a package
:returns string
"""
template = ("{repo}/{arch}/{pkgname} {epoch}:{pkgver}-{pkgrel}{ood}\n"
" Updated: {last_update} Built: {build_date}")
data = {}
data["repo"] = curtsies.fmtstr(pkg["repo"], fg="magenta", bold=True)
data["arch"] = curtsies.fmtstr(pkg["arch"], fg="yellow", bold=True)
data["pkgname"] = curtsies.fmtstr(pkg["pkgname"], fg="green", bold=True)
if pkg["flag_date"]:
ver_colour = "red"
data["ood"] = curtsies.fmtstr(" <!>", fg=ver_colour)
else:
ver_colour = "green"
data["ood"] = ""
for itm in ("epoch", "pkgver", "pkgrel"):
# fmtstr doesn't like ints
data[itm] = curtsies.fmtstr(str(pkg[itm]), fg=ver_colour, bold=True)
data["last_update"] = pkg["last_update"]
data["build_date"] = pkg["build_date"]
return template.format(**data)
def formatted_docstring(docstring, columns, config):
if isinstance(docstring, bytes):
docstring = docstring.decode("utf8")
elif isinstance(docstring, str):
pass
else:
# TODO: fail properly here and catch possible exceptions in callers.
return []
color = func_for_letter(config.color_scheme["comment"])
return sum(
([
color(x)
for x in (display_linize(line, columns) if line else fmtstr(""))
] for line in docstring.split("\n")),
[],
)
def multiple_convert_and_save(self):
documents = Data.get_not_converted()
mention_db = MentionDB()
author_info_db = AuthorInfoDB()
for document in documents:
try:
converted_document = ConverterEngine.convert(
document).to_dict()
mention_db.mention = converted_document
mention_db.save()
author_info_db.generate_info(converted_document)
author_info_db.save()
except pymongo.errors.DuplicateKeyError:
print(fmtstr("[convert][error] Duplicate Document!", "red"))
finally:
mention_db.set_as_converted(source_db=Database.get_db())
def display_linize(msg, columns, blank_line=False):
"""Returns lines obtained by splitting msg over multiple lines.
Warning: if msg is empty, returns an empty list of lines"""
if not msg:
return [""] if blank_line else []
msg = fmtstr(msg)
try:
display_lines = list(msg.width_aware_splitlines(columns))
# use old method if wcwidth can't determine width of msg
except ValueError:
display_lines = [
msg[start:end] for start, end in zip(
range(0, len(msg), columns),
range(columns,
len(msg) + columns, columns),
)
]
return display_lines
def cursor_winch():
global rows, columns
print('this should be just off-screen')
w = CursorAwareWindow(sys.stdout, sys.stdin, keep_last_line=True, hide_cursor=False)
def sigwinch_handler(signum, frame):
global rows, columns
dy = w.get_cursor_vertical_diff()
old_rows, old_columns = rows, columns
rows, columns = w.height, w.width
print('sigwinch! Changed from %r to %r' % ((old_rows, old_columns), (rows, columns)))
print('cursor moved %d lines down' % dy)
w.write(w.t.move_up)
w.write(w.t.move_up)
signal.signal(signal.SIGWINCH, sigwinch_handler)
with w:
for e in input.Input():
rows, columns = w.height, w.width
a = [fmtstr((('.%sx%s.' % (rows, columns)) * rows)[:columns]) for row in range(rows)]
w.render_to_terminal(a)
def matches_lines(rows, columns, matches, current, config, format):
highlight_color = func_for_letter(config.color_scheme['operator'].lower())
if not matches:
return []
color = func_for_letter(config.color_scheme['main'])
max_match_width = max(len(m) for m in matches)
words_wide = max(1, (columns - 1) // (max_match_width + 1))
matches = [format(m) for m in matches]
if current:
current = format(current)
matches_lines = [fmtstr(' ').join(color(m.ljust(max_match_width))
if m != current
else highlight_color(m.ljust(max_match_width))
for m in matches[i:i+words_wide])
for i in range(0, len(matches), words_wide)]
logger.debug('match: %r' % current)
logger.debug('matches_lines: %r' % matches_lines)
return matches_lines
def array_size_test(window):
"""Tests arrays one row too small or too large"""
with window as w:
print(
'a displays a screen worth of input, s one line less, and d one line more'
)
with input.Input(sys.stdin) as input_generator:
while True:
c = input_generator.next()
rows, columns = w.height, w.width
if c == "":
sys.exit() # same as raise SystemExit()
elif c == "h":
a = w.array_from_text("a for small array")
elif c == "a":
a = [fmtstr(c * columns) for _ in range(rows)]
elif c == "s":
a = [fmtstr(c * columns) for _ in range(rows - 1)]
elif c == "d":
a = [fmtstr(c * columns) for _ in range(rows + 1)]
elif c == "f":
a = [fmtstr(c * columns) for _ in range(rows - 2)]
elif c == "q":
a = [fmtstr(c * columns) for _ in range(1)]
elif c == "w":
a = [fmtstr(c * columns) for _ in range(1)]
elif c == "e":
a = [fmtstr(c * columns) for _ in range(1)]
elif c == "c":
w.write(w.t.move(w.t.height - 1, 0))
w.scroll_down()
elif isinstance(c, events.WindowChangeEvent):
a = w.array_from_text(
"window just changed to %d rows and %d columns" %
(c.rows, c.columns))
elif c == u'<ESC>': # allows exit without keyboard interrupt
break
elif c == '\x0c': # ctrl-L
[w.write('\n') for _ in range(rows)]
continue
else:
a = w.array_from_text("unknown command")
w.render_to_terminal(a)
def tick(self):
""" Do one frame of work. Returns the winner if there
is a crash
"""
# list of players alive this frame
temp_players = self.players[:]
for bike in self.players:
bike.move(self.grid)
if self.grid[bike.y, bike.x] == [" "]:
self.grid[bike.y, bike.x] = bike.appearance
else: # crashed
temp_players.remove(bike)
self.grid[bike.y, bike.x] = fmtstr("X", "red", "bold",
"on_black")
if len(temp_players) == 0:
return "tie"
elif len(temp_players) < len(self.players):
return temp_players
def matches_lines(rows, columns, matches, current, config, format):
highlight_color = func_for_letter(config.color_scheme['operator'].lower())
if not matches:
return []
color = func_for_letter(config.color_scheme['main'])
max_match_width = max(len(m) for m in matches)
words_wide = max(1, (columns - 1) // (max_match_width + 1))
matches = [format(m) for m in matches]
if current:
current = format(current)
matches_lines = [fmtstr(' ').join(color(m.ljust(max_match_width))
if m != current
else highlight_color(m.ljust(max_match_width))
for m in matches[i:i+words_wide])
for i in range(0, len(matches), words_wide)]
logger.debug('match: %r' % current)
logger.debug('matches_lines: %r' % matches_lines)
return matches_lines
def _string_to_fsarray(self, msg):
if not self._inside_window_context:
raise RuntimeError(
'Calling _string_to_fsarray outside of window context')
rows, columns = self._window_context.get_term_hw()
msg = fmtstr(msg)
arr = FSArray(0, columns)
i = 0
lines = msg.split('\n') if '\n' in str(msg) else [msg]
for line in lines:
for j in xrange(len(line)):
c = line[j]
if i >= rows * columns:
return arr
else:
arr[i // arr.width, i % arr.width] = [c]
i += 1
i = ((i // columns) + 1) * columns
if len(arr) == 0:
return fsarray([''])
return arr
def fullscreen_winch_with_input():
"""
Monitors user input as well as screen size and acknowledges changes to both.
"""
print('this should be just off-screen')
w = FullscreenWindow(sys.stdout)
def sigwinch_handler(signum, frame):
print('sigwinch! Changed from {!r} to {!r}'.format(
(rows, columns), (w.height, w.width)))
signal.signal(signal.SIGWINCH, sigwinch_handler)
with w:
with Cbreak(sys.stdin):
for e in input.Input():
rows, columns = w.height, w.width
a = [
fmtstr(((f'.{rows}x{columns}.{e!r}.') * rows)[:columns])
for row in range(rows)
]
w.render_to_terminal(a)
if e == '<ESC>':
break
def array_size_test(window):
"""Tests arrays one row to small or too large"""
with window as w:
print('a displays a screen worth of input, s one line less, and d one line more')
with input.Input(sys.stdin) as input_generator:
while True:
c = input_generator.next()
rows, columns = w.height, w.width
if c == "":
sys.exit() # same as raise SystemExit()
elif c == "h":
a = w.array_from_text("a for small array")
elif c == "a":
a = [fmtstr(c*columns) for _ in range(rows)]
elif c == "s":
a = [fmtstr(c*columns) for _ in range(rows-1)]
elif c == "d":
a = [fmtstr(c*columns) for _ in range(rows+1)]
elif c == "f":
a = [fmtstr(c*columns) for _ in range(rows-2)]
elif c == "q":
a = [fmtstr(c*columns) for _ in range(1)]
elif c == "w":
a = [fmtstr(c*columns) for _ in range(1)]
elif c == "e":
a = [fmtstr(c*columns) for _ in range(1)]
elif c == "c":
w.write(w.t.move(w.t.height-1, 0))
w.scroll_down()
elif isinstance(c, events.WindowChangeEvent):
a = w.array_from_text("window just changed to %d rows and %d columns" % (c.rows, c.columns))
elif c == '\x0c': # ctrl-L
[w.write('\n') for _ in range(rows)]
continue
else:
a = w.array_from_text("unknown command")
w.render_to_terminal(a)
from lib.engine.converter import ConverterEngine
from curtsies import fmtstr
import time
import random
if __name__ == "__main__":
while True:
engine = ConverterEngine()
engine.convert()
rnd = random.randint(60,500)
print(fmtstr("Sleeping %s seconds" % rnd,"yellow"))
time.sleep(rnd)
def format_str(self, s):
return fmtstr(s, **self.kwargs)
monitor = Monitor()
while True:
run_config = ConfigFactory.get_config(ConfigFactory.RUN)
crawler_names = run_config.get("run")
section_name = run_config.get("section")
# monitor.capture_section_start(name=section_name.title())
for crawler_name in crawler_names:
# monitor.capture_crawler_start(name=crawler_name.title())
name = crawler_name.replace(" ","_")
name = name.lower()
crawlers = []
print("[run][debug] Preparing %s" % name)
for file_name in glob.iglob(os.path.join(os.getcwd(),"crawlers","%s_*.py" % name)):
file_name = file_name.replace(os.getcwd(),"")
file_name = file_name.replace("crawlers","")
file_name = file_name.replace(".py","")
file_name = file_name.replace("/","")
crawlers.append(file_name)
print("[run][debug] Crawling...")
run_config = ConfigFactory.get_config(ConfigFactory.RUN)
workers = run_config.get("workers")
with Pool(workers) as pool:
pool.map(execute_worker, crawlers)
# monitor.capture_crawler_stop(name=crawler_name.title())
# monitor.capture_section_stop(name=section_name.title())
rnd = random.randint(60,360)
print(fmtstr("[run][debug] Sleeping %s seconds" % rnd, "yellow"))
time.sleep(rnd)
def formatted_docstring(docstring, columns, config):
color = func_for_letter(config.color_scheme['comment'])
return sum(([
color(x)
for x in (display_linize(line, columns) if line else fmtstr(''))
] for line in docstring.split('\n')), [])
def fmtfsa(fsa, **kwargs):
o_fsa = FSArray(fsa.height, fsa.width)
for y in range(fsa.height):
raw = map(lambda x: fmtstr(x.new_with_atts_removed(xforms.keys()), **kwargs), fsa[y:y+1, 0:fsa.width])
o_fsa[y:y+1, 0:o_fsa.width] = raw
return o_fsa
def to_fsarray(self, width, prompt, tail=""):
"""Returns wrapped and coloured output as an fsarray. Includes the given
prompt and tail and fits in the given width."""
string = yellow(prompt) + fmtstr("".join(self.typed)) + tail
chunks = [string[i:i + width] for i in range(0, len(string), width)]
return fsarray(chunks)
def scan_controls(camera, menu):
global imgcount
global hdrframe
global exprange
global energize
global forward
global scanning
menu.pause()
delay = mindelay
stopscan = th.Event()
stopscan.clear()
izero = 5
iss = 3 + izero
isd = 5 + izero
isc = 7 + izero
isr = 9 + izero
iim = 11 + izero
ihd = 13 + izero
with ci.FullscreenWindow() as window:
with ci.Input() as inputgen:
scr = ci.FSArray(window.height, window.width)
ilast = window.height
scr[izero - 1,
0:window.width - 1] = ci.fsarray([u'_' * window.width])
scr[ilast - 1,
0:window.width - 2] = ci.fsarray([u'_' * window.width])
scr[ilast - 2,
0:window.width - 2] = ci.fsarray([u'_' * window.width])
msg = ci.fmtstr(on_blue(bold(yellow(u'CONTROL INTERFACE'))))
center = int((window.width - msg.width) / 2)
scr[izero, center:msg.width] = [msg]
msgspeed = ci.fmtstr(u'delay: ')
scr[iss, 0:msgspeed.width] = [msgspeed]
ispeed = msgspeed.width + 1
msgcw = ci.fmtstr(u'direction:')
scr[isd, 0:msgcw.width] = [msgcw]
icw = msgcw.width + 2
msgfwd = ci.fmtstr('FORWARD ')
msgback = ci.fmtstr('BACKWARD')
msgamp = ci.fmtstr(u'position:')
scr[isc, 0:msgamp.width] = [msgamp]
msgampon = ci.fmtstr(bold(green('LOCKED ')))
msgampoff = ci.fmtstr(bold(yellow('UNLOCKED')))
msgrun = ci.fmtstr(u'state:')
scr[isr, 0:msgrun.width] = [msgrun]
msgon = ci.fmtstr(bold(green('SCANNING')))
msgoff = ci.fmtstr(bold(red('STOP ')))
msgimg = ci.fmtstr(u'imgcount: ')
scr[iim, 0:msgimg.width] = [msgimg]
imgstg = ci.fmtstr(str(imgcount).zfill(imglgth))
scr[iim, icw:icw + imgstg.width] = [imgstg]
delaylab = ci.fmtstr(on_blue(bold(yellow('delay (s) ='))))
delaystg = ci.fmtstr(on_blue(red(bold(' ' + str(int(delay))))))
scr[izero, 0:delaylab.width] = [delaylab]
scr[izero, delaylab.width:delaylab.width + delaystg.width +
1] = [delaystg]
isolab = ci.fmtstr(on_blue(bold(yellow('iso ='))))
if camera.error == 0:
isostg = ci.fmtstr(
on_blue(red(bold(' ' + str(camera.get_iso())))))
else:
isostg = ci.fmtstr(on_blue(red(bold(' ' + 'No Cam'))))
scr[izero, window.width - isolab.width -
isostg.width:window.width - isostg.width] = [isolab]
scr[izero, window.width - isostg.width:window.width] = [isostg]
shutlab = ci.fmtstr(on_blue(bold(yellow('exptime ='))))
if camera.error == 0:
shutstg = ci.fmtstr(
on_blue(red(bold(' ' + str(camera.get_exposure_time())))))
else:
shutstg = ci.fmtstr(on_blue(red(bold(' ' + 'No Cam'))))
icenter = int((window.width + shutlab.width + shutstg.width) / 2)
scr[ilast - 2, icenter - shutlab.width - shutstg.width:icenter -
shutstg.width] = [shutlab]
scr[ilast - 2, icenter - shutstg.width:icenter] = [shutstg]
hdrlab = ci.fmtstr(on_blue(bold(yellow('hdrframe ='))))
hdrstg = ci.fmtstr(on_blue(red(bold(' ' + str(hdrframe)))))
scr[ilast - 2, window.width - hdrlab.width -
hdrstg.width:window.width - hdrstg.width] = [hdrlab]
scr[ilast - 2, window.width - hdrstg.width:window.width] = [hdrstg]
explab = ci.fmtstr(on_blue(bold(yellow('exprange ='))))
expstg = ci.fmtstr(on_blue(red(bold(' ' + str(exprange)))))
scr[ilast - 2, 0:explab.width] = [explab]
scr[ilast - 2,
explab.width:explab.width + expstg.width + 1] = [expstg]
scanning = False
if not scanning:
scr[isr, icw:icw + msgoff.width] = [msgoff]
else:
scr[isr, icw:icw + msgon.width] = [msgon]
if forward:
scr[isd, icw:icw + msgfwd.width] = [msgfwd]
else:
scr[isd, icw:icw + msgback.width] = [msgback]
if energize:
scr[isc, icw:icw + msgampon.width] = [msgampon]
else:
scr[isc, icw:icw + msgampoff.width] = [msgampoff]
#displays initial values
window.render_to_terminal(scr)
for c in inputgen:
if c == '<ESC>':
if scanning:
stopscan.set()
thscan.join(timeout=None)
scanning = False
break
elif c == '<UP>':
ispeed = max(ispeed + 1, msgspeed.width + 1)
ispeed = min(ispeed, window.width - 1)
scr[iss, ispeed:ispeed + 1] = [ci.fmtstr(yellow('|'))]
delay = int(mindelay + float(ispeed - msgspeed.width - 1) /
float(window.width - msgspeed.width - 2) *
(maxdelay - mindelay))
elif c == '<DOWN>':
scr[iss, ispeed:ispeed + 1] = [ci.fmtstr(u' ')]
ispeed = max(ispeed - 1, msgspeed.width + 1)
ispeed = min(ispeed, window.width - 1)
delay = int(mindelay + float(ispeed - msgspeed.width - 1) /
float(window.width - msgspeed.width - 2) *
(maxdelay - mindelay))
elif c == '<RIGHT>':
if not scanning:
if not forward:
forward = True
scr[isd, icw:icw + msgfwd.width] = [msgfwd]
elif c == '<LEFT>':
if not scanning:
if forward:
forward = False
scr[isd, icw:icw + msgback.width] = [msgback]
elif c == '<SPACE>':
scanning = not (scanning)
if scanning:
stopscan.clear()
thscan = th.Thread(name='scan',
target=scan_frames,
args=[camera, delay, stopscan])
thscan.start()
scr[isr, icw:icw + msgon.width] = [msgon]
else:
stopscan.set()
thscan.join(timeout=None)
scr[isr, icw:icw + msgoff.width] = [msgoff]
elif c == '<Ctrl-j>':
energize = not (energize)
if energize:
scr[isc, icw:icw + msgampon.width] = [msgampon]
else:
scr[isc, icw:icw + msgampoff.width] = [msgampoff]
else:
msghelp = ci.fmtstr(
bold(
yellow(
u'Use enter, arrow keys and space bar for control. Escape to exit'
)))
centerhelp = int((window.width - msghelp.width) / 2)
scr[ilast - 1,
centerhelp:centerhelp + msghelp.width] = [msghelp]
#display updated values
# delaylab=ci.fmtstr(on_blue(bold(yellow('delay ='))))
delaystg = ci.fmtstr(on_blue(red(bold(' ' + str(int(delay))))))
scr[izero, 0:delaylab.width] = [delaylab]
scr[izero, delaylab.width:delaylab.width + delaystg.width +
1] = [delaystg]
imgstg = ci.fmtstr(str(imgcount).zfill(imglgth))
scr[iim, icw:icw + imgstg.width] = [imgstg]
window.render_to_terminal(scr)
menu.resume()
return
def _react_to_log_parser(self, params, process):
ret_value = (LogParser.CONTINUE, None)
line_data = []
iteration = None
if params['iterations']:
iteration = int(params['iterations'])
line_data.append(('iters', iteration))
if params['last_minibatch_loss']:
if self._termination_rules['nan'] and math.isnan(
params['last_minibatch_loss']):
ret_value = (LogParser.STOP, TrainPlan.SKIPPED)
loss = float(params['last_minibatch_loss'])
line_data.append(('loss', loss))
if iteration is not None and (
len(self._loss_history) == 0
or self._loss_history[-1][0] < iteration):
self._loss_history.append((iteration, loss))
if params['last_lr']:
line_data.append(('lr', params['last_lr']))
for i, test_res in enumerate(params['test_results']):
def format_percentage(x):
try:
float_value = float(x)
return '%.2f%%' % (
100 * float_value) if float_value < 1.0 else '100%'
except ValueError:
return x
loss = next(itertools.ifilter(lambda x: x[0] == 'loss', test_res),
None)
if loss:
if self._termination_rules['nan'] and 'nan' in loss[1]:
ret_value = (LogParser.STOP, TrainPlan.SKIPPED)
accuracy = next(
itertools.ifilter(lambda x: x[0] == 'accuracy', test_res),
None)
if accuracy:
formatted_value = format_percentage(accuracy[1])
best_accuracy = next(
itertools.ifilter(lambda x: x[0] == 'best_accuracy',
test_res), None)
moving_avg = next(
itertools.ifilter(lambda x: x[0] == 'moving_avg',
test_res), None)
if moving_avg:
avg_formatted_value = format_percentage(moving_avg[1])
else:
avg_formatted_value = ''
if best_accuracy:
best_formatted_value = format_percentage(best_accuracy[1])
line_data.append(('acc %d' % i, '%s (b: %s, a: %s)' %
(formatted_value, best_formatted_value,
avg_formatted_value)))
else:
line_data.append(('acc %d' % i, '%s (a: %s)' %
(formatted_value, avg_formatted_value)))
n = len(self._loss_history)
if n > 10:
mid = int(round(n / 2.0))
current_average = np.median(
[x for _, x in self._loss_history[mid:]])
previous_average = np.median(
[x for _, x in self._loss_history[:mid]])
iter_gap = self._loss_history[-1][0] - self._loss_history[-mid -
1][0]
if iter_gap > 0:
delta = (current_average -
previous_average) / (abs(previous_average) * iter_gap)
line_data.append(('rel_avg_delta', delta))
if delta > -self._termination_rules['delta']:
ret_value = (LogParser.STOP, TrainPlan.SKIPPED)
self._reprint(
bold(
fmtstr(' | ').join([
underline(key) + ':' + (' %s' %
(ensure_precision(value, 2), ))
for key, value in line_data
])))
user_input = self._special_user_input()
if TrainPlan.NEWLINE in user_input:
self._print(' ', add_new_line=False)
if TrainPlan.SIGINT in user_input:
self._print(' ')
ans = ''
while ans.lower() not in ['s', 'c', 'e', 'r']:
ans = self._raw_input(
'Do you want to (s)kip this run, (c)ontinue running, (r)etry or (e)xit? '
)
if ans.lower() == 's':
return (LogParser.STOP, TrainPlan.SKIPPED)
elif ans.lower() == 'e':
process.terminate()
self._user_ask_exit = True
return (LogParser.STOP, TrainPlan.USER_ASK_EXIT)
elif ans.lower() == 'r':
return (LogParser.STOP, TrainPlan.RETRY)
return ret_value
def format_str(self, s):
return fmtstr(s, **self.kwargs)
def formatted_docstring(docstring, columns, config):
color = func_for_letter(config.color_scheme['comment'])
return sum(([color(x) for x in (display_linize(line, columns) if line else
fmtstr(''))]
for line in docstring.split('\n')), [])
from lib.engine.converter import ConverterEngine
from curtsies import fmtstr
import time
import random
if __name__ == "__main__":
while True:
engine = ConverterEngine()
engine.convert()
rnd = random.randint(60, 500)
print(fmtstr("Sleeping %s seconds" % rnd, "yellow"))
time.sleep(rnd)