def showhelp(self):
enabled = self.cx.enabled
broken = self.cx.broken
if not self.values or self.values[0] not in self.libs or self.values[
0] not in self.cx.helpmenu:
cats = sorted(self.libs)
cats = [
colorize(lib, 'green') if lib in enabled else lib
for lib in self.libs
]
cats = [
colorize(lib, 'red') if lib in broken else lib for lib in cats
]
cats = ', '.join(cats)
return '%shelp WHAT where WHAT is one of the following: %s' % (
self.cx.settings.bot.command_prefix, cats)
which = self.values[0]
if which in broken:
return '%s is currently broken.' % which
if which not in enabled:
return '%s is not enabled.' % which
return self.cx.helpmenu[which]
def showhelp(self):
enabled = self.cx.master.ENABLED
broken = self.cx.broken
if not self.values or self.values[0] not in self.libs:
cats = sorted(self.libs)
cats = [
colorize(lib, 'lightgrey') if lib not in enabled else lib
for lib in self.libs
]
cats = [
colorize(lib, 'red') if lib in broken else lib for lib in cats
]
cats = ', '.join(cats)
self.chat('%shelp WHAT where WHAT is one of the following: %s' %
(CONTROL_KEY, cats))
return
which = self.values[0]
if which in broken:
return '%s is currently broken.' % which
if which not in enabled:
return '%s is not enabled.' % which
return self.cx.helpmenu[which]
def showhelp(self):
enabled = self.cx.master.ENABLED
broken = self.cx.broken
if not self.values or self.values[0] not in self.libs:
cats = sorted(self.libs)
print 'Cats: %s' % cats
print 'Enabled: %s' % enabled
print 'Broken: %s' % broken
cats = [colorize(lib, 'green') if lib in enabled else lib for lib in self.libs]
cats = [colorize(lib, 'red') if lib in broken else lib for lib in cats]
cats = ', '.join(cats)
print cats
self.chat('%shelp WHAT where WHAT is one of the following: %s' % (self.cx.settings.bot.command_prefix, cats))
return
which = self.values[0]
if which in broken:
return '%s is currently broken.' % which
if which not in enabled:
return '%s is not enabled.' % which
return self.cx.helpmenu[which]
def showhelp(self):
enabled = self.cx.master.ENABLED
broken = self.cx.broken
if not self.values or self.values[0] not in self.libs:
cats = sorted(self.libs)
print 'Cats: %s' % cats
print 'Enabled: %s' % enabled
print 'Broken: %s' % broken
cats = [
colorize(lib, 'green') if lib in enabled else lib
for lib in self.libs
]
cats = [
colorize(lib, 'red') if lib in broken else lib for lib in cats
]
cats = ', '.join(cats)
print cats
self.chat('%shelp WHAT where WHAT is one of the following: %s' %
(self.cx.settings.bot.command_prefix, cats))
return
which = self.values[0]
if which in broken:
return '%s is currently broken.' % which
if which not in enabled:
return '%s is not enabled.' % which
return self.cx.helpmenu[which]
def format(self, record):
msg = logging.Formatter.format(self, record)
if record.levelno >= logging.WARNING:
return util.colorize(msg, util.RED, bold=True)
elif record.levelno == logging.INFO:
return util.colorize(msg, util.GREEN)
else:
return msg
def format(self, record):
msg = logging.Formatter.format(self, record)
if record.levelno >= logging.WARNING:
return util.colorize(msg, util.RED, bold=True)
elif record.levelno == logging.INFO:
return util.colorize(msg, util.GREEN)
else:
return msg
def __log_request(self, url, verb, params):
full_url = url
if bool(params):
if verb == 'get':
full_url += '?' + urllib.urlencode(params)
full_url = util.colorize('cyan', full_url)
verb = util.colorize('magenta', verb.upper())
logger.debug(verb + ' ' + full_url)
if bool(params) and verb != 'get':
logger.debug(json.dumps(params, indent=2))
def pprint(self, short=False):
"""Prettyprints the journal's entries"""
sep = "\n"
pp = sep.join([e.pprint(short=short) for e in self.entries])
if self.config['highlight']: # highlight tags
if self.search_tags:
for tag in self.search_tags:
tagre = re.compile(re.escape(tag), re.IGNORECASE)
pp = re.sub(tagre,
lambda match: util.colorize(match.group(0)),
pp, re.UNICODE)
else:
pp = re.sub(Entry.Entry.tag_regex(self.config['tagsymbols']),
lambda match: util.colorize(match.group(0)), pp)
return pp
def reward(self):
if not self.values:
self.chat("Reward whom?")
return
kinder = self.values[0]
self.chat("Good job, " + kinder + ". Here's your star: " + colorize(u'\u2605', "yellow"))
self._act(" pats " + kinder + "'s head.")
def pprint(self, short=False):
"""Prettyprints the journal's entries"""
sep = "\n"
pp = sep.join([e.pprint(short=short) for e in self.entries])
if self.config['highlight']: # highlight tags
if self.search_tags:
for tag in self.search_tags:
tagre = re.compile(re.escape(tag), re.IGNORECASE)
pp = re.sub(tagre,
lambda match: util.colorize(match.group(0)),
pp, re.UNICODE)
else:
pp = re.sub(r"(?u)([{tags}]\w+)".format(tags=self.config['tagsymbols']),
lambda match: util.colorize(match.group(0)),
pp)
return pp
def plot(self,state,action):
r = state%10
c = state/10
desc = self.map.tolist()
desc[r][c] = util.colorize(desc[r][c], "red", highlight=True)
print("action: ", ["UP","DOWN","LEFT","RIGHT"][action] if action is not None else None)
print("\n".join("".join(row) for row in desc))
def showquote(self, context):
if not self.stock:
return False
name = "%s (%s)" % (self.company, self.symbol.upper())
changestring = str(
self.change) + " (" + ("%.2f" % self.perc_change) + "%)"
if self.change < 0:
changestring = colorize(changestring, 'red')
else:
changestring = colorize(changestring, 'lightgreen')
message = [
name,
str(self.price),
changestring,
]
otherinfo = [
# ("pretty title", "dataname")
("Exchange", "exchange"),
("Trading volume", "volume"),
("Market cap", "market_cap"),
]
# TODO: Don't do this for not in channel, ensure it's a privmsg only. Currently not compatible with channeling
# if context != CHANNEL:
# for item in otherinfo:
# pretty, id = item
# addon = pretty + ": " + getattr(self, id, 'N/A')
# message.append(addon)
link = 'http://finance.yahoo.com/q?s=' + self.symbol
roasted = shorten(link)
message.append(roasted)
output = ', '.join(message)
return output
def showquote(self, context):
if not self.stock:
return False
name = "%s (%s)" % (self.company, self.symbol.upper())
changestring = str(self.change) + " (" + ("%.2f" % self.perc_change) + "%)"
if self.change < 0:
changestring = colorize(changestring, 'red')
else:
changestring = colorize(changestring, 'lightgreen')
message = [
name,
str(self.price),
changestring,
]
otherinfo = [
# ("pretty title", "dataname")
("Exchange", "exchange"),
("Trading volume", "volume"),
("Market cap", "market_cap"),
]
# TODO: Don't do this for not in channel, ensure it's a privmsg only. Currently not compatible with channeling
# if context != CHANNEL:
# for item in otherinfo:
# pretty, id = item
# addon = pretty + ": " + getattr(self, id, 'N/A')
# message.append(addon)
link = 'http://finance.yahoo.com/q?s=' + self.symbol
roasted = shorten(link)
message.append(roasted)
output = ', '.join(message)
return output
def compile_css_js():
css_out = PROJ_ROOT / 'css' / 'compiled-gen.css'
logger.info('Cleaning up compiled css')
try:
os.unlink(css_out)
except OSError:
pass
logger.info('Compiling css')
cmds = (['java', '-jar', PROJ_ROOT / 'tmp' / CSS_COMPILER] +
glob(PROJ_ROOT / 'css' / '*.css') +
['--allowed-non-standard-function', 'skewY',
'--allowed-non-standard-function', 'radial-gradient',
'--allowed-non-standard-function', '-webkit-radial-gradient',
'--allowed-non-standard-function', '-moz-radial-gradient',
'--output-file', css_out])
subprocess.call(cmds)
# To shut up js compiler's long log
jsc_err = StringIO()
logger.info('Compiling js')
jsc_cmds = ['python',
CLOSURE_BIN_BUILD / 'closurebuilder.py',
'--root=%s' % (PROJ_ROOT / 'js'),
'--root=%s' % (PROJ_ROOT / 'third_party'),
'--namespace=postile.entry',
'--output_mode=compiled',
'--compiler_jar=%s' % (PROJ_ROOT / 'tmp' / 'compiler.jar'),
'--compiler_flags=--compilation_level=SIMPLE_OPTIMIZATIONS',
'--compiler_flags=--define=postile.conf.USING_COMPILED_CSS',
'--output_file=%s' % (PROJ_ROOT / 'compiled.js')]
try:
subprocess.check_output(jsc_cmds, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
logger.error(e.output)
logger.error(util.colorize('JS compilation failed', 'yellow'))
sys.exit(1)
logger.info(util.colorize('Done', 'green'))
def showhelp(self):
enabled = self.cx.master.ENABLED
broken = self.cx.broken
if not self.values or self.values[0] not in self.libs:
cats = sorted(self.libs)
cats = [colorize(lib, 'lightgrey') if lib not in enabled else lib for lib in self.libs]
cats = [colorize(lib, 'red') if lib in broken else lib for lib in cats]
cats = ', '.join(cats)
self.chat('%shelp WHAT where WHAT is one of the following: %s' % (CONTROL_KEY, cats))
return
which = self.values[0]
if which in broken:
return '%s is currently broken.' % which
if which not in enabled:
return '%s is not enabled.' % which
return self.cx.helpmenu[which]
def showhelp(self):
enabled = self.cx.enabled
broken = self.cx.broken
if not self.values or self.values[0] not in self.libs or self.values[0] not in self.cx.helpmenu:
cats = sorted(self.libs)
cats = [colorize(lib, "green") if lib in enabled else lib for lib in self.libs]
cats = [colorize(lib, "red") if lib in broken else lib for lib in cats]
cats = ", ".join(cats)
return "%shelp WHAT where WHAT is one of the following: %s" % (self.cx.settings.bot.command_prefix, cats)
which = self.values[0]
if which in broken:
return "%s is currently broken." % which
if which not in enabled:
return "%s is not enabled." % which
return self.cx.helpmenu[which]
def fetch_tools():
tmp = PROJ_ROOT / 'tmp'
logger.info('Creating temporary directory at %s', tmp)
try:
os.mkdir(tmp)
except OSError:
pass
# Downloading
download_if_not_exist(CLOSURE_ZIP_URL, tmp / CLOSURE_ZIP)
download_if_not_exist(JS_COMPILER_ZIP_URL, tmp / JS_COMPILER_ZIP)
download_if_not_exist(SOY_ZIP_URL, tmp / SOY_ZIP)
download_if_not_exist(CSS_COMPILER_URL, tmp / CSS_COMPILER)
# Installing
if not util.which('scss'):
logger.info('Installing scss')
subprocss.call(maybe_add_sudo(['gem', 'install', 'scss']))
with ZipFile(tmp / CLOSURE_ZIP) as f:
logger.info('Extracting closure library')
f.extractall(tmp)
with ZipFile(tmp / JS_COMPILER_ZIP) as f:
logger.info('Extracting closure compiler')
f.extractall(tmp)
with ZipFile(tmp / SOY_ZIP) as f:
logger.info('Extracting closure template compiler')
f.extractall(tmp)
logger.info('Copying closure library into project tree')
shutil.rmtree(PROJ_ROOT / 'third_party', True)
shutil.copytree(tmp / 'third_party', PROJ_ROOT / 'third_party')
shutil.rmtree(PROJ_ROOT / 'js' / 'goog', True)
shutil.copytree(tmp / 'closure' / 'goog', PROJ_ROOT / 'js' / 'goog')
soyutilroot = PROJ_ROOT / 'js' / 'goog' / 'soyutil'
try:
os.makedirs(soyutilroot)
except OSError:
pass
for filename in glob(tmp / 'soy*.js'):
shutil.copy2(filename, soyutilroot)
logger.info(util.colorize('Done', 'green'))
def build_css():
logger.info('Cleaning up css generated from sass')
scss_glob = PROJ_ROOT / 'css' / '*.scss'
for name in glob(scss_glob):
name = name.replace('.scss', '.css')
try:
os.unlink(name)
except OSError:
pass
logger.info('Building up css from sass')
subprocess.call(['ruby', util.which('sass'), '--update'] + glob(scss_glob))
logger.info('Building up css from gss')
gss_glob = PROJ_ROOT / 'css' / '*.gss'
for name in glob(gss_glob):
subprocess.call(['java', '-jar', PROJ_ROOT / 'tmp' / CSS_COMPILER,
'--pretty-print', name,
'--output-file', name.replace('.gss', '-gen.css')])
logger.info(util.colorize('Done', 'green'))
def build_js():
logger.info('Compiling closure templates')
soys = list(util.find(JS_POSTILE, '*.soy'))
subprocess.call(['java', '-jar',
PROJ_ROOT / 'tmp' / 'SoyToJsSrcCompiler.jar',
'--outputPathFormat', '{INPUT_DIRECTORY}/{INPUT_FILE_NAME}.js',
'--shouldProvideRequireSoyNamespaces',
'--srcs'] + soys)
logger.info('Building deps.js')
subprocess.call(['python', CLOSURE_BIN_BUILD / 'depswriter.py',
'--root_with_prefix=%s %s' % (JS_POSTILE, '../postile'),
'--output_file=%s' % (JS_POSTILE / 'deps.js')])
logger.info('Adding soyutil to deps.js')
with open(JS_POSTILE / 'deps.js', 'a') as fout:
with open(PROJ_ROOT / 'hacks' / 'soyutil-deps.js') as fin:
fout.write(fin.read())
logger.info(util.colorize('Done', 'green'))
def __log_response(self, response):
status_code = response.status_code
if status_code == 200 or status_code == 201 or status_code == 204:
status_code = util.colorize('green', str(status_code))
elif status_code == 422:
status_code = util.colorize('yellow', str(status_code))
elif status_code >= 500:
status_code = util.colorize('red', str(status_code))
else:
status_code = util.colorize('bold', str(status_code))
logger.debug(util.colorize('bold', 'Server Response: ') + util.colorize('bold', status_code))
if response.status_code != 204: # delete may have no json
logger.debug(json.dumps(response.json(), indent=2))
def plot_animation(t):
global wave_function
rgb_map = None
if flag_intensity:
cmap = plt.cm.inferno
data = np.abs(wave_function)**2
norm = plt.Normalize(data.min(), data.max())
rgb_map = cmap(norm(data))
rgb_map = rgb_map[:, :, :3]
else:
rgb_map = util.colorize(wave_function)
for i, j in potential_boudnary:
rgb_map[i][j] = 1, 1, 1
plt.imshow(rgb_map,
interpolation='none',
extent=[-size / 2, size / 2, -size / 2, size / 2])
vector_selon_x = util.x_concatenate(wave_function, N)
vector_derive_y_selon_x = util.x_concatenate(
util.dy_square(wave_function, N, step), N)
U_selon_x = vector_selon_x + (1j * delta_t / 2) * (
vector_derive_y_selon_x - V_x * vector_selon_x)
U_selon_x_plus = scipy.sparse.linalg.spsolve(HX, U_selon_x)
wave_function = util.x_deconcatenate(U_selon_x_plus, N)
vector_selon_y = util.y_concatenate(wave_function, N)
vector_derive_x_selon_y = util.y_concatenate(
util.dx_square(wave_function, N, step), N)
U_selon_y = vector_selon_y + (1j * delta_t / 2) * (
vector_derive_x_selon_y - V_y * vector_selon_y)
U_selon_y_plus = scipy.sparse.linalg.spsolve(HY, U_selon_y)
wave_function = util.y_deconcatenate(U_selon_y_plus, N)
print_update()
def reward(self):
if not self.values:
self.chat("Reward whom?")
return
kinder = self.values[0]
if kinder == self.ego.nick:
self.chat("Service is own reward for " + self.ego.nick)
return
drinker = Drinker.objects(name=kinder)
if drinker:
drinker = drinker[0]
rewards = drinker.rewards + 1
else:
drinker = Drinker(name=kinder)
rewards = 1
drinker.rewards = rewards
drinker.save()
self.chat("Good job, " + kinder + ". Here's your star: " + colorize(u'\u2605', "yellow"))
self._act(" pats " + kinder + "'s head.")
def reward(self):
if not self.values:
self.chat("Reward whom?")
return
kinder = self.values[0]
if kinder == self.ego.nick:
self.chat("Service is own reward for " + self.ego.nick)
return
drinker = Drinker.objects(name=kinder)
if drinker:
drinker = drinker[0]
rewards = drinker.rewards + 1
else:
drinker = Drinker(name=kinder)
rewards = 1
drinker.rewards = rewards
drinker.save()
self.chat("Good job, " + kinder + ". Here's your star: " + colorize(u'\u2605', "yellow"))
self._act(" pats " + kinder + "'s head.")
def inner(*args, **kwargs):
print colorize(flask.request.method, "yellow"), "request for", colorize(
flask.request.path, "red"
), "args:", colorize(args, "cyan"), "kwargs:", colorize(kwargs, "bright blue")
print colorize(flask.request.values, "green")
try:
for k, v in flask.request.values.iteritems():
assert k not in kwargs, "cannot take a duped param"
kwargs[k] = v
response = fn(*args, **kwargs)
if isinstance(response, (list, int, set, basestring, dict, tuple)) or response is None:
response_contents = json.dumps(response)
response = flask.Response(response_contents, mimetype="text/json")
# print '\t', colorize(response_contents, 'yellow')
else:
response = flask.Response("%s" % response, mimetype="text/html")
# CORS header
response.headers["Access-Control-Allow-Origin"] = "*"
return response
except AssertionError, e:
return flask.make_response("Server Assertion Error: " + e.message, 400)
def plot(self,state,action):
cell = self.idx2cell[int(state/8)]
desc = self.map.tolist()
desc[cell[0]][cell[1]] = util.colorize(desc[cell[0]][cell[1]], "red", highlight=True)
print("action: ", ["UP","DOWN","LEFT","RIGHT"][action] if action is not None else None)
print("\n".join("".join(row) for row in desc))
def plot(self,state,action):
desc = self.map.tolist()
desc[0][state] = util.colorize(desc[0][state], "red", highlight=True)
print("action: ", ["1","2"][action] if action is not None else None)
print("\n".join("".join(row) for row in desc))
def redraw(self):
# brush = self.cmap.map(np.sum(self.vel * self.vel, axis=1), 'qcolor') * 10
# brush = self.cmap.map(self.charges, 'qcolor')
self.g.setData(pos=self.position, pen=0.5, symbol='o', size=self.eq_dist, pxMode=False, c=colorize(self.charges[None,:,0] + 1j * self.charges[None,:,1]))
self.v.setXRange(-1 * self.view_window[0], self.view_window[0])
self.v.setYRange(-1 * self.view_window[1], self.view_window[1])
def write_err(self, msg, **opts):
color = opts.pop("color", None)
Shell.write_err(self, util.colorize(msg, color=color), **opts)
def toss(self):
self.ticker = int(time.time())
if self.lastsender not in self.players and self.broke:
return "No new players after the limit's broken"
self.joinup(self.lastsender)
if self.playerorder[self.turn] != self.lastsender:
return "Not your turn"
rolling = 5 - self.scoring
if self.values:
try:
rolling = int(self.values[0])
except:
return "Don't roll like that"
if rolling > (5 - self.min):
return "You have %s dice to roll." % (5 - self.min)
if self.scoring == 0:
rolling = 5
# Apply last roll score after determining how many
# dice got picked back up. Maybe better done with a
# pendingscore variable but this works.
if self.scoredice:
self.scoring = 5 - len(self.scoredice) + len(
self.scoredice[:-rolling])
self.scoredice = self.scoredice[:-rolling]
score, scoredice, scoring, min, triple = self.getscore(
self.scoredice)
self.score += score
self.scoredice = []
result = self.roll(rolling)
score, scoredice, scoring, min, triple = self.getscore(result)
busted = False
message = 'and rolling'
color = None
if scoring == 0:
self.turn = (self.turn + 1) % len(self.playerorder)
if result == [4, 2]:
message = 'and possession with intent to distribute.'
else:
message = 'and bust.'
if not self.players[self.playerorder[self.turn]]['broke']:
message += ' %s to roll.' % self.playerorder[self.turn]
self.score = 0
busted = True
color = 'red'
clear = False
if scoring == rolling:
message = 'and clear!'
self.score += score
color = 'lightcyan'
clear = True
# Common resets
if scoring in [0, rolling]:
scoring = 0
score = 0
self.scoring = 0
self.scoredice = []
self.min = 0
min = 0
if scoredice and not clear:
o = [1, 5, 2, 3, 4, 6]
self.scoredice = sorted(scoredice,
cmp=lambda x, y: o.index(x) - o.index(y))
self.scoredice.extend([None] * (rolling - len(self.scoredice)))
self.debug('end of roll' + str(self.scoredice))
self.scoring += scoring
self.min += min
if not color:
display = [
colorize(str(x), 'lightgreen')
if x in [1, 5, triple] else str(x) for x in result
]
message = '%s for %s %s' % (' '.join(display), score + self.score,
message)
else:
message = '%s for %s %s' % (' '.join(
str(s) for s in result), score + self.score, message)
message = colorize(message, color)
if not busted and self.broke:
self.gethigh()
total_now = self.players[self.playerorder[
self.turn]]['score'] + score + self.score
if total_now <= self.top:
message += '. %s to go for the win.' % (self.top - total_now +
50)
else:
message += '. Winning with %s.' % total_now
if busted:
winner = self.checkwin()
if winner:
message = [message, winner]
self.reset()
return message
def toss(self):
self.ticker = int(time.time())
if self.lastsender not in self.players and self.broke:
return "No new players after the limit's broken"
self.joinup(self.lastsender)
if self.playerorder[self.turn] != self.lastsender:
return "Not your turn"
rolling = 5 - self.scoring
if self.values:
try:
rolling = int(self.values[0])
except:
return "Don't roll like that"
if rolling > (5 - self.min):
return "You have %s dice to roll." % (5 - self.min)
if self.scoring == 0:
rolling = 5
# Apply last roll score after determining how many
# dice got picked back up. Maybe better done with a
# pendingscore variable but this works.
if self.scoredice:
self.scoring = 5 - len(self.scoredice) + len(self.scoredice[:-rolling])
self.scoredice = self.scoredice[:-rolling]
score, scoredice, scoring, min, triple = self.getscore(self.scoredice)
self.score += score
self.scoredice = []
result = self.roll(rolling)
score, scoredice, scoring, min, triple = self.getscore(result)
busted = False
message = 'and rolling'
color = None
if scoring == 0:
self.turn = (self.turn + 1) % len(self.playerorder)
if result == [4,2]:
message = 'and possession with intent to distribute.'
else:
message = 'and bust.'
if not self.players[self.playerorder[self.turn]]['broke']:
message += ' %s to roll.' % self.playerorder[self.turn]
self.score = 0
busted = True
color = 'red'
clear = False
if scoring == rolling:
message = 'and clear!'
self.score += score
color = 'lightcyan'
clear = True
# Common resets
if scoring in [0, rolling]:
scoring = 0
score = 0
self.scoring = 0
self.scoredice = []
self.min = 0
min = 0
if scoredice and not clear:
o = [1,5,2,3,4,6]
self.scoredice = sorted(scoredice, cmp=lambda x,y: o.index(x) - o.index(y))
self.scoredice.extend([None] * (rolling - len(self.scoredice)))
self.debug('end of roll' + str(self.scoredice))
self.scoring += scoring
self.min += min
if not color:
display = [colorize(str(x), 'lightgreen') if x in [1,5, triple] else str(x) for x in result]
message = '%s for %s %s' % (' '.join(display), score + self.score, message)
else:
message = '%s for %s %s' % (' '.join(str(s) for s in result), score + self.score, message)
message = colorize(message, color)
if not busted and self.broke:
self.gethigh()
total_now = self.players[self.playerorder[self.turn]]['score'] + score + self.score
if total_now <= self.top:
message += '. %s to go for the win.' % (self.top - total_now + 50)
else:
message += '. Winning with %s.' % total_now
if busted:
winner = self.checkwin()
if winner:
message = [message, winner]
self.reset()
return message
def write_err(self, msg, **opts):
color = opts.pop("color", None)
Shell.write_err(self, util.colorize(msg, color=color), **opts)
def run(self, run_type):
is_training = True if run_type == 'train' else False
self.log('{} epoch: {}'.format(run_type, self.epoch))
image_filenames, label_filenames = self.dataset.load_filenames(
run_type)
global_step = tf.Variable(1, name='global_step', trainable=False)
images, labels = inputs.load_batches(image_filenames,
label_filenames,
shape=self.dataset.SHAPE,
batch_size=self.batch_size,
resize_shape=self.dataset.SHAPE,
crop_shape=(256, 512),
augment=True)
with tf.name_scope('labels'):
color_labels = util.colorize(labels, self.dataset.augmented_labels)
labels = tf.cast(labels, tf.int32)
ignore_mask = util.get_ignore_mask(labels,
self.dataset.augmented_labels)
tf.summary.image('label', color_labels, 1)
tf.summary.image('weights', tf.cast(ignore_mask * 255, tf.uint8),
1)
tf.summary.image('image', images, 1)
logits = self.model.inference(images,
num_classes=self.num_classes,
is_training=is_training)
with tf.name_scope('outputs'):
predictions = layers.predictions(logits)
color_predictions = util.colorize(predictions,
self.dataset.augmented_labels)
tf.summary.image('prediction', color_predictions, 1)
# Add some metrics
with tf.name_scope('metrics'):
accuracy_op, accuracy_update_op = tf.contrib.metrics.streaming_accuracy(
predictions, labels, weights=ignore_mask)
mean_iou_op, mean_iou_update_op = tf.contrib.metrics.streaming_mean_iou(
predictions,
labels,
num_classes=self.num_classes,
weights=ignore_mask)
if is_training:
loss_op = layers.loss(logits,
labels,
mask=ignore_mask,
weight_decay=self.weight_decay)
train_op = layers.optimize(loss_op,
learning_rate=self.learning_rate,
global_step=global_step)
# Merge all summaries into summary op
summary_op = tf.summary.merge_all()
# Create restorer for restoring
saver = tf.train.Saver()
# Initialize session and local variables (for input pipeline and metrics)
sess = tf.Session()
sess.run(tf.local_variables_initializer())
if self.checkpoint is None:
sess.run(tf.global_variables_initializer())
self.log('{} {} from scratch.'.format(run_type, self.model_name))
else:
start_time = time.time()
saver.restore(sess, self.checkpoint)
duration = time.time() - start_time
self.log('{} from previous checkpoint {:s} ({:.2f}s)'.format(
run_type, self.checkpoint, duration))
# Create summary writer
summary_path = os.path.join(self.model_path, run_type)
step_writer = tf.summary.FileWriter(summary_path, sess.graph)
# Start filling the input queues
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
num_examples = self.dataset.NUM_TRAIN_EXAMPLES if is_training else self.dataset.NUM_VALID_EXAMPLES
for local_step in range(num_examples // self.batch_size):
# Take time!
start_time = time.time()
if is_training:
_, loss, accuracy, mean_iou, summary = sess.run([
train_op, loss_op, accuracy_update_op, mean_iou_update_op,
summary_op
])
duration = time.time() - start_time
self.log('Epoch: {} train step: {} loss: {:.4f} accuracy: {:.2f}% duration: {:.2f}s' \
.format(self.epoch, local_step + 1, loss, accuracy * 100, duration))
else:
accuracy, mean_iou, summary = sess.run(
[accuracy_update_op, mean_iou_update_op, summary_op])
duration = time.time() - start_time
self.log('Epoch: {} eval step: {} accuracy: {:.2f}% duration: {:.2f}s'\
.format(self.epoch, local_step + 1, accuracy * 100, duration))
# Save summary and print stats
step_writer.add_summary(summary,
global_step=global_step.eval(session=sess))
# Write additional epoch summaries
epoch_writer = tf.summary.FileWriter(summary_path)
epoch_summaries = []
if is_training:
epoch_summaries.append(
tf.summary.scalar('params/weight_decay', self.weight_decay))
epoch_summaries.append(
tf.summary.scalar('params/learning_rate', self.learning_rate))
epoch_summaries.append(
tf.summary.scalar('params/batch_size', self.batch_size))
epoch_summaries.append(
tf.summary.scalar('metrics/accuracy', accuracy_op))
epoch_summaries.append(
tf.summary.scalar('metrics/mean_iou', mean_iou_op))
epoch_summary_op = tf.summary.merge(epoch_summaries)
summary = sess.run(epoch_summary_op)
epoch_writer.add_summary(summary, global_step=self.epoch)
# Save after each epoch when training
if is_training:
checkpoint_path = os.path.join(self.model_path,
self.model_name + '.checkpoint')
start_time = time.time()
self.checkpoint = saver.save(sess,
checkpoint_path,
global_step=self.epoch)
duration = time.time() - start_time
self.log('Model saved as {:s} ({:.2f}s)'.format(
self.checkpoint, duration))
# Stop queue runners and reset the graph
coord.request_stop()
coord.join(threads)
sess.close()
tf.reset_default_graph()