def _base64_to_file(b64str, outfpath, writetostrio=False):
'''This converts the base64 encoded string to a file.
Parameters
----------
b64str : str
A base64 encoded strin that is the output of `base64.b64encode`.
outfpath : str
The path to where the file will be written. This should include an
appropriate extension for the file (e.g. a base64 encoded string that
represents a PNG should have its `outfpath` end in a '.png') so the OS
can open these files correctly.
writetostrio : bool
If this is True, will return a StringIO object with the binary stream
decoded from the base64-encoded input string `b64str`. This can be
useful to embed these into other files without having to write them to
disk.
Returns
-------
str or StringIO object
If `writetostrio` is False, will return the output file's path as a
str. If it is True, will return a StringIO object directly. If writing
the file fails in either case, will return None.
'''
try:
filebytes = base64.b64decode(b64str)
# if we're writing back to a stringio object
if writetostrio:
outobj = StrIO(filebytes)
return outobj
# otherwise, we're writing to an actual file
else:
with open(outfpath,'wb') as outfd:
outfd.write(filebytes)
if os.path.exists(outfpath):
return outfpath
else:
LOGERROR('could not write output file: %s' % outfpath)
return None
except Exception:
LOGEXCEPTION('failed while trying to convert '
'b64 string to file %s' % outfpath)
return None
def test_main_err(self):
parser = get_argparser()
out = StrIO()
err = StrIO()
def flush():
out.truncate(0)
err.truncate(0)
with replace_attr(sys, 'stdout', out, 'stderr', err):
for raising_args, raising_reg in [
([],
'too few arguments|the following arguments are required: ext'
),
(['-r'], 'expected one argument'),
(['shell', '-r'], 'expected one argument'),
(['shell', '-w'], 'expected one argument'),
(['shell', '-c'], 'expected one argument'),
(['shell', '-t'], 'expected one argument'),
(['shell', '-p'], 'expected one argument'),
(['shell', '-R'], 'expected one argument'),
(['shell', '--renderer_module'], 'expected one argument'),
(['shell', '--width'], 'expected one argument'),
(['shell', '--last_exit_code'], 'expected one argument'),
(['shell', '--last_pipe_status'], 'expected one argument'),
(['shell', '--config'], 'expected one argument'),
(['shell', '--theme_option'], 'expected one argument'),
(['shell', '--config_path'], 'expected one argument'),
(['shell', '--renderer_arg'], 'expected one argument'),
(['shell', '--jobnum'], 'expected one argument'),
(['-r', 'zsh_prompt'],
'too few arguments|the following arguments are required: ext'
),
(['shell', '--last_exit_code', 'i'], 'invalid int value'),
(['shell', '--last_pipe_status',
'1 i'], 'invalid <lambda> value'),
(['shell', '-R', 'abc'], 'invalid <lambda> value'),
]:
self.assertRaises(SystemExit, parser.parse_args, raising_args)
self.assertFalse(out.getvalue())
self.assertRegexpMatches(err.getvalue(), raising_reg)
flush()
def test_main_normal(self):
parser = get_argparser()
out = StrIO()
err = StrIO()
with replace_attr(sys, 'stdout', out, 'stderr', err):
for argv, expargs in [
(['shell'], {
'ext': ['shell']
}),
(['shell', '-r', '.zsh'], {
'ext': ['shell'],
'renderer_module': '.zsh'
}),
([
'shell',
'left',
'-r',
'.zsh',
'--last_exit_code',
'10',
'--last_pipe_status',
'10 20 30',
'--jobnum=10',
'-w',
'100',
'-c',
'common.term_truecolor=true',
'-c',
'common.spaces=4',
'-t',
'default.segment_data.hostname.before=H:',
'-p',
'.',
'-p',
'..',
'-R',
'smth={"abc":"def"}',
], {
'ext': ['shell'],
'side': 'left',
'renderer_module': '.zsh',
'last_exit_code': 10,
'last_pipe_status': [10, 20, 30],
'jobnum': 10,
'width': 100,
'config': {
'common': {
'term_truecolor': True,
'spaces': 4
}
},
'theme_option': {
'default': {
'segment_data': {
'hostname': {
'before': 'H:'
}
}
}
},
'config_path': ['.', '..'],
'renderer_arg': {
'smth': {
'abc': 'def'
}
},
}),
(['shell', '-R', 'arg=true'], {
'ext': ['shell'],
'renderer_arg': {
'arg': True
}
}),
(['shell', '-R', 'arg=true', '-R', 'arg='], {
'ext': ['shell'],
'renderer_arg': {}
}),
(['shell', '-R', 'arg='], {
'ext': ['shell'],
'renderer_arg': {}
}),
(['shell', '-t', 'default.segment_info={"hostname": {}}'], {
'ext': ['shell'],
'theme_option': {
'default': {
'segment_info': {
'hostname': {}
}
}
},
}),
(['shell', '-c', 'common={ }'], {
'ext': ['shell'],
'config': {
'common': {}
}
}),
]:
args = parser.parse_args(argv)
finish_args(args)
for key, val in expargs.items():
self.assertEqual(getattr(args, key), val)
for key, val in args.__dict__.items():
if key not in expargs:
self.assertFalse(
val,
msg=
'key {0} is {1} while it should be something false'
.format(key, val))
self.assertFalse(err.getvalue() + out.getvalue(),
msg='unexpected output: {0!r} {1!r}'.format(
err.getvalue(),
out.getvalue(),
))
def pformat(self, obj: object) -> str:
sio = StrIO()
PythonPrinter(stream=sio)._format(obj)
return sio.getvalue()
def add_cmd_to_checkplot(cpx,
cmdpkl,
require_cmd_magcolor=True,
save_cmd_pngs=False):
'''This adds CMD figures to a checkplot dict or pickle.
Looks up the CMDs in `cmdpkl`, adds the object from `cpx` as a gold(-ish)
star in the plot, and then saves the figure to a base64 encoded PNG, which
can then be read and used by the `checkplotserver`.
Parameters
----------
cpx : str or dict
This is the input checkplot pickle or dict to add the CMD to.
cmdpkl : str or dict
The CMD pickle generated by the `colormagdiagram_cplist` or
`colormagdiagram_cpdir` functions above, or the dict produced by reading
this pickle in.
require_cmd_magcolor : bool
If this is True, a CMD plot will not be made if the color and mag keys
required by the CMD are not present or are nan in this checkplot's
objectinfo dict.
save_cmd_png : bool
If this is True, then will save the CMD plots that were generated and
added back to the checkplotdict as PNGs to the same directory as
`cpx`. If `cpx` is a dict, will save them to the current working
directory.
Returns
-------
str or dict
If `cpx` was a str filename of checkplot pickle, this will return that
filename to indicate that the CMD was added to the file. If `cpx` was a
checkplotdict, this will return the checkplotdict with a new key called
'colormagdiagram' containing the base64 encoded PNG binary streams of
all CMDs generated.
'''
# get the checkplot
if isinstance(cpx, str) and os.path.exists(cpx):
cpdict = _read_checkplot_picklefile(cpx)
elif isinstance(cpx, dict):
cpdict = cpx
else:
LOGERROR('unknown type of checkplot provided as the cpx arg')
return None
# get the CMD
if isinstance(cmdpkl, str) and os.path.exists(cmdpkl):
with open(cmdpkl, 'rb') as infd:
cmd = pickle.load(infd)
elif isinstance(cmdpkl, dict):
cmd = cmdpkl
cpdict['colormagdiagram'] = {}
# get the mags and colors from the CMD dict
cplist_mags = cmd['mags']
cplist_colors = cmd['colors']
# now make the CMD plots for each color-mag combination in the CMD
for c1, c2, ym, ind in zip(cmd['color_mag1'],
cmd['color_mag2'], cmd['yaxis_mag'],
range(len(cmd['color_mag1']))):
# get these from the checkplot for this object
if (c1 in cpdict['objectinfo']
and cpdict['objectinfo'][c1] is not None):
c1mag = cpdict['objectinfo'][c1]
else:
c1mag = np.nan
if (c2 in cpdict['objectinfo']
and cpdict['objectinfo'][c2] is not None):
c2mag = cpdict['objectinfo'][c2]
else:
c2mag = np.nan
if (ym in cpdict['objectinfo']
and cpdict['objectinfo'][ym] is not None):
ymmag = cpdict['objectinfo'][ym]
else:
ymmag = np.nan
if (require_cmd_magcolor
and not (np.isfinite(c1mag) and np.isfinite(c2mag)
and np.isfinite(ymmag))):
LOGWARNING("required color: %s-%s or mag: %s are not "
"in this checkplot's objectinfo dict "
"(objectid: %s), skipping CMD..." %
(c1, c2, ym, cpdict['objectid']))
continue
# make the CMD for this color-mag combination
try:
thiscmd_title = r'%s-%s/%s' % (CMD_LABELS[c1], CMD_LABELS[c2],
CMD_LABELS[ym])
# make the scatter plot
fig = plt.figure(figsize=(10, 8))
plt.plot(cplist_colors[:, ind],
cplist_mags[:, ind],
rasterized=True,
marker='o',
linestyle='none',
mew=0,
ms=3)
# put this object on the plot
plt.plot([c1mag - c2mag], [ymmag],
ms=20,
color='#b0ff05',
marker='*',
mew=0)
plt.xlabel(r'$%s - %s$' % (CMD_LABELS[c1], CMD_LABELS[c2]))
plt.ylabel(r'$%s$' % CMD_LABELS[ym])
plt.title('%s - $%s$ CMD' % (cpdict['objectid'], thiscmd_title))
plt.gca().invert_yaxis()
# now save the figure to StrIO and put it back in the checkplot
cmdpng = StrIO()
plt.savefig(cmdpng,
bbox_inches='tight',
pad_inches=0.0,
format='png')
cmdpng.seek(0)
cmdb64 = base64.b64encode(cmdpng.read())
cmdpng.close()
plt.close('all')
plt.gcf().clear()
cpdict['colormagdiagram']['%s-%s/%s' % (c1, c2, ym)] = cmdb64
# if we're supposed to export to PNG, do so
if save_cmd_pngs:
if isinstance(cpx, str):
outpng = os.path.join(
os.path.dirname(cpx), 'cmd-%s-%s-%s.%s.png' %
(cpdict['objectid'], c1, c2, ym))
else:
outpng = 'cmd-%s-%s-%s.%s.png' % (cpdict['objectid'], c1,
c2, ym)
_base64_to_file(cmdb64, outpng)
except Exception:
LOGEXCEPTION('CMD for %s-%s/%s does not exist in %s, skipping...' %
(c1, c2, ym, cmdpkl))
continue
#
# end of making CMDs
#
if isinstance(cpx, str):
cpf = _write_checkplot_picklefile(cpdict, outfile=cpx, protocol=4)
return cpf
elif isinstance(cpx, dict):
return cpdict
def stream_fixture():
return StrIO()
def post(self, cpfile):
'''This handles POST requests.
Also an AJAX endpoint. Updates the persistent checkplot dict using the
changes from the UI, and then saves it back to disk. This could
definitely be faster by just loading the checkplot into a server-wide
shared dict or something.
'''
# if self.readonly is set, then don't accept any changes
# return immediately with a 400
if self.readonly:
msg = "checkplotserver is in readonly mode. no updates allowed."
resultdict = {'status':'error',
'message':msg,
'readonly':self.readonly,
'result':None}
self.write(resultdict)
raise tornado.web.Finish()
# now try to update the contents
try:
self.cpfile = base64.b64decode(url_unescape(cpfile)).decode()
cpcontents = self.get_argument('cpcontents', default=None)
savetopng = self.get_argument('savetopng', default=None)
if not self.cpfile or not cpcontents:
msg = "did not receive a checkplot update payload"
resultdict = {'status':'error',
'message':msg,
'readonly':self.readonly,
'result':None}
self.write(resultdict)
raise tornado.web.Finish()
cpcontents = json.loads(cpcontents)
# the only keys in cpdict that can updated from the UI are from
# varinfo, objectinfo (objecttags), uifilters, and comments
updated = {'varinfo': cpcontents['varinfo'],
'objectinfo':cpcontents['objectinfo'],
'comments':cpcontents['comments'],
'uifilters':cpcontents['uifilters']}
# we need to reform the self.cpfile so it points to the full path
cpfpath = os.path.join(
os.path.abspath(os.path.dirname(self.cplistfile)),
self.cpfile
)
LOGGER.info('loading %s...' % cpfpath)
if not os.path.exists(cpfpath):
msg = "couldn't find checkplot %s" % cpfpath
LOGGER.error(msg)
resultdict = {'status':'error',
'message':msg,
'readonly':self.readonly,
'result':None}
self.write(resultdict)
raise tornado.web.Finish()
# dispatch the task
updated = yield self.executor.submit(checkplot_pickle_update,
cpfpath, updated)
# continue processing after this is done
if updated:
LOGGER.info('updated checkplot %s successfully' % updated)
resultdict = {'status':'success',
'message':'checkplot update successful',
'readonly':self.readonly,
'result':{'checkplot':updated,
'unixtime':utime.time(),
'changes':cpcontents,
'cpfpng': None}}
# handle a savetopng trigger
if savetopng:
cpfpng = os.path.abspath(cpfpath.replace('.pkl','.png'))
cpfpng = StrIO()
pngdone = yield self.executor.submit(
checkplot_pickle_to_png,
cpfpath, cpfpng
)
if pngdone is not None:
# we'll send back the PNG, which can then be loaded by
# the frontend and reformed into a download
pngdone.seek(0)
pngbin = pngdone.read()
pngb64 = base64.b64encode(pngbin)
pngdone.close()
del pngbin
resultdict['result']['cpfpng'] = pngb64
else:
resultdict['result']['cpfpng'] = ''
self.write(resultdict)
self.finish()
else:
LOGGER.error('could not handle checkplot update for %s: %s' %
(self.cpfile, cpcontents))
msg = "checkplot update failed because of a backend error"
resultdict = {'status':'error',
'message':msg,
'readonly':self.readonly,
'result':None}
self.write(resultdict)
self.finish()
# if something goes wrong, inform the user
except Exception:
LOGGER.exception('could not handle checkplot update for %s: %s' %
(self.cpfile, cpcontents))
msg = "checkplot update failed because of an exception"
resultdict = {'status':'error',
'message':msg,
'readonly':self.readonly,
'result':None}
self.write(resultdict)
self.finish()
