def test_encode_images():
# invalid data, but the header and footer are from real files
pngdata = b'\x89PNG\r\n\x1a\nblahblahnotactuallyvalidIEND\xaeB`\x82'
jpegdata = b'\xff\xd8\xff\xe0\x00\x10JFIFblahblahjpeg(\xa0\x0f\xff\xd9'
pdfdata = b'%PDF-1.\ntrailer<</Root<</Pages<</Kids[<</MediaBox[0 0 3 3]>>]>>>>>>'
fmt = {
'image/png' : pngdata,
'image/jpeg' : jpegdata,
'application/pdf' : pdfdata
}
encoded = encode_images(fmt)
for key, value in iteritems(fmt):
# encoded has unicode, want bytes
decoded = decodestring(encoded[key].encode('ascii'))
nt.assert_equal(decoded, value)
encoded2 = encode_images(encoded)
nt.assert_equal(encoded, encoded2)
b64_str = {}
for key, encoded in iteritems(encoded):
b64_str[key] = unicode_to_str(encoded)
encoded3 = encode_images(b64_str)
nt.assert_equal(encoded3, b64_str)
for key, value in iteritems(fmt):
# encoded3 has str, want bytes
decoded = decodestring(str_to_bytes(encoded3[key]))
nt.assert_equal(decoded, value)
def _add_arguments(self, aliases=None, flags=None):
self.alias_flags = {}
# print aliases, flags
if aliases is None:
aliases = self.aliases
if flags is None:
flags = self.flags
paa = self.parser.add_argument
for key,value in iteritems(aliases):
if key in flags:
# flags
nargs = '?'
else:
nargs = None
if len(key) is 1:
paa('-'+key, '--'+key, type=unicode_type, dest=value, nargs=nargs)
else:
paa('--'+key, type=unicode_type, dest=value, nargs=nargs)
for key, (value, help) in iteritems(flags):
if key in self.aliases:
#
self.alias_flags[self.aliases[key]] = value
continue
if len(key) is 1:
paa('-'+key, '--'+key, action='append_const', dest='_flags', const=value)
else:
paa('--'+key, action='append_const', dest='_flags', const=value)
def __init__(self, shell=None, **kwargs):
if not(self.__class__.registered):
raise ValueError('Magics subclass without registration - '
'did you forget to apply @magics_class?')
if shell is not None:
if hasattr(shell, 'configurables'):
shell.configurables.append(self)
if hasattr(shell, 'config'):
kwargs.setdefault('parent', shell)
kwargs['shell'] = shell
self.shell = shell
self.options_table = {}
# The method decorators are run when the instance doesn't exist yet, so
# they can only record the names of the methods they are supposed to
# grab. Only now, that the instance exists, can we create the proper
# mapping to bound methods. So we read the info off the original names
# table and replace each method name by the actual bound method.
# But we mustn't clobber the *class* mapping, in case of multiple instances.
class_magics = self.magics
self.magics = {}
for mtype in magic_kinds:
tab = self.magics[mtype] = {}
cls_tab = class_magics[mtype]
for magic_name, meth_name in iteritems(cls_tab):
if isinstance(meth_name, string_types):
# it's a method name, grab it
tab[magic_name] = getattr(self, meth_name)
else:
# it's the real thing
tab[magic_name] = meth_name
# Configurable **needs** to be initiated at the end or the config
# magics get screwed up.
super(Magics, self).__init__(**kwargs)
def print_alias_help(self):
"""Print the alias part of the help."""
if not self.aliases:
return
lines = []
classdict = {}
for cls in self.classes:
# include all parents (up to, but excluding Configurable) in
# available names
for c in cls.mro()[:-3]:
classdict[c.__name__] = c
for alias, longname in iteritems(self.aliases):
classname, traitname = longname.split('.', 1)
cls = classdict[classname]
trait = cls.class_traits(config=True)[traitname]
help = cls.class_get_trait_help(trait).splitlines()
# reformat first line
help[0] = help[0].replace(longname, alias) + ' (%s)' % longname
if len(alias) == 1:
help[0] = help[0].replace('--%s=' % alias, '-%s ' % alias)
lines.extend(help)
# lines.append('')
print(os.linesep.join(lines))
def _convert_to_config(self):
"""self.parsed_data->self.config, parse unrecognized extra args via KVLoader."""
# remove subconfigs list from namespace before transforming the Namespace
if '_flags' in self.parsed_data:
subcs = self.parsed_data._flags
del self.parsed_data._flags
else:
subcs = []
for k, v in iteritems(vars(self.parsed_data)):
if v is None:
# it was a flag that shares the name of an alias
subcs.append(self.alias_flags[k])
else:
# eval the KV assignment
self._exec_config_str(k, v)
for subc in subcs:
self._load_flag(subc)
if self.extra_args:
sub_parser = KeyValueConfigLoader(log=self.log)
sub_parser.load_config(self.extra_args)
self.config.merge(sub_parser.config)
self.extra_args = sub_parser.extra_args
def deactivate(self):
"""Remove any builtins which might have been added by add_builtins, or
restore overwritten ones to their previous values."""
remove_builtin = self.remove_builtin
for key, val in iteritems(self._orig_builtins):
remove_builtin(key, val)
self._orig_builtins.clear()
self._builtins_added = False
def uncan_dict(obj, g=None):
if istype(obj, dict):
newobj = {}
for k, v in iteritems(obj):
newobj[k] = uncan(v,g)
return newobj
else:
return obj
def _flags_changed(self, name, old, new):
"""ensure flags dict is valid"""
for key, value in iteritems(new):
assert len(value) == 2, "Bad flag: %r:%s" % (key, value)
assert isinstance(
value[0], (dict, Config)), "Bad flag: %r:%s" % (key, value)
assert isinstance(
value[1], string_types), "Bad flag: %r:%s" % (key, value)
def test_get_dict(self):
n = len(self.client)
ar = self.client[:].apply_async(lambda : 5)
self.assertEqual(ar.get(), [5]*n)
d = ar.get_dict()
self.assertEqual(sorted(d.keys()), sorted(self.client.ids))
for eid,r in iteritems(d):
self.assertEqual(r, 5)
def _render_expression(self, check):
"""Turn a mongodb-style search dict into an SQL query."""
expressions = []
args = []
skeys = set(check.keys())
skeys.difference_update(set(self._keys))
skeys.difference_update(set(['buffers', 'result_buffers']))
if skeys:
raise KeyError("Illegal testing key(s): %s" % skeys)
for name, sub_check in iteritems(check):
if isinstance(sub_check, dict):
for test, value in iteritems(sub_check):
try:
op = operators[test]
except KeyError:
raise KeyError("Unsupported operator: %r" % test)
if isinstance(op, tuple):
op, join = op
if value is None and op in null_operators:
expr = "%s %s" % (name, null_operators[op])
else:
expr = "%s %s ?" % (name, op)
if isinstance(value, (tuple, list)):
if op in null_operators and any([v is None for v in value]):
# equality tests don't work with NULL
raise ValueError(
"Cannot use %r test with NULL values on SQLite backend" % test)
expr = '( %s )' % (join.join([expr] * len(value)))
args.extend(value)
else:
args.append(value)
expressions.append(expr)
else:
# it's an equality check
if sub_check is None:
expressions.append("%s IS NULL" % name)
else:
expressions.append("%s = ?" % name)
args.append(sub_check)
expr = " AND ".join(expressions)
return expr, args
def validate_string_dict(dct):
"""Validate that the input is a dict with string keys and values.
Raises ValueError if not."""
for k,v in iteritems(dct):
if not isinstance(k, string_types):
raise ValueError('key %r in dict must be a string' % k)
if not isinstance(v, string_types):
raise ValueError('value %r in dict must be a string' % v)
def can_dict(obj):
"""can the *values* of a dict"""
if istype(obj, dict):
newobj = {}
for k, v in iteritems(obj):
newobj[k] = can(v)
return newobj
else:
return obj
def get_env_vars(self):
env_vars = ET.Element('EnvironmentVariables')
for k, v in iteritems(self.environment_variables):
variable = ET.SubElement(env_vars, "Variable")
name = ET.SubElement(variable, "Name")
name.text = k
value = ET.SubElement(variable, "Value")
value.text = v
return env_vars
def _notice_engine_stopped(self, data):
pid = data["pid"]
for idx, el in iteritems(self.launchers):
if el.process.pid == pid:
break
self.launchers.pop(idx)
self.stop_data[idx] = data
if not self.launchers:
self.notify_stop(self.stop_data)
def _load_flag(self, cfg):
"""update self.config from a flag, which can be a dict or Config"""
if isinstance(cfg, (dict, Config)):
# don't clobber whole config sections, update
# each section from config:
for sec,c in iteritems(cfg):
self.config[sec].update(c)
else:
raise TypeError("Invalid flag: %r" % cfg)
def flatten_flags(self):
"""flatten flags and aliases, so cl-args override as expected.
This prevents issues such as an alias pointing to InteractiveShell,
but a config file setting the same trait in TerminalInteraciveShell
getting inappropriate priority over the command-line arg.
Only aliases with exactly one descendent in the class list
will be promoted.
"""
# build a tree of classes in our list that inherit from a particular
# it will be a dict by parent classname of classes in our list
# that are descendents
mro_tree = defaultdict(list)
for cls in self.classes:
clsname = cls.__name__
for parent in cls.mro()[1:-3]:
# exclude cls itself and Configurable,HasTraits,object
mro_tree[parent.__name__].append(clsname)
# flatten aliases, which have the form:
# { 'alias' : 'Class.trait' }
aliases = {}
for alias, cls_trait in iteritems(self.aliases):
cls, trait = cls_trait.split('.', 1)
children = mro_tree[cls]
if len(children) == 1:
# exactly one descendent, promote alias
cls = children[0]
aliases[alias] = '.'.join([cls, trait])
# flatten flags, which are of the form:
# { 'key' : ({'Cls' : {'trait' : value}}, 'help')}
flags = {}
for key, (flagdict, help) in iteritems(self.flags):
newflag = {}
for cls, subdict in iteritems(flagdict):
children = mro_tree[cls]
# exactly one descendent, promote flag section
if len(children) == 1:
cls = children[0]
newflag[cls] = subdict
flags[key] = (newflag, help)
return flags, aliases
def __init__(cls, name, bases, classdict):
"""Finish initializing the HasTraits class.
This sets the :attr:`this_class` attribute of each TraitType in the
class dict to the newly created class ``cls``.
"""
for k, v in iteritems(classdict):
if isinstance(v, TraitType):
v.this_class = cls
super(MetaHasTraits, cls).__init__(name, bases, classdict)
def update(self, **config):
"""Update this
:param config: dict of properties to be updated
"""
for name, config_value in iteritems(config):
if hasattr(self, name):
setattr(self, name, config_value)
else:
self.log.error("Unknown config for document '%s': '%s:%s'. Ignored...",
self.name, name, config_value)
def squash_dates(obj):
"""squash datetime objects into ISO8601 strings"""
if isinstance(obj, dict):
obj = dict(obj) # don't clobber
for k, v in iteritems(obj):
obj[k] = squash_dates(v)
elif isinstance(obj, (list, tuple)):
obj = [squash_dates(o) for o in obj]
elif isinstance(obj, datetime):
obj = obj.isoformat()
return obj
def extract_dates(obj):
"""extract ISO8601 dates from unpacked JSON"""
if isinstance(obj, dict):
new_obj = {} # don't clobber
for k, v in iteritems(obj):
new_obj[k] = extract_dates(v)
obj = new_obj
elif isinstance(obj, (list, tuple)):
obj = [extract_dates(o) for o in obj]
elif isinstance(obj, string_types):
obj = parse_date(obj)
return obj
def print_flag_help(self):
"""Print the flag part of the help."""
if not self.flags:
return
lines = []
for m, (cfg, help) in iteritems(self.flags):
prefix = '--' if len(m) > 1 else '-'
lines.append(prefix + m)
lines.append(indent(dedent(help.strip())))
# lines.append('')
print(os.linesep.join(lines))
def __new__(mcls, name, bases, classdict):
# FIXME: this duplicates the code from MetaHasTraits.
# I don't think a super() call will help me here.
for k, v in iteritems(classdict):
if isinstance(v, TraitType):
v.name = k
elif inspect.isclass(v):
if issubclass(v, TraitType):
vinst = v()
vinst.name = k
classdict[k] = vinst
cls = MetaQObject.__new__(mcls, name, bases, classdict)
return cls
def _match(self, check):
"""Find all the matches for a check dict."""
matches = []
tests = {}
for k,v in iteritems(check):
if isinstance(v, dict):
tests[k] = CompositeFilter(v)
else:
tests[k] = lambda o: o==v
for rec in itervalues(self._records):
if self._match_one(rec, tests):
matches.append(copy(rec))
return matches
def merge(self, other):
"""merge another config object into this one"""
to_update = {}
for k, v in iteritems(other):
if k not in self:
to_update[k] = copy.deepcopy(v)
else: # I have this key
if isinstance(v, Config) and isinstance(self[k], Config):
# Recursively merge common sub Configs
self[k].merge(v)
else:
# Plain updates for non-Configs
to_update[k] = copy.deepcopy(v)
self.update(to_update)
def extract_dates(obj):
"""extract ISO8601 dates from unpacked JSON"""
if isinstance(obj, dict):
obj = dict(obj) # don't clobber
for k,v in iteritems(obj):
obj[k] = extract_dates(v)
elif isinstance(obj, (list, tuple)):
obj = [ extract_dates(o) for o in obj ]
elif isinstance(obj, string_types):
m = ISO8601_PAT.match(obj)
if m:
# FIXME: add actual timezone support
# this just drops the timezone info
notz = m.groups()[0]
obj = datetime.strptime(notz, ISO8601)
return obj
def print_subcommands(self):
"""Print the subcommand part of the help."""
if not self.subcommands:
return
lines = ["Subcommands"]
lines.append('-' * len(lines[0]))
lines.append('')
for p in wrap_paragraphs(self.subcommand_description):
lines.append(p)
lines.append('')
for subc, (cls, help) in iteritems(self.subcommands):
lines.append(subc)
if help:
lines.append(indent(dedent(help.strip())))
lines.append('')
print(os.linesep.join(lines))
def class_config_section(cls):
"""Get the config class config section"""
def c(s):
"""return a commented, wrapped block."""
s = '\n\n'.join(wrap_paragraphs(s, 78))
return '# ' + s.replace('\n', '\n# ')
# section header
breaker = '#' + '-' * 78
s = "# %s configuration" % cls.__name__
lines = [breaker, s, breaker, '']
# get the description trait
desc = cls.class_traits().get('description')
if desc:
desc = desc.default_value
else:
# no description trait, use __doc__
desc = getattr(cls, '__doc__', '')
if desc:
lines.append(c(desc))
lines.append('')
parents = []
for parent in cls.mro():
# only include parents that are not base classes
# and are not the class itself
# and have some configurable traits to inherit
if parent is not cls and issubclass(parent, Configurable) and \
parent.class_traits(config=True):
parents.append(parent)
if parents:
pstr = ', '.join([p.__name__ for p in parents])
lines.append(
c('%s will inherit config from: %s' % (cls.__name__, pstr)))
lines.append('')
for name, trait in iteritems(cls.class_traits(config=True)):
help = trait.get_metadata('help') or ''
lines.append(c(help))
lines.append('# c.%s.%s = %r' %
(cls.__name__, name, trait.get_default_value()))
lines.append('')
return '\n'.join(lines)
def uncan(obj, g=None):
"""invert canning"""
import_needed = False
for cls,uncanner in iteritems(uncan_map):
if isinstance(cls, string_types):
import_needed = True
break
elif isinstance(obj, cls):
return uncanner(obj, g)
if import_needed:
# perform uncan_map imports, then try again
# this will usually only happen once
_import_mapping(uncan_map, _original_uncan_map)
return uncan(obj, g)
return obj
def __new__(mcls, name, bases, classdict):
"""Create the HasTraits class.
This instantiates all TraitTypes in the class dict and sets their
:attr:`name` attribute.
"""
# print "MetaHasTraitlets (mcls, name): ", mcls, name
# print "MetaHasTraitlets (bases): ", bases
# print "MetaHasTraitlets (classdict): ", classdict
for k,v in iteritems(classdict):
if isinstance(v, TraitType):
v.name = k
elif inspect.isclass(v):
if issubclass(v, TraitType):
vinst = v()
vinst.name = k
classdict[k] = vinst
return super(MetaHasTraits, mcls).__new__(mcls, name, bases, classdict)
def can(obj):
"""prepare an object for pickling"""
import_needed = False
for cls,canner in iteritems(can_map):
if isinstance(cls, string_types):
import_needed = True
break
elif istype(obj, cls):
return canner(obj)
if import_needed:
# perform can_map imports, then try again
# this will usually only happen once
_import_mapping(can_map, _original_can_map)
return can(obj)
return obj
def __iter__(self):
return iter(iteritems(self.__dict__))
def __init__(self, *args, **kwargs):
dict.__init__(self, *args, **kwargs)
self._reverse = dict()
for key, value in iteritems(self):
self._reverse[value] = key
def activate(self):
"""Store ipython references in the __builtin__ namespace."""
add_builtin = self.add_builtin
for name, func in iteritems(self.auto_builtins):
add_builtin(name, func)
def macro(self, parameter_s=''):
"""Define a macro for future re-execution. It accepts ranges of history,
filenames or string objects.
Usage:\\
%macro [options] name n1-n2 n3-n4 ... n5 .. n6 ...
Options:
-r: use 'raw' input. By default, the 'processed' history is used,
so that magics are loaded in their transformed version to valid
Python. If this option is given, the raw input as typed at the
command line is used instead.
-q: quiet macro definition. By default, a tag line is printed
to indicate the macro has been created, and then the contents of
the macro are printed. If this option is given, then no printout
is produced once the macro is created.
This will define a global variable called `name` which is a string
made of joining the slices and lines you specify (n1,n2,... numbers
above) from your input history into a single string. This variable
acts like an automatic function which re-executes those lines as if
you had typed them. You just type 'name' at the prompt and the code
executes.
The syntax for indicating input ranges is described in %history.
Note: as a 'hidden' feature, you can also use traditional python slice
notation, where N:M means numbers N through M-1.
For example, if your history contains (print using %hist -n )::
44: x=1
45: y=3
46: z=x+y
47: print x
48: a=5
49: print 'x',x,'y',y
you can create a macro with lines 44 through 47 (included) and line 49
called my_macro with::
In [55]: %macro my_macro 44-47 49
Now, typing `my_macro` (without quotes) will re-execute all this code
in one pass.
You don't need to give the line-numbers in order, and any given line
number can appear multiple times. You can assemble macros with any
lines from your input history in any order.
The macro is a simple object which holds its value in an attribute,
but IPython's display system checks for macros and executes them as
code instead of printing them when you type their name.
You can view a macro's contents by explicitly printing it with::
print macro_name
"""
opts,args = self.parse_options(parameter_s,'rq',mode='list')
if not args: # List existing macros
return sorted(k for k,v in iteritems(self.shell.user_ns) if\
isinstance(v, Macro))
if len(args) == 1:
raise UsageError(
"%macro insufficient args; usage '%macro name n1-n2 n3-4...")
name, codefrom = args[0], " ".join(args[1:])
#print 'rng',ranges # dbg
try:
lines = self.shell.find_user_code(codefrom, 'r' in opts)
except (ValueError, TypeError) as e:
print(e.args[0])
return
macro = Macro(lines)
self.shell.define_macro(name, macro)
if not ( 'q' in opts) :
print('Macro `%s` created. To execute, type its name (without quotes).' % name)
print('=== Macro contents: ===')
print(macro, end=' ')
def _convert_to_config(self):
"""self.parsed_data->self.config"""
for k, v in iteritems(vars(self.parsed_data)):
exec("self.config.%s = v" % k, locals(), globals())
def json_clean(obj):
"""Clean an object to ensure it's safe to encode in JSON.
Atomic, immutable objects are returned unmodified. Sets and tuples are
converted to lists, lists are copied and dicts are also copied.
Note: dicts whose keys could cause collisions upon encoding (such as a dict
with both the number 1 and the string '1' as keys) will cause a ValueError
to be raised.
Parameters
----------
obj : any python object
Returns
-------
out : object
A version of the input which will not cause an encoding error when
encoded as JSON. Note that this function does not *encode* its inputs,
it simply sanitizes it so that there will be no encoding errors later.
Examples
--------
>>> json_clean(4)
4
>>> json_clean(list(range(10)))
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
>>> sorted(json_clean(dict(x=1, y=2)).items())
[('x', 1), ('y', 2)]
>>> sorted(json_clean(dict(x=1, y=2, z=[1,2,3])).items())
[('x', 1), ('y', 2), ('z', [1, 2, 3])]
>>> json_clean(True)
True
"""
# types that are 'atomic' and ok in json as-is. bool doesn't need to be
# listed explicitly because bools pass as int instances
atomic_ok = (unicode_type, int, type(None))
# containers that we need to convert into lists
container_to_list = (tuple, set, types.GeneratorType)
if isinstance(obj, float):
# cast out-of-range floats to their reprs
if math.isnan(obj) or math.isinf(obj):
return repr(obj)
return obj
if isinstance(obj, atomic_ok):
return obj
if isinstance(obj, bytes):
return obj.decode(DEFAULT_ENCODING, 'replace')
if isinstance(obj,
container_to_list) or (hasattr(obj, '__iter__')
and hasattr(obj, next_attr_name)):
obj = list(obj)
if isinstance(obj, list):
return [json_clean(x) for x in obj]
if isinstance(obj, dict):
# First, validate that the dict won't lose data in conversion due to
# key collisions after stringification. This can happen with keys like
# True and 'true' or 1 and '1', which collide in JSON.
nkeys = len(obj)
nkeys_collapsed = len(set(map(str, obj)))
if nkeys != nkeys_collapsed:
raise ValueError('dict can not be safely converted to JSON: '
'key collision would lead to dropped values')
# If all OK, proceed by making the new dict that will be json-safe
out = {}
for k, v in iteritems(obj):
out[str(k)] = json_clean(v)
return out
# If we get here, we don't know how to handle the object, so we just get
# its repr and return that. This will catch lambdas, open sockets, class
# objects, and any other complicated contraption that json can't encode
return repr(obj)
def __init__(self, msg_dict):
dct = self.__dict__
for k, v in iteritems(dict(msg_dict)):
if isinstance(v, dict):
v = Message(v)
dct[k] = v
def __init__(self, *args, **kw):
# Allow trait values to be set using keyword arguments.
# We need to use setattr for this to trigger validation and
# notifications.
for key, value in iteritems(kw):
setattr(self, key, value)
def _data_changed(self, name, old, new):
for k, v in iteritems(new):
assert mime_pat.match(k)
nt.assert_is_instance(v, string_types)
# Basic things from IPython
from IPython.config.configurable import LoggingConfigurable
from IPython.utils.traitlets import Bool, Unicode, CaselessStrEnum, List, Instance
from IPython.utils.py3compat import iteritems
from .utils import is_iterable, is_string
TEXT, OUTPUT, CODE, ASIS = "text", "output", "code", "asis"
IMAGE_MIMETYPE_TO_FILEEXTENSION = OrderedDict([("image/png", "png"),
("image/svg+xml", "svg"),
("image/jpeg", "jpg"),
("application/pdf", "pdf")])
IMAGE_FILEEXTENSION_TO_MIMETYPE = dict([
(v, k) for k, v in iteritems(IMAGE_MIMETYPE_TO_FILEEXTENSION)
])
MARKUP_FORMAT_CONVERTER = OrderedDict([("text/markdown", "markdown"),
("text/x-markdown", "markdown"),
("text/html", "html"),
("text/latex", "latex")])
class KnitpyOutputException(Exception):
pass
# this is the intersection of what matplotlib supports (eps, pdf, pgf, png, ps, raw, rgba, svg,
# svgz) and what IPython supports ('png', 'png2x', 'retina', 'jpg', 'jpeg', 'svg', 'pdf')...
_possible_image_formats = CaselessStrEnum(values=['pdf', 'png', 'svg'])
def __init__(self, **kwargs):
for key,value in iteritems(kwargs):
setattr(self, key, value)