def imshow(mat, **kwargs):
'''Show the matrix on the graph'''
fig, ax = new_figure(**kwargs)
kw_imshow = utils.filter_kwargs(kwargs, ax.imshow, plt.imshow)
im = ax.imshow(mat, **kw_imshow)
ax = add_text(ax, **kwargs)
fig.colorbar(im)
return im
def hist(data, **kwargs):
'''Histogram of the data'''
_, ax = new_figure(**kwargs)
kw_hist = utils.filter_kwargs(kwargs, ax.hist, plt.hist)
ax.hist(data, **kw_hist)
ax.legend()
add_text(ax, **kwargs)
return
def quiver(X, Y, U, V, **kwargs):
'''Quiver plot for the vector in origin+dx,dy'''
_, ax = new_figure(**kwargs)
C = kwargs.pop('C', None) # for the color of the plot
kw_quiver = utils.filter_kwargs(kwargs, ax.quiver)
# ax.quiver(X, Y, U, V, C, **kw_quiver)
ax.quiver(X, Y, U, V, scale=1, **kw_quiver)
add_text(ax, **kwargs)
return
def scatter(x, y, ax, **kwargs):
'''Scatter plot'''
# pdb.set_trace()
kwargs_scatter = utils.filter_kwargs(kwargs,
plt.scatter,
ax.scatter,
add_kwargs=['zorder', 'color'])
ax.scatter(x, y, **kwargs_scatter)
ax = add_text(ax, **kwargs)
ax.legend()
return ax
def build_component(model, name, user_args, **extra_args):
class_name = user_args.pop('class')
try:
class_ = globals()[class_name]
except KeyError as ex:
raise ValueError(
"Class named '{0}' does not exist".format(class_name)) from ex
valid_args, invalid_args = utils.filter_kwargs(user_args, class_.__init__)
if invalid_args:
raise ValueError(
'{0} does not accept the following arguments: {1}'.format(
class_name, list(invalid_args.keys())))
valid_args.update(extra_args)
valid_args['model'] = model
valid_args['name'] = name
final_valid_args, _ = utils.filter_kwargs(valid_args, class_.__init__)
instance = class_(**final_valid_args)
return instance
def scatter_3d(x, y, z, **kwargs):
'''3d plot'''
_, ax = new_figure(projection='3d', **kwargs)
kwargs_scatter = utils.filter_kwargs(
kwargs, ax.scatter,
add_kwargs=['label']) # we filter the keyword arguments
ax.scatter(x, y, z, **kwargs_scatter)
ax = add_text(ax, **kwargs)
ax.set_xlabel(kwargs.pop('xlabel', 'x'))
ax.set_ylabel(kwargs.pop('ylabel', 'y'))
ax.set_zlabel(kwargs.pop('zlabel', 'z'))
return ax
def build_component(model, name, user_args, **extra_args):
# Instantiate component using the 'class' argument
class_name = user_args.pop('class')
try:
class_ = globals()[class_name]
except KeyError as ex:
raise ValueError("Class named '{0}' does not exist".format(class_name)) from ex
# First we check that the user defined dict itself does not contain invalid args
valid_args, invalid_args = filter_kwargs(user_args, class_.__init__)
if invalid_args:
raise ValueError(
'{0} does not accept the following arguments: {1}'.format(class_name, list(invalid_args.keys())))
# Now we add some "hard-coded" args, which some classes may accept and some may not
# So then we filter again, this time ignoring any invalid args
valid_args.update(extra_args)
valid_args['model'] = model
valid_args['name'] = name
final_valid_args, _ = filter_kwargs(valid_args, class_.__init__)
instance = class_(**final_valid_args)
return instance
def plot(t, x, **kwargs):
'''Plot x wrt t
:kwargs: arguments for the pyplot function
'''
ax = kwargs.pop('ax', None) # the default is having a new figure
if ax is None:
_, ax = new_figure(**kwargs)
kwargs_plot = utils.filter_kwargs(kwargs, ax.plot)
# pdb.set_trace()
ax.plot(t, x, **kwargs_plot)
add_text(ax, **kwargs)
return
def new_figure(**kwargs):
'''Confgures a new figure, should be called before any plotting'''
# title = kwargs.pop('title', '')
subplots = kwargs.pop('subplots', (1, 1)) # should be iterable
# projection = kwargs.pop('projection', None)
nrows, ncols = subplots
xlim = kwargs.pop('xlim', None)
ylim = kwargs.pop('ylim', None)
fig = plt.figure()
subplot_kw = utils.filter_kwargs(kwargs,
fig.add_subplot,
add_kwargs=['projection'])
plt.close('all')
# if projection is None:
fig, axes = plt.subplots(nrows, ncols, subplot_kw=subplot_kw)
# else:
# fig, axes = plt.subplots(nrows, ncols, projection=projection, subplot_kw=subplot_kw)
# axes.set_title(title)
if xlim is not None:
axes.set_xlim(xlim)
if ylim is not None:
axes.set_ylim(ylim)
return fig, axes
def create_index(connection, index_name, doc_type, logger, **kwargs):
"""
Creates an Elasticsearch index needed for similarity based searching
Args:
connection: Elasticsearch client object
index_name: The name of the index
doc_type: The type of the documents that will be indexed
logger: logging object to log at debug and exception level
kwargs:
master_timeout: Specify timeout for connection to master
timeout: Explicit operation timeout
update_all_types: Whether to update the mapping for all fields with the same name across all types or not
wait_for_active_shards: Set the number of active shards to wait for before the operation returns.
doc_type: The name of the document type
allow_no_indices: Whether to ignore if a wildcard indices expression resolves into no concrete indices.
(This includes _all string or when no indices have been specified)
expand_wildcards: Whether to expand wildcard expression to concrete indices that are open, closed or both.,
default 'open', valid choices are: 'open', 'closed', 'none', 'all'
ignore_unavailable: Whether specified concrete indices should be ignored when unavailable
(missing or closed)
Refer https://elasticsearch-py.readthedocs.io/en/master/api.html#elasticsearch.client.IndicesClient.create
Refer https://elasticsearch-py.readthedocs.io/en/master/api.html#elasticsearch.client.IndicesClient.put_mapping
"""
try:
body = {
'index': {
'analysis': {
'analyzer': {
'my_analyzer': {
'tokenizer': 'whitespace',
'filter': ['standard', 'lowercase', 'my_stemmer']
}
},
'filter': {
'my_stemmer': {
'type': 'stemmer',
'name': 'english'
}
}
}
}
}
# At this point in time, elasticsearch-py doesn't accept arbitrary kwargs, so we have to filter kwargs per
# method. Refer https://github.com/elastic/elasticsearch-py/blob/master/elasticsearch/client/indices.py
create_kwargs = filter_kwargs(kwargs=kwargs,
keep_kwargs_keys=[
'master_timeout', 'timeout',
'update_all_types',
'wait_for_active_shards'
])
connection.indices.create(index=index_name, body=body, **create_kwargs)
mapping_body = {
doc_type: {
'properties': {
'variants': {
'type': 'string',
'analyzer': 'my_analyzer'
}
}
}
}
put_mapping_kwargs = filter_kwargs(kwargs=kwargs,
keep_kwargs_keys=[
'allow_no_indices',
'expand_wildcards',
'ignore_unavailable',
'master_timeout', 'timeout',
'update_all_types'
])
if doc_type:
connection.indices.put_mapping(body=mapping_body,
index=index_name,
doc_type=doc_type,
**put_mapping_kwargs)
else:
logger.debug(
'%s: doc_type not in arguments, skipping put_mapping on index ...'
% log_prefix)
logger.debug('%s: Create Index: Operation successfully completed' %
log_prefix)
except Exception as e:
logger.exception(
'%s:Exception: while creating index, Rolling back \n %s' %
(log_prefix, e))
delete_index(connection=connection,
index_name=index_name,
logger=logger)
def _pre_process_container(self, container, prefix=''):
def replace_msg(module_name, modules=None):
msglogger.debug('Module ' + module_name)
if modules:
msglogger.debug('\tReplacing: {}.{}'.format(
modules[0].__module__, modules[0].__class__.__name__))
msglogger.debug('\tWith: {}.{}'.format(
modules[1].__module__, modules[1].__class__.__name__))
else:
msglogger.debug('\tSkipping')
# Iterate through model, insert quantization functions as appropriate
for name, module in container.named_children():
full_name = prefix + name
if isinstance(module, tuple(self.replacement_blacklist)):
replace_msg(full_name)
continue
if module in self.modules_processed:
previous_name, previous_wrapper = self.modules_processed[
module]
warnings.warn(
"Module '{0}' references to same module as '{1}'."
' Replacing with reference the same wrapper.'.format(
full_name, previous_name), UserWarning)
if previous_wrapper:
replace_msg(full_name, (module, previous_wrapper))
setattr(container, name, previous_wrapper)
else:
replace_msg(full_name)
continue
current_qbits = self.module_qbits_map[full_name]
# TODO - Review necessity of the block below
if current_qbits.acts is None and current_qbits.wts is None and not self.module_overrides_map[
full_name]:
# We indicate this module wasn't replaced by a wrapper
replace_msg(full_name)
self.modules_processed[module] = full_name, None
else:
# We use a type hint comment to let IDEs know replace_fn is a function
replace_fn = self.replacement_factory.get(
type(module),
self.default_repalcement_fn) # type: Optional[Callable]
# If the replacement function wasn't specified - continue without replacing this module.
if replace_fn is not None:
valid_kwargs, invalid_kwargs = utils.filter_kwargs(
self.module_overrides_map[full_name], replace_fn)
if invalid_kwargs:
raise TypeError(
"""Quantizer of type %s doesn't accept \"%s\"
as override arguments for %s. Allowed kwargs: %s"""
% (type(self), list(invalid_kwargs), type(module),
list(valid_kwargs)))
new_module = replace_fn(module, full_name,
self.module_qbits_map,
**valid_kwargs)
if new_module != module:
replace_msg(full_name, (module, new_module))
# Add to history of prepared submodules
self.modules_processed[module] = full_name, new_module
# To allow recreating this wrapper later on
valid_args = full_name, deepcopy(self.module_qbits_map)
self.modules_processed_args[
full_name] = valid_args, valid_kwargs
setattr(container, name, new_module)
# If a "leaf" module was replaced by a container, add the new layers to the QBits mapping
if not utils.has_children(
module) and utils.has_children(new_module):
for sub_module_name, sub_module in new_module.named_modules(
):
self._add_qbits_entry(
full_name + '.' + sub_module_name,
type(sub_module), current_qbits)
self.module_qbits_map[full_name] = QBits(
acts=current_qbits.acts, wts=None, bias=None)
else:
replace_msg(full_name)
self.modules_processed[module] = full_name, None
if utils.has_children(module):
# For container we call recursively
self._pre_process_container(module, full_name + '.')