def _load_metadata(self, metadata):
the_metadata = metadata
if the_metadata is None:
the_metadata = {}
if isinstance(the_metadata, six.string_types):
try:
if os.path.isfile(the_metadata):
with open(the_metadata, 'r') as f:
the_metadata = json.load(f)
else:
req = requests.get(the_metadata)
req.raise_for_status()
the_metadata = req.json()
except (IOError,
ValueError,
requests.exceptions.RequestException) as e:
msg = 'Unable to load JSON at \'{0}\''.format(metadata)
six.raise_from(DataPackageException(msg), e)
if hasattr(the_metadata, 'read'):
try:
the_metadata = json.load(the_metadata)
except ValueError as e:
six.raise_from(DataPackageException(str(e)), e)
if not isinstance(the_metadata, dict):
msg = 'Data must be a \'dict\', but was a \'{0}\''
raise DataPackageException(msg.format(type(the_metadata).__name__))
return the_metadata
def delete_plan(self, plan_name):
"""Deletes a plan and associated files
Deletes a plan by deleting the container matching plan_name. It
will not delete the plan if a stack exists with the same name.
Raises StackInUseError if a stack with the same name as plan_name
exists.
:param plan_name: The name of the container to delete
:type plan_name: str
"""
# heat throws HTTPNotFound if the stack is not found
try:
stack = self.heatclient.stacks.get(plan_name)
if stack is not None:
raise exception.StackInUseError(name=plan_name)
except heatexceptions.HTTPNotFound:
try:
self.plan_store.delete(plan_name)
except swiftexceptions.ClientException as ce:
LOG.exception("Swift error deleting plan.")
if ce.http_status == 404:
six.raise_from(exception.PlanDoesNotExistError(
name=plan_name), ce)
except Exception:
LOG.exception("Error deleting plan.")
raise
def __call__(self, data):
"""
Validate data against source schema, then apply transformation, and
finally validate resulting data against target schema.
:param data: Data to transform
:type data: any
:return: Transformed data
:rtype: any
:raises JsonValidationError: if source data or target data isn't valid
:raises JsonTransformationError: if transformation has gone wrong
"""
self.source.validate(data)
try:
result = self.patch.apply(data)
except JsonPatchException as err:
raise_from(
JsonTransformationError(str(err)),
err
)
self.target.validate(result)
return result
def to_bytes(proto, string):
try:
return struct.pack('>H', int(string, 10))
except ValueError as exc:
six.raise_from(ValueError("Not a base 10 integer"), exc)
except struct.error as exc:
six.raise_from(ValueError("Integer not in range(65536)"), exc)
def validate_plan(self, plan_name):
"""Validate Plan
This private method provides validations to ensure a plan
meets the proper criteria before allowed to persist in storage.
:param plan_files: The files to import into the container.
:type plan_files: dict
:returns boolean
"""
plan = self.get_plan(plan_name)
# there can only be up to one root-template file in metadata
rt = {k: v for (k, v) in plan.files.items()
if v.get('meta', {}).get('file-type') == 'root-template'}
if len(rt) > 1:
raise exception.TooManyRootTemplatesError()
# the plan needs to be validated with heat to ensure it conforms
template, environment, files = templates.process_plan_data(plan.files)
try:
self.heatclient.stacks.validate(
template=template,
files=files,
environment=environment,
show_nested=True)
except heatexceptions.HTTPBadRequest as exc:
LOG.exception("Error validating the plan.")
six.raise_from(exception.HeatValidationFailedError(msg=exc), exc)
# no validation issues found
return True
def raise_with_cause(exc_cls, message, *args, **kwargs):
"""Helper to raise + chain exceptions (when able) and associate a *cause*.
NOTE(harlowja): Since in py3.x exceptions can be chained (due to
:pep:`3134`) we should try to raise the desired exception with the given
*cause* (or extract a *cause* from the current stack if able) so that the
exception formats nicely in old and new versions of python. Since py2.x
does **not** support exception chaining (or formatting) our root exception
class has a :py:meth:`~taskflow.exceptions.TaskFlowException.pformat`
method that can be used to get *similar* information instead (and this
function makes sure to retain the *cause* in that case as well so
that the :py:meth:`~taskflow.exceptions.TaskFlowException.pformat` method
shows them).
:param exc_cls: the :py:class:`~taskflow.exceptions.TaskFlowException`
class to raise.
:param message: the text/str message that will be passed to
the exceptions constructor as its first positional
argument.
:param args: any additional positional arguments to pass to the
exceptions constructor.
:param kwargs: any additional keyword arguments to pass to the
exceptions constructor.
"""
if not issubclass(exc_cls, TaskFlowException):
raise ValueError("Subclass of taskflow exception is required")
if 'cause' not in kwargs:
exc_type, exc, exc_tb = sys.exc_info()
if exc is not None:
kwargs['cause'] = exc
del(exc_type, exc, exc_tb)
six.raise_from(exc_cls(message, *args, **kwargs), kwargs.get('cause'))
def save(self):
parent_product = self.parent_product
current_products = set(parent_product.get_package_child_to_quantity_map())
selected_products, removed_products, selected_quantities = self.get_selected_and_removed()
with atomic():
try:
clear_existing_package(parent_product)
parent_product.make_package(package_def=selected_quantities)
except ImpossibleProductModeException as ipme:
six.raise_from(
Problem(
_("Unable to make package %(product)s: %(error)s") %
{"product": parent_product, "error": ipme}
), ipme
)
products_to_add = selected_products - current_products
products_to_remove = current_products & removed_products
message_parts = []
if products_to_add:
message_parts.append(_("New: %d") % len(products_to_add))
if products_to_remove:
message_parts.append(_("Removed: %d") % len(products_to_remove))
if message_parts and self.request:
messages.success(self.request, ", ".join(message_parts))
def __init__(self, wfs, features, **kwargs):
"""
Parameters
----------
wfs: string or :class:`owslib.wfs.WebFeatureService` instance
The WebFeatureService instance, or URL of a WFS service, from which
to retrieve the geometries.
features: string or list of strings
The typename(s) of features available from the web service that
will be retrieved. Somewhat analogous to layers in WMS/WMTS.
Other Parameters
----------------
**kwargs
Keyword arguments to be used when drawing this feature.
"""
try:
from cartopy.io.ogc_clients import WFSGeometrySource
except ImportError as e:
six.raise_from(ImportError(
'WFSFeature requires additional dependencies. If installed '
'via pip, try `pip install cartopy[ows]`.\n'), e)
self.source = WFSGeometrySource(wfs, features)
crs = self.source.default_projection()
super(WFSFeature, self).__init__(crs, **kwargs)
# Default kwargs
self._kwargs.setdefault('edgecolor', 'black')
self._kwargs.setdefault('facecolor', 'none')
def convert(credentials):
"""Convert oauth2client credentials to google-auth credentials.
This class converts:
- :class:`oauth2client.client.OAuth2Credentials` to
:class:`google.oauth2.credentials.Credentials`.
- :class:`oauth2client.client.GoogleCredentials` to
:class:`google.oauth2.credentials.Credentials`.
- :class:`oauth2client.service_account.ServiceAccountCredentials` to
:class:`google.oauth2.service_account.Credentials`.
- :class:`oauth2client.service_account._JWTAccessCredentials` to
:class:`google.oauth2.service_account.Credentials`.
- :class:`oauth2client.contrib.gce.AppAssertionCredentials` to
:class:`google.auth.compute_engine.Credentials`.
- :class:`oauth2client.contrib.appengine.AppAssertionCredentials` to
:class:`google.auth.app_engine.Credentials`.
Returns:
google.auth.credentials.Credentials: The converted credentials.
Raises:
ValueError: If the credentials could not be converted.
"""
credentials_class = type(credentials)
try:
return _CLASS_CONVERSION_MAP[credentials_class](credentials)
except KeyError as caught_exc:
new_exc = ValueError(_CONVERT_ERROR_TMPL.format(credentials_class))
six.raise_from(new_exc, caught_exc)
def string_iter(string):
if not string.startswith(u'/'):
raise exceptions.StringParseError("Must begin with /", string)
# consume trailing slashes
string = string.rstrip(u'/')
sp = string.split(u'/')
# skip the first element, since it starts with /
sp.pop(0)
while sp:
element = sp.pop(0)
try:
proto = protocol_with_name(element)
codec = codec_by_name(proto.codec)
except (ImportError, exceptions.ProtocolNotFoundError) as exc:
six.raise_from(exceptions.StringParseError("Unknown Protocol", string, element), exc)
value = None
if codec.SIZE != 0:
if len(sp) < 1:
raise exceptions.StringParseError("Protocol requires address", string, proto.name)
if codec.IS_PATH:
value = "/" + "/".join(sp)
if not six.PY2:
sp.clear()
else:
sp = []
else:
value = sp.pop(0)
yield proto, codec, value
def wait_until_render_complete(driver):
'''
'''
from selenium.webdriver.support.ui import WebDriverWait
from selenium.common.exceptions import TimeoutException
def is_bokeh_loaded(driver):
return driver.execute_script('''
const b = window.Bokeh;
return b && b.documents && b.documents.length > 0;
''')
try:
WebDriverWait(driver, 5, poll_frequency=0.1).until(is_bokeh_loaded)
except TimeoutException as e:
raise_from(RuntimeError('Bokeh was not loaded in time. Something may have gone wrong.'), e)
driver.execute_script(_WAIT_SCRIPT)
def is_bokeh_render_complete(driver):
return driver.execute_script('return window._bokeh_render_complete;')
try:
WebDriverWait(driver, 5, poll_frequency=0.1).until(is_bokeh_render_complete)
except TimeoutException:
log.warn("The webdriver raised a TimeoutException while waiting for \
a 'bokeh:idle' event to signify that the layout has rendered. \
Something may have gone wrong.")
finally:
browser_logs = driver.get_log('browser')
severe_errors = [l for l in browser_logs if l.get('level') == 'SEVERE']
if len(severe_errors) > 0:
log.warn("There were severe browser errors that may have affected your export: {}".format(severe_errors))
def details(self, sent_id):
"""
Retrieve additional details about a given sentence.
:param sent_id:
A valid sentence ID.
:raises MissingDataError:
Raised if detailed sentence information was not loaded into the
reader.
:raises InvalidIDError:
Raised if an invalid sentence ID is passed.
:returns:
Returns a tuple of the form:
``(username, date_added, date_modified)``
All three are strings.
"""
if self._detailed_info_dict is None:
raise MissingDataError('Detailed information not loaded.')
try:
return self._detailed_info_dict[sent_id]
except KeyError as e:
raise_from(InvalidIDError('Detailed information not found for '
'sentence ID {}'.format(sent_id)), e)
def parse_sentence(self, text):
"""
Takes a Tanaka corpus formatted sentence and parses it into tagged
:class:`TatoebaIndexReader.WordClass` (by default :class:`TanakaWord`)
word objects.
:param text:
A Tanaka-corpus formatted sentence.
:return:
Returns a :py:class:`list` of :class:`TatoebaIndexReader.WordClass`
objects representing a given sentence.
"""
words = self.sentence_splitter(text)
sentence = []
for word in words:
if not len(word):
continue
try:
wobj = self.WordClass.from_text(word)
except InvalidEntryError as e:
msg = u'Failed to interpret word {w} in sentence {s}'
raise_from(InvalidEntryError(msg.format(w=word, s=text)), e)
sentence.append(wobj)
return sentence
def _get_profile(self, profile_id):
'''dict: Return the profile with the received ID as a dict (None if it
doesn't exist).'''
profile_metadata = self._registry.get(profile_id)
if not profile_metadata:
return
path = self._get_absolute_path(profile_metadata.get('schema_path'))
url = profile_metadata.get('schema')
if path:
try:
return self._load_json_file(path)
except IOError as local_exc:
if not url:
raise local_exc
try:
return self._load_json_url(url)
except IOError:
msg = (
'Error loading profile locally at "{path}" '
'and remotely at "{url}".'
).format(path=path, url=url)
six.raise_from(IOError(msg), local_exc)
elif url:
return self._load_json_url(url)
def _get(self, *args, **kwargs):
try:
response = self._session.get(*args, **kwargs)
response.raise_for_status()
except requests.RequestException as e:
six.raise_from(NetworkError, e)
return response
def assert_reset(self, asserted):
"""Assert or de-assert target reset line"""
try:
self._link.drive_nreset(asserted)
self._nreset_state = asserted
except STLinkException as exc:
six.raise_from(self._convert_exception(exc), exc)
def __call__(self, url, method='GET', body=None, headers=None,
timeout=None, **kwargs):
"""Make an HTTP request using requests.
Args:
url (str): The URI to be requested.
method (str): The HTTP method to use for the request. Defaults
to 'GET'.
body (bytes): The payload / body in HTTP request.
headers (Mapping[str, str]): Request headers.
timeout (Optional[int]): The number of seconds to wait for a
response from the server. If not specified or if None, the
requests default timeout will be used.
kwargs: Additional arguments passed through to the underlying
requests :meth:`~requests.Session.request` method.
Returns:
google.auth.transport.Response: The HTTP response.
Raises:
google.auth.exceptions.TransportError: If any exception occurred.
"""
try:
_LOGGER.debug('Making request: %s %s', method, url)
response = self.session.request(
method, url, data=body, headers=headers, timeout=timeout,
**kwargs)
return _Response(response)
except requests.exceptions.RequestException as caught_exc:
new_exc = exceptions.TransportError(caught_exc)
six.raise_from(new_exc, caught_exc)
def swo_start(self, baudrate):
"""! @brief Start receiving SWO data at the given baudrate."""
try:
self._link.swo_configure(True, baudrate)
self._link.swo_control(True)
except DAPAccess.Error as exc:
six.raise_from(self._convert_exception(exc), exc)
def connect(self, protocol=None):
"""Initialize DAP IO pins for JTAG or SWD"""
try:
self._link.enter_debug(stlink.STLink.Protocol.SWD)
self._is_connected = True
except STLinkException as exc:
six.raise_from(self._convert_exception(exc), exc)
def assert_reset(self, asserted):
"""! @brief Assert or de-assert target reset line"""
try:
self._invalidate_cached_registers()
self._link.assert_reset(asserted)
except DAPAccess.Error as exc:
six.raise_from(self._convert_exception(exc), exc)
def read_dp(self, addr, now=True):
"""! @brief Read a DP register.
@param self
@param addr Integer register address being one of (0x0, 0x4, 0x8, 0xC).
@param now
@todo Handle auto DPBANKSEL.
"""
reg_id = self.REG_ADDR_TO_ID_MAP[self.DP, addr]
try:
result = self._link.read_reg(reg_id, now=now)
except DAPAccess.Error as error:
self._invalidate_cached_registers()
six.raise_from(self._convert_exception(error), error)
# Read callback returned for async reads.
def read_dp_result_callback():
try:
return result()
except DAPAccess.Error as error:
self._invalidate_cached_registers()
six.raise_from(self._convert_exception(error), error)
return result if now else read_dp_result_callback
def _config_operator_list_modifiers(self, attr_name, values, line, line_no,
op,
section):
"""Execute + <list>, -<list> or =<list> line in config file
Used in the extension and directories section
:param attr_name: The instance attribute to change
:param values: The values from the config file
:param line: The full line from the config file
:param line_no: the line number of this line
:param op: The actual operation (one of +,- or =)
:param section: The section of the config file
"""
if op == '=':
setattr(self, attr_name, values)
return
if not values:
six.raise_from(ConfigError(
'Invalid value in [{section}] section :'
' \'{line}\' on line {line_no}'.format(
section=section, line=line, line_no=line_no)), None)
if op == '+':
setattr(self, attr_name, getattr(self, attr_name) | values)
elif op == '-':
setattr(self, attr_name, getattr(self, attr_name) - values)
else:
six.raise_from(ConfigError(
'Invalid operator in [{section}] section :'
' \'{line}\' on line {line_no}'.format(
section=section, line=line, line_no=line_no)), None)
def _config_line_directories_section(self, line, line_no):
"""Called for each line in the directories section
line is in one of 3 formats:
= <new directory list>
- <remove directory list>
+ <additional directory list
:param line: The full line from the config file
:param line_no : The line number in the config file
:raises ConfigError: When an invalid line is detected
"""
# Extract Attribute name from the config sections data
attr_name = \
self.config_sections_and_attrs['directories']['directories'][0]
op, direct = line[0], set(
d.strip() for d in line[1:].strip().split(','))
direct = direct if direct != {''} else set()
if direct:
# Identify null strings or strings which are just dots
if any(d == '' for d in direct):
six.raise_from(ConfigError(
'Invalid value in [directories] section :'
' \'{}\' on line {}'.format(
line, line_no)), None)
self._config_operator_list_modifiers(
attr_name, direct, line, line_no, op, 'directories')
def acquire(self):
"""
Attempt to acquire the lock.
If the lock is successfully acquired, this method returns a context
manager that will automatically release the lock when exited. If the
lock cannot be acquired, an ``UnableToAcquireLock`` error will be
raised.
"""
try:
self.backend.acquire(self.key, self.duration, self.routing_key)
except Exception as error:
six.raise_from(
UnableToAcquireLock('Unable to acquire {!r} due to error: {}'.format(self, error)),
error
)
@contextmanager
def releaser():
try:
yield
finally:
self.release()
return releaser()
def _config_line_reports_section(self, line, line_no):
"""Called for each line in the directories section
line is in one of 3 formats:
= <new directory list>
- <remove directory list>
+ <additional directory list
:param line: The full line from the config file
:param line_no : The line number in the config file
:raises ConfigError: When an invalid line is detected"""
attrs_options = self.config_sections_and_attrs['reports']
try:
option, _, value = (x.strip() for x in line.partition('='))
if option != 'verbose':
value = value.lower() in ['true', 'yes']
else:
try:
value = int(value)
except ValueError:
six.raise_from(ConfigError(
'Invalid value in section [reports] :'
' \'{}\' on line {}'.format(
line, line_no)), None)
setattr(self, attrs_options[option][0], value)
except KeyError:
six.raise_from(ConfigError(
'Invalid option in section [reports] :'
' \'{}\' on line {}'.format(
line, line_no)), None)
def _load_schema(self, schema, registry):
the_schema = schema
if isinstance(schema, six.string_types):
try:
the_schema = registry.get(schema)
if not the_schema:
if os.path.isfile(schema):
with open(schema, 'r') as f:
the_schema = json.load(f)
else:
req = requests.get(schema)
req.raise_for_status()
the_schema = req.json()
except (IOError, ValueError, requests.exceptions.RequestException) as ex:
message = 'Unable to load profile at "{0}"'
six.raise_from(
exceptions.ValidationError(message.format(schema)),
ex
)
elif isinstance(the_schema, dict):
the_schema = copy.deepcopy(the_schema)
else:
message = 'Schema must be a "dict", but was a "{0}"'
raise exceptions.ValidationError(message.format(type(the_schema).__name__))
return the_schema
def get_changes(self, resource):
if resource not in self.changes:
try:
self.changes[resource] = list(self.get_plan(resource).get_actions())
except Exception as e:
six.raise_from(errors.Error("{}: {}".format(resource, e)), e)
return self.changes[resource]
def _config_line_extensions_section(self, line, line_no):
"""Called for each line in the extensions section
line is in one of 3 formats:
= <new directory list>
- <remove directory list>
+ <additional directory list
:param line: The full line from the config file
:param line_no : The line number in the config file
"""
# Extract Attribute name from the config sections data
section_data = self.config_sections_and_attrs['extensions']
attr_name = section_data['extensions'][0]
op, ext = line[0], set(e.strip() for e in line[1:].strip().split(','))
ext = ext if ext != {''} else set()
if ext:
# Identify null strings or strings which are just dots
if any(len(e) <= 1 for e in ext):
six.raise_from(ConfigError(
'Invalid value in [extension] section :'
' \'{}\' on line {}'.format(
line, line_no)), None)
# Identify extension values without a leading dot
if any(e[0] != '.' for e in ext):
six.raise_from(ConfigError(
'Invalid value in [extension] section :'
' \'{}\' on line {}'.format(
line, line_no)), None)
self._config_operator_list_modifiers(
attr_name, ext, line, line_no, op, 'extension')
def raise_with_cause(exc_cls, message, *args, **kwargs):
"""Helper to raise + chain exceptions (when able) and associate a *cause*.
NOTE(harlowja): Since in py3.x exceptions can be chained (due to
:pep:`3134`) we should try to raise the desired exception with the given
*cause* (or extract a *cause* from the current stack if able) so that the
exception formats nicely in old and new versions of python. Since py2.x
does **not** support exception chaining (or formatting) the exception
class provided should take a ``cause`` keyword argument (which it may
discard if it wants) to its constructor which can then be
inspected/retained on py2.x to get *similar* information as would be
automatically included/obtainable in py3.x.
:param exc_cls: the exception class to raise (typically one derived
from :py:class:`.CausedByException` or equivalent).
:param message: the text/str message that will be passed to
the exceptions constructor as its first positional
argument.
:param args: any additional positional arguments to pass to the
exceptions constructor.
:param kwargs: any additional keyword arguments to pass to the
exceptions constructor.
.. versionadded:: 1.6
"""
if 'cause' not in kwargs:
exc_type, exc, exc_tb = sys.exc_info()
try:
if exc is not None:
kwargs['cause'] = exc
finally:
# Leave no references around (especially with regards to
# tracebacks and any variables that it retains internally).
del(exc_type, exc, exc_tb)
six.raise_from(exc_cls(message, *args, **kwargs), kwargs.get('cause'))
def reset(self):
"""! @brief Reset the target"""
try:
self._invalidate_cached_registers()
self._link.reset()
except DAPAccess.Error as exc:
six.raise_from(self._convert_exception(exc), exc)
def _pasteObjects(self, cp, cb_maxsize=0):
"""Paste previously copied objects into the current object.
``cp`` is the list of objects for paste as encoded by ``_cb_encode``.
If calling _pasteObjects from python code, pass the result of a
previous call to manage_cutObjects or manage_copyObjects as the first
argument.
``cb_maxsize`` is the maximum size of the JSON representation of the
object list. Set it to a non-zero value to prevent DoS attacks with
huge object lists or zlib bombs.
This method sends IObjectCopiedEvent and IObjectClonedEvent
or IObjectWillBeMovedEvent and IObjectMovedEvent.
Returns tuple of (operator, list of {'id': orig_id, 'new_id': new_id}).
Where `operator` is 0 for a copy operation and 1 for a move operation.
"""
if cp is None:
raise CopyError('No clipboard data found.')
try:
op, mdatas = _cb_decode(cp, cb_maxsize)
except Exception as e:
six.raise_from(CopyError('Clipboard Error'), e)
oblist = []
app = self.getPhysicalRoot()
for mdata in mdatas:
m = loadMoniker(mdata)
try:
ob = m.bind(app)
except ConflictError:
raise
except Exception:
raise CopyError('Item Not Found')
self._verifyObjectPaste(ob, validate_src=op + 1)
oblist.append(ob)
result = []
if op == 0:
# Copy operation
for ob in oblist:
orig_id = ob.getId()
if not ob.cb_isCopyable():
raise CopyError('Not Supported')
try:
ob._notifyOfCopyTo(self, op=0)
except ConflictError:
raise
except Exception:
raise CopyError('Copy Error')
id = self._get_id(orig_id)
result.append({'id': orig_id, 'new_id': id})
orig_ob = ob
ob = ob._getCopy(self)
ob._setId(id)
notify(ObjectCopiedEvent(ob, orig_ob))
self._setObject(id, ob)
ob = self._getOb(id)
ob.wl_clearLocks()
ob._postCopy(self, op=0)
compatibilityCall('manage_afterClone', ob, ob)
notify(ObjectClonedEvent(ob))
elif op == 1:
# Move operation
for ob in oblist:
orig_id = ob.getId()
if not ob.cb_isMoveable():
raise CopyError('Not Supported')
try:
ob._notifyOfCopyTo(self, op=1)
except ConflictError:
raise
except Exception:
raise CopyError('Move Error')
if not sanity_check(self, ob):
raise CopyError("This object cannot be pasted into itself")
orig_container = aq_parent(aq_inner(ob))
if aq_base(orig_container) is aq_base(self):
id = orig_id
else:
id = self._get_id(orig_id)
result.append({'id': orig_id, 'new_id': id})
notify(ObjectWillBeMovedEvent(ob, orig_container, orig_id,
self, id))
# try to make ownership explicit so that it gets carried
# along to the new location if needed.
ob.manage_changeOwnershipType(explicit=1)
try:
orig_container._delObject(orig_id, suppress_events=True)
except TypeError:
orig_container._delObject(orig_id)
warnings.warn(
"%s._delObject without suppress_events is discouraged."
% orig_container.__class__.__name__,
DeprecationWarning)
ob = aq_base(ob)
ob._setId(id)
try:
self._setObject(id, ob, set_owner=0, suppress_events=True)
except TypeError:
self._setObject(id, ob, set_owner=0)
warnings.warn(
"%s._setObject without suppress_events is discouraged."
% self.__class__.__name__, DeprecationWarning)
ob = self._getOb(id)
notify(ObjectMovedEvent(ob, orig_container, orig_id, self, id))
notifyContainerModified(orig_container)
if aq_base(orig_container) is not aq_base(self):
notifyContainerModified(self)
ob._postCopy(self, op=1)
# try to make ownership implicit if possible
ob.manage_changeOwnershipType(explicit=0)
return op, result
def k_feature_routing_function(input_data, tree_parameters, tree_biases, layer_num, max_nodes, num_features_per_node, random_seed, name=None):
r""" Returns the probability that each input will reach each leaf node. Each
decision is made based on k features.
layer_num: The layer number of this tree.
max_nodes: The number of nodes in the tree.
num_features_per_node: The number of features each node can use to make a
decision.
random_seed: The base random seed.
input_data: The training batch's features as a 2-d tensor; `input_data[i][j]`
gives the j-th feature of the i-th input.
tree_parameters: `tree_parameters[i]` gives the weight of
the logistic regression model that translates from node features to
probabilities.
tree_biases: `tree_biases[i]` gives the bias of the logistic
regression model that translates from node features to
probabilities.
tree_features: `tree_features[i]` gives the decision feature for node i.
probabilities: `probabilities[i][j]` is the probability that input i
will reach node j.
Args:
input_data: A `Tensor` of type `float32`.
tree_parameters: A `Tensor` of type `float32`.
tree_biases: A `Tensor` of type `float32`.
layer_num: An `int`.
max_nodes: An `int`.
num_features_per_node: An `int`.
random_seed: An `int`.
name: A name for the operation (optional).
Returns:
A `Tensor` of type `float32`.
"""
_ctx = _context._context
if _ctx is None or not _ctx._eager_context.is_eager:
layer_num = _execute.make_int(layer_num, "layer_num")
max_nodes = _execute.make_int(max_nodes, "max_nodes")
num_features_per_node = _execute.make_int(num_features_per_node, "num_features_per_node")
random_seed = _execute.make_int(random_seed, "random_seed")
_, _, _op = _op_def_lib._apply_op_helper(
"KFeatureRoutingFunction", input_data=input_data,
tree_parameters=tree_parameters, tree_biases=tree_biases,
layer_num=layer_num, max_nodes=max_nodes,
num_features_per_node=num_features_per_node, random_seed=random_seed,
name=name)
_result = _op.outputs[:]
_inputs_flat = _op.inputs
_attrs = ("layer_num", _op.get_attr("layer_num"), "max_nodes",
_op.get_attr("max_nodes"), "num_features_per_node",
_op.get_attr("num_features_per_node"), "random_seed",
_op.get_attr("random_seed"))
_execute.record_gradient(
"KFeatureRoutingFunction", _inputs_flat, _attrs, _result, name)
_result, = _result
return _result
else:
try:
_result = _pywrap_tensorflow.TFE_Py_FastPathExecute(
_ctx._context_handle, _ctx._eager_context.device_name,
"KFeatureRoutingFunction", name, _ctx._post_execution_callbacks,
input_data, tree_parameters, tree_biases, "layer_num", layer_num,
"max_nodes", max_nodes, "num_features_per_node",
num_features_per_node, "random_seed", random_seed)
return _result
except _core._FallbackException:
return k_feature_routing_function_eager_fallback(
input_data, tree_parameters, tree_biases, layer_num=layer_num,
max_nodes=max_nodes, num_features_per_node=num_features_per_node,
random_seed=random_seed, name=name, ctx=_ctx)
except _core._NotOkStatusException as e:
if name is not None:
message = e.message + " name: " + name
else:
message = e.message
_six.raise_from(_core._status_to_exception(e.code, message), None)
def k_feature_gradient(input_data, tree_parameters, tree_biases, routes, layer_num, random_seed, name=None):
r""" Computes the derivative of the routing loss with respect to each decision
node. Each decision node is constrained to make a decision based on only
k features.
layer_num: The layer number of this tree.
random_seed: The base random seed.
input_data: The training batch's features as a 2-d tensor;
`input_data[i][j]` gives the j-th feature of the i-th input.
tree_parameters: `tree_parameters[i]` gives the weight of
the logistic regression model that translates from node features to
probabilities.
tree_biases: `tree_biases[i]` gives the bias of the logistic
regression model that translates from node features to
probabilities.
routes: The routes computed by routing_function_op.
routing_gradient: `routing_gradient` provides du / df, where u is the
routing function and f is the (vector of) decision functions. A decision
function f_i computes the routing decision at node i.
data_gradient: `data_gradient` provides df / dx, where f is the (vector
of) decision functions and x is a batch of data.
weights_gradient: `weights_gradient` provides df / dw, where f is the
(vector of) decision functions and w is the matrix of parameters that
determine how instances are routed through a tree.
f_i, the decision function at node i, is parameterized by t_i (parameters)
and b_i (bias) and takes data x as input. This op is called in
training_ops.py to compute du / df, and we use that to compute
du / dx = du / df * df / dx,
du / dt = du / df * df / dt, and
du / db = du / df * df / db.
Args:
input_data: A `Tensor` of type `float32`.
tree_parameters: A `Tensor` of type `float32`.
tree_biases: A `Tensor` of type `float32`.
routes: A `Tensor` of type `float32`.
layer_num: An `int`.
random_seed: An `int`.
name: A name for the operation (optional).
Returns:
A tuple of `Tensor` objects (routing_gradient, data_gradient, weight_gradient).
routing_gradient: A `Tensor` of type `float32`.
data_gradient: A `Tensor` of type `float32`.
weight_gradient: A `Tensor` of type `float32`.
"""
_ctx = _context._context
if _ctx is None or not _ctx._eager_context.is_eager:
layer_num = _execute.make_int(layer_num, "layer_num")
random_seed = _execute.make_int(random_seed, "random_seed")
_, _, _op = _op_def_lib._apply_op_helper(
"KFeatureGradient", input_data=input_data,
tree_parameters=tree_parameters, tree_biases=tree_biases,
routes=routes, layer_num=layer_num, random_seed=random_seed,
name=name)
_result = _op.outputs[:]
_inputs_flat = _op.inputs
_attrs = ("layer_num", _op.get_attr("layer_num"), "random_seed",
_op.get_attr("random_seed"))
_execute.record_gradient(
"KFeatureGradient", _inputs_flat, _attrs, _result, name)
_result = _KFeatureGradientOutput._make(_result)
return _result
else:
try:
_result = _pywrap_tensorflow.TFE_Py_FastPathExecute(
_ctx._context_handle, _ctx._eager_context.device_name,
"KFeatureGradient", name, _ctx._post_execution_callbacks, input_data,
tree_parameters, tree_biases, routes, "layer_num", layer_num,
"random_seed", random_seed)
_result = _KFeatureGradientOutput._make(_result)
return _result
except _core._FallbackException:
return k_feature_gradient_eager_fallback(
input_data, tree_parameters, tree_biases, routes,
layer_num=layer_num, random_seed=random_seed, name=name, ctx=_ctx)
except _core._NotOkStatusException as e:
if name is not None:
message = e.message + " name: " + name
else:
message = e.message
_six.raise_from(_core._status_to_exception(e.code, message), None)
from __future__ import absolute_import, print_function
# checks
try:
import tensorflow
del tensorflow
except ModuleNotFoundError as e:
from six import raise_from
raise_from(RuntimeError('Please install TensorFlow: https://www.tensorflow.org/install/'), e)
try:
import keras
del keras
except ModuleNotFoundError as e:
if e.name in {'theano','cntk'}:
from six import raise_from
raise_from(RuntimeError(
"Keras is configured to use the '%s' backend, which is not installed. "
"Please change it to use 'tensorflow' instead: "
"https://keras.io/getting-started/faq/#where-is-the-keras-configuration-file-stored" % e.name
), e)
else:
raise e
import keras.backend as K
if K.backend() != 'tensorflow':
raise NotImplementedError(
"Keras is configured to use the '%s' backend, which is currently not supported. "
"Please configure Keras to use 'tensorflow' instead: "
"https://keras.io/getting-started/faq/#where-is-the-keras-configuration-file-stored" % K.backend()
)
def _check_schema(self):
try:
self._validator.check_schema(self._schema)
except jsonschema.exceptions.SchemaError as ex:
six.raise_from(
exceptions.ValidationError('Profile is invalid: %s' % ex), ex)
# *************** Dependencies *********
INSTALL_REQUIRES = [
'decopatch', 'makefun>=1.5', 'funcsigs;python_version<"3.3"', 'six'
]
DEPENDENCY_LINKS = []
SETUP_REQUIRES = ['pytest-runner', 'setuptools_scm', 'six']
TESTS_REQUIRE = ['pytest', 'pytest-logging', 'pandas', 'tabulate']
EXTRAS_REQUIRE = {}
# simple check
try:
from setuptools_scm import get_version
except Exception as e:
raise_from(
Exception(
"Required packages for setup not found. Please install 'setuptools_scm'"
), e)
# ************** ID card *****************
DISTNAME = 'pytest-harvest'
DESCRIPTION = 'Store data created during your pytest tests execution, and retrieve it at the end of the session, ' \
'e.g. for applicative benchmarking purposes.'
MAINTAINER = 'Sylvain MARIE'
MAINTAINER_EMAIL = '*****@*****.**'
URL = 'https://github.com/smarie/python-pytest-harvest'
LICENSE = 'BSD 3-Clause'
LICENSE_LONG = 'License :: OSI Approved :: BSD License'
version_for_download_url = get_version()
DOWNLOAD_URL = URL + '/tarball/' + version_for_download_url
def decode_token(token):
try:
return jwt.decode(token, JWT_SECRET, algorithms=[JWT_ALGORITHM])
except JWTError as e:
six.raise_from(Unauthorized, e)
def ragged_gather(params_nested_splits, params_dense_values, indices, OUTPUT_RAGGED_RANK, name=None):
r"""Gather ragged slices from `params` axis `0` according to `indices`.
Outputs a `RaggedTensor` output composed from `output_dense_values` and
`output_nested_splits`, such that:
```python
output.shape = indices.shape + params.shape[1:]
output.ragged_rank = indices.shape.ndims + params.ragged_rank
output[i...j, d0...dn] = params[indices[i...j], d0...dn]
```
where
* `params =
ragged.from_nested_row_splits(params_dense_values, params_nested_splits)`
provides the values that should be gathered.
* `indices` ia a dense tensor with dtype `int32` or `int64`, indicating which
values should be gathered.
* `output =
ragged.from_nested_row_splits(output_dense_values, output_nested_splits)`
is the output tensor.
(Note: This c++ op is used to implement the higher-level python
`tf.ragged.gather` op, which also supports ragged indices.)
Args:
params_nested_splits: A list of at least 1 `Tensor` objects with the same type in: `int32`, `int64`.
The `nested_row_splits` tensors that define the row-partitioning for the
`params` RaggedTensor input.
params_dense_values: A `Tensor`.
The `flat_values` for the `params` RaggedTensor. There was a terminology change
at the python level from dense_values to flat_values, so dense_values is the
deprecated name.
indices: A `Tensor`. Must be one of the following types: `int32`, `int64`.
Indices in the outermost dimension of `params` of the values that should be
gathered.
OUTPUT_RAGGED_RANK: An `int` that is `>= 0`.
The ragged rank of the output RaggedTensor. `output_nested_splits` will contain
this number of `row_splits` tensors. This value should equal
`indices.shape.ndims + params.ragged_rank - 1`.
name: A name for the operation (optional).
Returns:
A tuple of `Tensor` objects (output_nested_splits, output_dense_values).
output_nested_splits: A list of `OUTPUT_RAGGED_RANK` `Tensor` objects with the same type as `params_nested_splits`.
output_dense_values: A `Tensor`. Has the same type as `params_dense_values`.
"""
_ctx = _context._context or _context.context()
if _ctx is not None and _ctx._thread_local_data.is_eager:
try:
_result = _pywrap_tensorflow.TFE_Py_FastPathExecute(
_ctx._context_handle, _ctx._thread_local_data.device_name,
"RaggedGather", name, _ctx.post_execution_callbacks,
params_nested_splits, params_dense_values, indices,
"OUTPUT_RAGGED_RANK", OUTPUT_RAGGED_RANK)
_result = _RaggedGatherOutput._make(_result)
return _result
except _core._FallbackException:
try:
return ragged_gather_eager_fallback(
params_nested_splits, params_dense_values, indices,
OUTPUT_RAGGED_RANK=OUTPUT_RAGGED_RANK, name=name, ctx=_ctx)
except _core._SymbolicException:
pass # Add nodes to the TensorFlow graph.
except _core._NotOkStatusException as e:
if name is not None:
message = e.message + " name: " + name
else:
message = e.message
_six.raise_from(_core._status_to_exception(e.code, message), None)
# Add nodes to the TensorFlow graph.
if not isinstance(params_nested_splits, (list, tuple)):
raise TypeError(
"Expected list for 'params_nested_splits' argument to "
"'ragged_gather' Op, not %r." % params_nested_splits)
_attr_PARAMS_RAGGED_RANK = len(params_nested_splits)
OUTPUT_RAGGED_RANK = _execute.make_int(OUTPUT_RAGGED_RANK, "OUTPUT_RAGGED_RANK")
_, _, _op = _op_def_lib._apply_op_helper(
"RaggedGather", params_nested_splits=params_nested_splits,
params_dense_values=params_dense_values,
indices=indices,
OUTPUT_RAGGED_RANK=OUTPUT_RAGGED_RANK, name=name)
_result = _op.outputs[:]
_inputs_flat = _op.inputs
_attrs = ("Tvalues", _op._get_attr_type("Tvalues"), "Tindices",
_op._get_attr_type("Tindices"), "Tsplits",
_op._get_attr_type("Tsplits"), "PARAMS_RAGGED_RANK",
_op.get_attr("PARAMS_RAGGED_RANK"), "OUTPUT_RAGGED_RANK",
_op.get_attr("OUTPUT_RAGGED_RANK"))
_execute.record_gradient(
"RaggedGather", _inputs_flat, _attrs, _result, name)
_result = [_result[:OUTPUT_RAGGED_RANK]] + _result[OUTPUT_RAGGED_RANK:]
_result = _RaggedGatherOutput._make(_result)
return _result
def process_input_v4(tree_handle,
stats_handle,
input_data,
sparse_input_indices,
sparse_input_values,
sparse_input_shape,
input_labels,
input_weights,
leaf_ids,
random_seed,
input_spec,
params,
name=None):
r"""Add labels to stats after traversing the tree for each example.
Outputs node ids that are finished.
Args:
tree_handle: A `Tensor` of type `resource`. The handle to the tree.
stats_handle: A `Tensor` of type `resource`. The handle to the stats.
input_data: A `Tensor` of type `float32`.
The training batch's features as a 2-d tensor; `input_data[i][j]`
gives the j-th feature of the i-th input.
sparse_input_indices: A `Tensor` of type `int64`.
The indices tensor from the SparseTensor input.
sparse_input_values: A `Tensor` of type `float32`.
The values tensor from the SparseTensor input.
sparse_input_shape: A `Tensor` of type `int64`.
The shape tensor from the SparseTensor input.
input_labels: A `Tensor` of type `float32`.
The training batch's labels as a 1 or 2-d tensor.
'input_labels[i][j]' gives the j-th label/target for the i-th input.
input_weights: A `Tensor` of type `float32`.
The training batch's weights as a 1-d tensor.
'input_weights[i]' gives the weight for the i-th input.
leaf_ids: A `Tensor` of type `int32`.
`leaf_ids[i]` is the leaf id for input i.
random_seed: An `int`.
input_spec: A `string`.
params: A `string`. A serialized TensorForestParams proto.
name: A name for the operation (optional).
Returns:
A `Tensor` of type `int32`.
A 1-d tensor of node ids that have finished and are ready to
grow.
"""
_ctx = _context._context
if _ctx is None or not _ctx._eager_context.is_eager:
random_seed = _execute.make_int(random_seed, "random_seed")
input_spec = _execute.make_str(input_spec, "input_spec")
params = _execute.make_str(params, "params")
_, _, _op = _op_def_lib._apply_op_helper(
"ProcessInputV4",
tree_handle=tree_handle,
stats_handle=stats_handle,
input_data=input_data,
sparse_input_indices=sparse_input_indices,
sparse_input_values=sparse_input_values,
sparse_input_shape=sparse_input_shape,
input_labels=input_labels,
input_weights=input_weights,
leaf_ids=leaf_ids,
random_seed=random_seed,
input_spec=input_spec,
params=params,
name=name)
_result = _op.outputs[:]
_inputs_flat = _op.inputs
_attrs = ("random_seed", _op.get_attr("random_seed"), "input_spec",
_op.get_attr("input_spec"), "params", _op.get_attr("params"))
_execute.record_gradient("ProcessInputV4", _inputs_flat, _attrs,
_result, name)
_result, = _result
return _result
else:
try:
_result = _pywrap_tensorflow.TFE_Py_FastPathExecute(
_ctx._context_handle, _ctx._eager_context.device_name,
"ProcessInputV4", name, _ctx._post_execution_callbacks,
tree_handle, stats_handle, input_data, sparse_input_indices,
sparse_input_values, sparse_input_shape, input_labels,
input_weights, leaf_ids, "random_seed", random_seed,
"input_spec", input_spec, "params", params)
return _result
except _core._FallbackException:
return process_input_v4_eager_fallback(tree_handle,
stats_handle,
input_data,
sparse_input_indices,
sparse_input_values,
sparse_input_shape,
input_labels,
input_weights,
leaf_ids,
random_seed=random_seed,
input_spec=input_spec,
params=params,
name=name,
ctx=_ctx)
except _core._NotOkStatusException as e:
if name is not None:
message = e.message + " name: " + name
else:
message = e.message
_six.raise_from(_core._status_to_exception(e.code, message), None)
from __future__ import absolute_import
import functools
import logging
import numbers
import os
import time
try:
import requests
except ImportError as caught_exc: # pragma: NO COVER
import six
six.raise_from(
ImportError(
"The requests library is not installed, please install the "
"requests package to use the requests transport."),
caught_exc,
)
import requests.adapters # pylint: disable=ungrouped-imports
import requests.exceptions # pylint: disable=ungrouped-imports
from requests.packages.urllib3.util.ssl_ import (
create_urllib3_context, ) # pylint: disable=ungrouped-imports
import six # pylint: disable=ungrouped-imports
from google.auth import environment_vars
from google.auth import exceptions
from google.auth import transport
import google.auth.transport._mtls_helper
from google.oauth2 import service_account
_LOGGER = logging.getLogger(__name__)
def save(self, file_or_path):
""""Validates and saves this Data Package contents into a zip file.
It creates a zip file into ``file_or_path`` with the contents of this
Data Package and its resources. Every resource which content lives in
the local filesystem will be copied to the zip file. Consider the
following Data Package descriptor::
{
"name": "gdp",
"resources": [
{"name": "local", "format": "CSV", "path": "data.csv"},
{"name": "inline", "data": [4, 8, 15, 16, 23, 42]},
{"name": "remote", "url": "http://someplace.com/data.csv"}
]
}
The final structure of the zip file will be::
./datapackage.json
./data/local.csv
With the contents of `datapackage.json` being the same as returned by
:func:`to_json`.
The resources' file names are generated based on their `name` and
`format` fields if they exist. If the resource has no `name`, it'll be
used `resource-X`, where `X` is the index of the resource in the
`resources` list (starting at zero). If the resource has `format`,
it'll be lowercased and appended to the `name`, becoming
"`name.format`".
Args:
file_or_path (string or file-like object): The file path or a
file-like object where the contents of this Data Package will
be saved into.
Raises:
ValidationError: If the Data Package is invalid.
DataPackageException: If there was some error writing the package.
"""
self.validate()
def arcname(resource):
basename = resource.descriptor.get('name')
resource_format = resource.descriptor.get('format')
if not basename:
index = self.resources.index(resource)
basename = 'resource-{index}'.format(index=index)
if resource_format:
basename = '.'.join([basename, resource_format.lower()])
return os.path.join('data', basename)
try:
with zipfile.ZipFile(file_or_path, 'w') as z:
descriptor = json.loads(self.to_json())
for i, resource in enumerate(self.resources):
path = None
if resource.source_type == 'local':
path = os.path.abspath(resource.source)
if path:
path_inside_dp = arcname(resource)
z.write(path, path_inside_dp)
descriptor['resources'][i]['path'] = path_inside_dp
z.writestr('datapackage.json', json.dumps(descriptor))
except (IOError, zipfile.BadZipfile, zipfile.LargeZipFile) as e:
six.raise_from(exceptions.DataPackageException(e), e)
def __getattr__(self, name):
try:
return self[name]
except KeyError:
raise_from(AttributeError(self.__attr_error__.format(name)), None)
def file_id_from_run_output(name, job_id, run_id, regex=False, client=None):
"""Find the file ID of a File run output with the name "name"
The run output is required to have type "File".
If using an approximate match and multiple names match the
provided string, return only the first file ID.
Parameters
----------
name : str
The "name" field of the run output you wish to retrieve
job_id : int
run_id : int
regex : bool, optional
If False (the default), require an exact string match between
``name`` and the name of the run output. If True, search for a
name which matches the regular expression ``name`` and
retrieve the first found.
client : :class:`civis.APIClient`, optional
If not provided, an :class:`civis.APIClient` object will be
created from the :envvar:`CIVIS_API_KEY`.
Returns
-------
file_id : int
The ID of a Civis File with name matching ``name``
Raises
------
IOError
If the provided job ID and run ID combination can't be found
FileNotFoundError
If the run exists, but ``name`` isn't in its run outputs
See Also
--------
APIClient.scripts.list_containers.runs_outputs
"""
client = APIClient() if client is None else client
# Retrieve run outputs
try:
outputs = client.scripts.list_containers_runs_outputs(job_id, run_id)
except CivisAPIError as err:
if err.status_code == 404:
six.raise_from(IOError('Could not find job/run ID {}/{}'
.format(job_id, run_id)), err)
else:
raise
# Find file in the run outputs.
if not regex:
# Require an exact match on the "name" string.
obj = find_one(outputs, name=name, object_type='File')
else:
# Search for a filename which contains the "name" string
obj_matches = [o for o in outputs
if re.search(name, o.name) and o.object_type == 'File']
if len(obj_matches) > 1:
log.warning('Found %s matches to "%s". Returning the first.',
len(obj_matches), name)
obj = None if not obj_matches else obj_matches[0]
if obj is None:
prefix = "A file containing the pattern" if regex else "File"
raise FileNotFoundError('{} "{}" is not an output of job/run ID '
'{}/{}.'.format(prefix, name, job_id, run_id))
return obj['object_id']
def sparse_feature_cross(indices,
values,
shapes,
dense,
hashed_output,
num_buckets,
out_type,
internal_type,
name=None):
r"""Generates sparse cross form a list of sparse tensors.
The op takes two lists, one of 2D `SparseTensor` and one of 2D `Tensor`, each
representing features of one feature column. It outputs a 2D `SparseTensor` with
the batchwise crosses of these features.
For example, if the inputs are
inputs[0]: SparseTensor with shape = [2, 2]
[0, 0]: "a"
[1, 0]: "b"
[1, 1]: "c"
inputs[1]: SparseTensor with shape = [2, 1]
[0, 0]: "d"
[1, 0]: "e"
inputs[2]: Tensor [["f"], ["g"]]
then the output will be
shape = [2, 2]
[0, 0]: "a_X_d_X_f"
[1, 0]: "b_X_e_X_g"
[1, 1]: "c_X_e_X_g"
if hashed_output=true then the output will be
shape = [2, 2]
[0, 0]: HashCombine(
Fingerprint64("f"), HashCombine(
Fingerprint64("d"), Fingerprint64("a")))
[1, 0]: HashCombine(
Fingerprint64("g"), HashCombine(
Fingerprint64("e"), Fingerprint64("b")))
[1, 1]: HashCombine(
Fingerprint64("g"), HashCombine(
Fingerprint64("e"), Fingerprint64("c")))
Args:
indices: A list of `Tensor` objects with type `int64`.
2-D. Indices of each input `SparseTensor`.
values: A list of `Tensor` objects with types from: `int64`, `string`.
1-D. values of each `SparseTensor`.
shapes: A list with the same length as `indices` of `Tensor` objects with type `int64`.
1-D. Shapes of each `SparseTensor`.
dense: A list of `Tensor` objects with types from: `int64`, `string`.
2-D. Columns represented by dense `Tensor`.
hashed_output: A `bool`.
num_buckets: An `int` that is `>= 0`.
out_type: A `tf.DType` from: `tf.int64, tf.string`.
internal_type: A `tf.DType` from: `tf.int64, tf.string`.
name: A name for the operation (optional).
Returns:
A tuple of `Tensor` objects (output_indices, output_values, output_shape).
output_indices: A `Tensor` of type `int64`. 2-D. Indices of the concatenated `SparseTensor`.
output_values: A `Tensor` of type `out_type`. 1-D. Non-empty values of the concatenated or hashed
`SparseTensor`.
output_shape: A `Tensor` of type `int64`. 1-D. Shape of the concatenated `SparseTensor`.
"""
_ctx = _context._context
if _ctx is None or not _ctx._eager_context.is_eager:
if not isinstance(indices, (list, tuple)):
raise TypeError("Expected list for 'indices' argument to "
"'sparse_feature_cross' Op, not %r." % indices)
_attr_N = len(indices)
if not isinstance(shapes, (list, tuple)):
raise TypeError("Expected list for 'shapes' argument to "
"'sparse_feature_cross' Op, not %r." % shapes)
if len(shapes) != _attr_N:
raise ValueError(
"List argument 'shapes' to 'sparse_feature_cross' Op with length %d "
"must match length %d of argument 'indices'." %
(len(shapes), _attr_N))
hashed_output = _execute.make_bool(hashed_output, "hashed_output")
num_buckets = _execute.make_int(num_buckets, "num_buckets")
out_type = _execute.make_type(out_type, "out_type")
internal_type = _execute.make_type(internal_type, "internal_type")
_, _, _op = _op_def_lib._apply_op_helper("SparseFeatureCross",
indices=indices,
values=values,
shapes=shapes,
dense=dense,
hashed_output=hashed_output,
num_buckets=num_buckets,
out_type=out_type,
internal_type=internal_type,
name=name)
_result = _op.outputs[:]
_inputs_flat = _op.inputs
_attrs = ("N", _op.get_attr("N"), "hashed_output",
_op.get_attr("hashed_output"), "num_buckets",
_op.get_attr("num_buckets"), "sparse_types",
_op.get_attr("sparse_types"), "dense_types",
_op.get_attr("dense_types"), "out_type",
_op.get_attr("out_type"), "internal_type",
_op.get_attr("internal_type"))
_execute.record_gradient("SparseFeatureCross", _inputs_flat, _attrs,
_result, name)
_result = _SparseFeatureCrossOutput._make(_result)
return _result
else:
try:
_result = _pywrap_tensorflow.TFE_Py_FastPathExecute(
_ctx._context_handle, _ctx._eager_context.device_name,
"SparseFeatureCross", name, _ctx._post_execution_callbacks,
indices, values, shapes, dense, "hashed_output", hashed_output,
"num_buckets", num_buckets, "out_type", out_type,
"internal_type", internal_type)
_result = _SparseFeatureCrossOutput._make(_result)
return _result
except _core._FallbackException:
return sparse_feature_cross_eager_fallback(
indices,
values,
shapes,
dense,
hashed_output=hashed_output,
num_buckets=num_buckets,
out_type=out_type,
internal_type=internal_type,
name=name,
ctx=_ctx)
except _core._NotOkStatusException as e:
if name is not None:
message = e.message + " name: " + name
else:
message = e.message
_six.raise_from(_core._status_to_exception(e.code, message), None)
def _load_registry(self):
try:
return datapackage_registry.Registry()
except DataPackageRegistryException as e:
six.raise_from(SchemaError(e), e)
def create_reservation(self, region, availability_zone, reservation):
"""Create a reservation in an Availability Zone.
On success:
Return True, reservation_id
On Failure:
Return False, ''
"""
try:
reservation_translated = {
translate_to_api_names(k): v for k, v in reservation.items()
}
try:
duration = parse_duration(self.ttl)
except Exception as err:
six.raise_from(AnsibleError("could not parse duration {}".format(self.ttl)), err)
if len(self.tags) > 0:
tag_spec =[{
'ResourceType': 'capacity-reservation',
'Tags':self.tags,
}]
else:
tag_spec = []
response = self.clients[region].create_capacity_reservation(
AvailabilityZone=availability_zone,
EndDate=datetime.datetime.utcnow() + duration,
EndDateType='limited',
InstanceMatchCriteria='open',
TagSpecifications=tag_spec,
**reservation_translated
)
check_response(response)
display.vvvv("response: %s" % pp.pformat(response))
display.vvv(pp.pformat(response))
except botocore.exceptions.ClientError as err:
if 'InstanceLimitExceeded' in str(err):
display.display("InstanceLimitExceeded %s - %d * %s"
%(availability_zone,
reservation['instance_count'],
reservation['instance_type']))
return False, ''
if 'InsufficientInstanceCapacity' in str(err):
display.display("InsufficientInstanceCapacity %s - %d * %s"
%(availability_zone, reservation['instance_count'],
reservation['instance_type']))
return False, ''
display.error(pp.pformat(err))
six.raise_from(AnsibleError("Client Error while creating reservation."), err)
if response['ResponseMetadata']['HTTPStatusCode'] == 200:
# if success, continue
r_id = response['CapacityReservation']['CapacityReservationId']
display.display("Reservation created: %s (%s)" %(r_id, availability_zone))
return True, r_id
return False, ''
def block_lstm(seq_len_max, x, cs_prev, h_prev, w, wci, wcf, wco, b, forget_bias=1, cell_clip=3, use_peephole=False, name=None):
r"""Computes the LSTM cell forward propagation for all the time steps.
This is equivalent to applying LSTMBlockCell in a loop, like so:
```python
for x1 in unpack(x):
i1, cs1, f1, o1, ci1, co1, h1 = LSTMBlock(
x1, cs_prev, h_prev, w, wci, wcf, wco, b)
cs_prev = cs1
h_prev = h1
i.append(i1)
cs.append(cs1)
f.append(f1)
o.append(o1)
ci.append(ci1)
co.append(co1)
h.append(h1)
return pack(i), pack(cs), pack(f), pack(o), pack(ci), pack(ch), pack(h)
```
Args:
seq_len_max: A `Tensor` of type `int64`.
Maximum time length actually used by this input. Outputs are padded
with zeros beyond this length.
x: A `Tensor`. Must be one of the following types: `half`, `float32`.
The sequence input to the LSTM, shape (timelen, batch_size, num_inputs).
cs_prev: A `Tensor`. Must have the same type as `x`.
Value of the initial cell state.
h_prev: A `Tensor`. Must have the same type as `x`.
Initial output of cell (to be used for peephole).
w: A `Tensor`. Must have the same type as `x`. The weight matrix.
wci: A `Tensor`. Must have the same type as `x`.
The weight matrix for input gate peephole connection.
wcf: A `Tensor`. Must have the same type as `x`.
The weight matrix for forget gate peephole connection.
wco: A `Tensor`. Must have the same type as `x`.
The weight matrix for output gate peephole connection.
b: A `Tensor`. Must have the same type as `x`. The bias vector.
forget_bias: An optional `float`. Defaults to `1`. The forget gate bias.
cell_clip: An optional `float`. Defaults to `3`.
Value to clip the 'cs' value to.
use_peephole: An optional `bool`. Defaults to `False`.
Whether to use peephole weights.
name: A name for the operation (optional).
Returns:
A tuple of `Tensor` objects (i, cs, f, o, ci, co, h).
i: A `Tensor`. Has the same type as `x`. The input gate over the whole time sequence.
cs: A `Tensor`. Has the same type as `x`. The cell state before the tanh over the whole time sequence.
f: A `Tensor`. Has the same type as `x`. The forget gate over the whole time sequence.
o: A `Tensor`. Has the same type as `x`. The output gate over the whole time sequence.
ci: A `Tensor`. Has the same type as `x`. The cell input over the whole time sequence.
co: A `Tensor`. Has the same type as `x`. The cell after the tanh over the whole time sequence.
h: A `Tensor`. Has the same type as `x`. The output h vector over the whole time sequence.
"""
_ctx = _context._context
if _ctx is not None and _ctx._eager_context.is_eager:
try:
_result = _pywrap_tensorflow.TFE_Py_FastPathExecute(
_ctx._context_handle, _ctx._eager_context.device_name, "BlockLSTM",
name, _ctx._post_execution_callbacks, seq_len_max, x, cs_prev, h_prev,
w, wci, wcf, wco, b, "forget_bias", forget_bias, "cell_clip",
cell_clip, "use_peephole", use_peephole)
_result = _BlockLSTMOutput._make(_result)
return _result
except _core._FallbackException:
try:
return block_lstm_eager_fallback(
seq_len_max, x, cs_prev, h_prev, w, wci, wcf, wco, b,
forget_bias=forget_bias, cell_clip=cell_clip,
use_peephole=use_peephole, name=name, ctx=_ctx)
except _core._SymbolicException:
pass # Add nodes to the TensorFlow graph.
except (TypeError, ValueError):
result = _dispatch.dispatch(
block_lstm, seq_len_max=seq_len_max, x=x, cs_prev=cs_prev,
h_prev=h_prev, w=w, wci=wci, wcf=wcf, wco=wco, b=b,
forget_bias=forget_bias, cell_clip=cell_clip,
use_peephole=use_peephole, name=name)
if result is not _dispatch.OpDispatcher.NOT_SUPPORTED:
return result
raise
except _core._NotOkStatusException as e:
if name is not None:
message = e.message + " name: " + name
else:
message = e.message
_six.raise_from(_core._status_to_exception(e.code, message), None)
# Add nodes to the TensorFlow graph.
if forget_bias is None:
forget_bias = 1
forget_bias = _execute.make_float(forget_bias, "forget_bias")
if cell_clip is None:
cell_clip = 3
cell_clip = _execute.make_float(cell_clip, "cell_clip")
if use_peephole is None:
use_peephole = False
use_peephole = _execute.make_bool(use_peephole, "use_peephole")
try:
_, _, _op = _op_def_lib._apply_op_helper(
"BlockLSTM", seq_len_max=seq_len_max, x=x, cs_prev=cs_prev,
h_prev=h_prev, w=w, wci=wci, wcf=wcf, wco=wco, b=b,
forget_bias=forget_bias, cell_clip=cell_clip,
use_peephole=use_peephole, name=name)
except (TypeError, ValueError):
result = _dispatch.dispatch(
block_lstm, seq_len_max=seq_len_max, x=x, cs_prev=cs_prev,
h_prev=h_prev, w=w, wci=wci, wcf=wcf, wco=wco, b=b,
forget_bias=forget_bias, cell_clip=cell_clip,
use_peephole=use_peephole, name=name)
if result is not _dispatch.OpDispatcher.NOT_SUPPORTED:
return result
raise
_result = _op.outputs[:]
_inputs_flat = _op.inputs
_attrs = ("forget_bias", _op.get_attr("forget_bias"), "cell_clip",
_op.get_attr("cell_clip"), "use_peephole",
_op.get_attr("use_peephole"), "T", _op.get_attr("T"))
_execute.record_gradient(
"BlockLSTM", _inputs_flat, _attrs, _result, name)
_result = _BlockLSTMOutput._make(_result)
return _result
def _check_schema(self):
try:
self._validator.check_schema(self._schema)
except jsonschema.exceptions.SchemaError as e:
six.raise_from(SchemaError.create_from(e), e)
def block_lstm_grad(seq_len_max, x, cs_prev, h_prev, w, wci, wcf, wco, b, i, cs, f, o, ci, co, h, cs_grad, h_grad, use_peephole, name=None):
r"""Computes the LSTM cell backward propagation for the entire time sequence.
This implementation is to be used in conjunction of LSTMBlock.
Args:
seq_len_max: A `Tensor` of type `int64`.
Maximum time length actually used by this input. Outputs are padded
with zeros beyond this length.
x: A `Tensor`. Must be one of the following types: `half`, `float32`.
The sequence input to the LSTM, shape (timelen, batch_size, num_inputs).
cs_prev: A `Tensor`. Must have the same type as `x`.
Value of the initial cell state.
h_prev: A `Tensor`. Must have the same type as `x`.
Initial output of cell (to be used for peephole).
w: A `Tensor`. Must have the same type as `x`. The weight matrix.
wci: A `Tensor`. Must have the same type as `x`.
The weight matrix for input gate peephole connection.
wcf: A `Tensor`. Must have the same type as `x`.
The weight matrix for forget gate peephole connection.
wco: A `Tensor`. Must have the same type as `x`.
The weight matrix for output gate peephole connection.
b: A `Tensor`. Must have the same type as `x`. The bias vector.
i: A `Tensor`. Must have the same type as `x`.
The input gate over the whole time sequence.
cs: A `Tensor`. Must have the same type as `x`.
The cell state before the tanh over the whole time sequence.
f: A `Tensor`. Must have the same type as `x`.
The forget gate over the whole time sequence.
o: A `Tensor`. Must have the same type as `x`.
The output gate over the whole time sequence.
ci: A `Tensor`. Must have the same type as `x`.
The cell input over the whole time sequence.
co: A `Tensor`. Must have the same type as `x`.
The cell after the tanh over the whole time sequence.
h: A `Tensor`. Must have the same type as `x`.
The output h vector over the whole time sequence.
cs_grad: A `Tensor`. Must have the same type as `x`.
The current gradient of cs.
h_grad: A `Tensor`. Must have the same type as `x`.
The gradient of h vector.
use_peephole: A `bool`. Whether to use peephole weights.
name: A name for the operation (optional).
Returns:
A tuple of `Tensor` objects (x_grad, cs_prev_grad, h_prev_grad, w_grad, wci_grad, wcf_grad, wco_grad, b_grad).
x_grad: A `Tensor`. Has the same type as `x`. The gradient of x to be back-propped.
cs_prev_grad: A `Tensor`. Has the same type as `x`. The gradient of cs_prev to be back-propped.
h_prev_grad: A `Tensor`. Has the same type as `x`. The gradient of h_prev to be back-propped.
w_grad: A `Tensor`. Has the same type as `x`. The gradient for w to be back-propped.
wci_grad: A `Tensor`. Has the same type as `x`. The gradient for wci to be back-propped.
wcf_grad: A `Tensor`. Has the same type as `x`. The gradient for wcf to be back-propped.
wco_grad: A `Tensor`. Has the same type as `x`. The gradient for wco to be back-propped.
b_grad: A `Tensor`. Has the same type as `x`. The gradient for w to be back-propped.
"""
_ctx = _context._context
if _ctx is not None and _ctx._eager_context.is_eager:
try:
_result = _pywrap_tensorflow.TFE_Py_FastPathExecute(
_ctx._context_handle, _ctx._eager_context.device_name,
"BlockLSTMGrad", name, _ctx._post_execution_callbacks, seq_len_max, x,
cs_prev, h_prev, w, wci, wcf, wco, b, i, cs, f, o, ci, co, h, cs_grad,
h_grad, "use_peephole", use_peephole)
_result = _BlockLSTMGradOutput._make(_result)
return _result
except _core._FallbackException:
try:
return block_lstm_grad_eager_fallback(
seq_len_max, x, cs_prev, h_prev, w, wci, wcf, wco, b, i, cs, f, o,
ci, co, h, cs_grad, h_grad, use_peephole=use_peephole, name=name,
ctx=_ctx)
except _core._SymbolicException:
pass # Add nodes to the TensorFlow graph.
except (TypeError, ValueError):
result = _dispatch.dispatch(
block_lstm_grad, seq_len_max=seq_len_max, x=x, cs_prev=cs_prev,
h_prev=h_prev, w=w, wci=wci, wcf=wcf, wco=wco,
b=b, i=i, cs=cs, f=f, o=o, ci=ci, co=co, h=h,
cs_grad=cs_grad, h_grad=h_grad,
use_peephole=use_peephole, name=name)
if result is not _dispatch.OpDispatcher.NOT_SUPPORTED:
return result
raise
except _core._NotOkStatusException as e:
if name is not None:
message = e.message + " name: " + name
else:
message = e.message
_six.raise_from(_core._status_to_exception(e.code, message), None)
# Add nodes to the TensorFlow graph.
use_peephole = _execute.make_bool(use_peephole, "use_peephole")
try:
_, _, _op = _op_def_lib._apply_op_helper(
"BlockLSTMGrad", seq_len_max=seq_len_max, x=x, cs_prev=cs_prev,
h_prev=h_prev, w=w, wci=wci, wcf=wcf, wco=wco, b=b,
i=i, cs=cs, f=f, o=o, ci=ci, co=co, h=h,
cs_grad=cs_grad, h_grad=h_grad,
use_peephole=use_peephole, name=name)
except (TypeError, ValueError):
result = _dispatch.dispatch(
block_lstm_grad, seq_len_max=seq_len_max, x=x, cs_prev=cs_prev,
h_prev=h_prev, w=w, wci=wci, wcf=wcf, wco=wco, b=b,
i=i, cs=cs, f=f, o=o, ci=ci, co=co, h=h,
cs_grad=cs_grad, h_grad=h_grad,
use_peephole=use_peephole, name=name)
if result is not _dispatch.OpDispatcher.NOT_SUPPORTED:
return result
raise
_result = _op.outputs[:]
_inputs_flat = _op.inputs
_attrs = ("use_peephole", _op.get_attr("use_peephole"), "T",
_op.get_attr("T"))
_execute.record_gradient(
"BlockLSTMGrad", _inputs_flat, _attrs, _result, name)
_result = _BlockLSTMGradOutput._make(_result)
return _result
def lstm_block_cell_grad(x, cs_prev, h_prev, w, wci, wcf, wco, b, i, cs, f, o, ci, co, cs_grad, h_grad, use_peephole, name=None):
r"""Computes the LSTM cell backward propagation for 1 timestep.
This implementation is to be used in conjunction of LSTMBlockCell.
Args:
x: A `Tensor`. Must be one of the following types: `half`, `float32`.
The input to the LSTM cell, shape (batch_size, num_inputs).
cs_prev: A `Tensor`. Must have the same type as `x`.
The previous cell state.
h_prev: A `Tensor`. Must have the same type as `x`. The previous h state.
w: A `Tensor`. Must have the same type as `x`. The weight matrix.
wci: A `Tensor`. Must have the same type as `x`.
The weight matrix for input gate peephole connection.
wcf: A `Tensor`. Must have the same type as `x`.
The weight matrix for forget gate peephole connection.
wco: A `Tensor`. Must have the same type as `x`.
The weight matrix for output gate peephole connection.
b: A `Tensor`. Must have the same type as `x`. The bias vector.
i: A `Tensor`. Must have the same type as `x`. The input gate.
cs: A `Tensor`. Must have the same type as `x`.
The cell state before the tanh.
f: A `Tensor`. Must have the same type as `x`. The forget gate.
o: A `Tensor`. Must have the same type as `x`. The output gate.
ci: A `Tensor`. Must have the same type as `x`. The cell input.
co: A `Tensor`. Must have the same type as `x`. The cell after the tanh.
cs_grad: A `Tensor`. Must have the same type as `x`.
The current gradient of cs.
h_grad: A `Tensor`. Must have the same type as `x`.
The gradient of h vector.
use_peephole: A `bool`. Whether the cell uses peephole connections.
name: A name for the operation (optional).
Returns:
A tuple of `Tensor` objects (cs_prev_grad, dicfo, wci_grad, wcf_grad, wco_grad).
cs_prev_grad: A `Tensor`. Has the same type as `x`. The gradient of cs to be back-propped.
dicfo: A `Tensor`. Has the same type as `x`. The derivative wrt to [i, cs, f, o].
wci_grad: A `Tensor`. Has the same type as `x`. The gradient for wci to be back-propped.
wcf_grad: A `Tensor`. Has the same type as `x`. The gradient for wcf to be back-propped.
wco_grad: A `Tensor`. Has the same type as `x`. The gradient for wco to be back-propped.
"""
_ctx = _context._context
if _ctx is not None and _ctx._eager_context.is_eager:
try:
_result = _pywrap_tensorflow.TFE_Py_FastPathExecute(
_ctx._context_handle, _ctx._eager_context.device_name,
"LSTMBlockCellGrad", name, _ctx._post_execution_callbacks, x, cs_prev,
h_prev, w, wci, wcf, wco, b, i, cs, f, o, ci, co, cs_grad, h_grad,
"use_peephole", use_peephole)
_result = _LSTMBlockCellGradOutput._make(_result)
return _result
except _core._FallbackException:
try:
return lstm_block_cell_grad_eager_fallback(
x, cs_prev, h_prev, w, wci, wcf, wco, b, i, cs, f, o, ci, co,
cs_grad, h_grad, use_peephole=use_peephole, name=name, ctx=_ctx)
except _core._SymbolicException:
pass # Add nodes to the TensorFlow graph.
except (TypeError, ValueError):
result = _dispatch.dispatch(
lstm_block_cell_grad, x=x, cs_prev=cs_prev, h_prev=h_prev, w=w,
wci=wci, wcf=wcf, wco=wco, b=b, i=i,
cs=cs, f=f, o=o, ci=ci, co=co,
cs_grad=cs_grad, h_grad=h_grad,
use_peephole=use_peephole, name=name)
if result is not _dispatch.OpDispatcher.NOT_SUPPORTED:
return result
raise
except _core._NotOkStatusException as e:
if name is not None:
message = e.message + " name: " + name
else:
message = e.message
_six.raise_from(_core._status_to_exception(e.code, message), None)
# Add nodes to the TensorFlow graph.
use_peephole = _execute.make_bool(use_peephole, "use_peephole")
try:
_, _, _op = _op_def_lib._apply_op_helper(
"LSTMBlockCellGrad", x=x, cs_prev=cs_prev, h_prev=h_prev, w=w,
wci=wci, wcf=wcf, wco=wco, b=b, i=i, cs=cs, f=f,
o=o, ci=ci, co=co, cs_grad=cs_grad,
h_grad=h_grad, use_peephole=use_peephole,
name=name)
except (TypeError, ValueError):
result = _dispatch.dispatch(
lstm_block_cell_grad, x=x, cs_prev=cs_prev, h_prev=h_prev, w=w,
wci=wci, wcf=wcf, wco=wco, b=b, i=i, cs=cs,
f=f, o=o, ci=ci, co=co, cs_grad=cs_grad,
h_grad=h_grad, use_peephole=use_peephole,
name=name)
if result is not _dispatch.OpDispatcher.NOT_SUPPORTED:
return result
raise
_result = _op.outputs[:]
_inputs_flat = _op.inputs
_attrs = ("use_peephole", _op.get_attr("use_peephole"), "T",
_op.get_attr("T"))
_execute.record_gradient(
"LSTMBlockCellGrad", _inputs_flat, _attrs, _result, name)
_result = _LSTMBlockCellGradOutput._make(_result)
return _result
def init(
job_type=None,
dir=None,
config=None, # TODO(jhr): type is a union for argparse/absl
project=None,
entity=None,
reinit=None,
tags=None,
group=None,
name=None,
notes=None,
magic=None, # TODO(jhr): type is union
config_exclude_keys=None,
config_include_keys=None,
anonymous=None,
mode=None,
allow_val_change=None,
resume=None,
force=None,
tensorboard=None, # alias for sync_tensorboard
sync_tensorboard=None,
monitor_gym=None,
save_code=None,
id=None,
settings=None,
):
"""Initialize a wandb Run.
Args:
entity: alias for team.
team: personal user or team to use for Run.
project: project name for the Run.
Raises:
Exception: if problem.
Returns:
wandb Run object
"""
assert not wandb._IS_INTERNAL_PROCESS
kwargs = locals()
error_seen = None
except_exit = None
try:
wi = _WandbInit()
wi.setup(kwargs)
except_exit = wi.settings._except_exit
try:
run = wi.init()
except_exit = wi.settings._except_exit
except (KeyboardInterrupt, Exception) as e:
if not isinstance(e, KeyboardInterrupt):
sentry_exc(e)
getcaller()
assert logger
if wi.settings.problem == "fatal":
raise
if wi.settings.problem == "warn":
pass
# TODO(jhr): figure out how to make this RunDummy
run = None
except UsageError:
raise
except KeyboardInterrupt as e:
assert logger
logger.warning("interrupted", exc_info=e)
six.raise_from(Exception("interrupted"), e)
except Exception as e:
error_seen = e
traceback.print_exc()
assert logger
logger.error("error", exc_info=e)
# Need to build delay into this sentry capture because our exit hooks
# mess with sentry's ability to send out errors before the program ends.
sentry_exc(e, delay=True)
# reraise(*sys.exc_info())
# six.raise_from(Exception("problem"), e)
finally:
if error_seen:
wandb.termerror("Abnormal program exit")
if except_exit:
os._exit(-1)
six.raise_from(Exception("problem"), error_seen)
return run
def lstm_block_cell(x, cs_prev, h_prev, w, wci, wcf, wco, b, forget_bias=1, cell_clip=3, use_peephole=False, name=None):
r"""Computes the LSTM cell forward propagation for 1 time step.
This implementation uses 1 weight matrix and 1 bias vector, and there's an
optional peephole connection.
This kernel op implements the following mathematical equations:
```python
xh = [x, h_prev]
[i, f, ci, o] = xh * w + b
f = f + forget_bias
if not use_peephole:
wci = wcf = wco = 0
i = sigmoid(cs_prev * wci + i)
f = sigmoid(cs_prev * wcf + f)
ci = tanh(ci)
cs = ci .* i + cs_prev .* f
cs = clip(cs, cell_clip)
o = sigmoid(cs * wco + o)
co = tanh(cs)
h = co .* o
```
Args:
x: A `Tensor`. Must be one of the following types: `half`, `float32`.
The input to the LSTM cell, shape (batch_size, num_inputs).
cs_prev: A `Tensor`. Must have the same type as `x`.
Value of the cell state at previous time step.
h_prev: A `Tensor`. Must have the same type as `x`.
Output of the previous cell at previous time step.
w: A `Tensor`. Must have the same type as `x`. The weight matrix.
wci: A `Tensor`. Must have the same type as `x`.
The weight matrix for input gate peephole connection.
wcf: A `Tensor`. Must have the same type as `x`.
The weight matrix for forget gate peephole connection.
wco: A `Tensor`. Must have the same type as `x`.
The weight matrix for output gate peephole connection.
b: A `Tensor`. Must have the same type as `x`. The bias vector.
forget_bias: An optional `float`. Defaults to `1`. The forget gate bias.
cell_clip: An optional `float`. Defaults to `3`.
Value to clip the 'cs' value to.
use_peephole: An optional `bool`. Defaults to `False`.
Whether to use peephole weights.
name: A name for the operation (optional).
Returns:
A tuple of `Tensor` objects (i, cs, f, o, ci, co, h).
i: A `Tensor`. Has the same type as `x`. The input gate.
cs: A `Tensor`. Has the same type as `x`. The cell state before the tanh.
f: A `Tensor`. Has the same type as `x`. The forget gate.
o: A `Tensor`. Has the same type as `x`. The output gate.
ci: A `Tensor`. Has the same type as `x`. The cell input.
co: A `Tensor`. Has the same type as `x`. The cell after the tanh.
h: A `Tensor`. Has the same type as `x`. The output h vector.
"""
_ctx = _context._context
if _ctx is not None and _ctx._eager_context.is_eager:
try:
_result = _pywrap_tensorflow.TFE_Py_FastPathExecute(
_ctx._context_handle, _ctx._eager_context.device_name,
"LSTMBlockCell", name, _ctx._post_execution_callbacks, x, cs_prev,
h_prev, w, wci, wcf, wco, b, "forget_bias", forget_bias, "cell_clip",
cell_clip, "use_peephole", use_peephole)
_result = _LSTMBlockCellOutput._make(_result)
return _result
except _core._FallbackException:
try:
return lstm_block_cell_eager_fallback(
x, cs_prev, h_prev, w, wci, wcf, wco, b, forget_bias=forget_bias,
cell_clip=cell_clip, use_peephole=use_peephole, name=name,
ctx=_ctx)
except _core._SymbolicException:
pass # Add nodes to the TensorFlow graph.
except (TypeError, ValueError):
result = _dispatch.dispatch(
lstm_block_cell, x=x, cs_prev=cs_prev, h_prev=h_prev, w=w,
wci=wci, wcf=wcf, wco=wco, b=b,
forget_bias=forget_bias, cell_clip=cell_clip,
use_peephole=use_peephole, name=name)
if result is not _dispatch.OpDispatcher.NOT_SUPPORTED:
return result
raise
except _core._NotOkStatusException as e:
if name is not None:
message = e.message + " name: " + name
else:
message = e.message
_six.raise_from(_core._status_to_exception(e.code, message), None)
# Add nodes to the TensorFlow graph.
if forget_bias is None:
forget_bias = 1
forget_bias = _execute.make_float(forget_bias, "forget_bias")
if cell_clip is None:
cell_clip = 3
cell_clip = _execute.make_float(cell_clip, "cell_clip")
if use_peephole is None:
use_peephole = False
use_peephole = _execute.make_bool(use_peephole, "use_peephole")
try:
_, _, _op = _op_def_lib._apply_op_helper(
"LSTMBlockCell", x=x, cs_prev=cs_prev, h_prev=h_prev, w=w, wci=wci,
wcf=wcf, wco=wco, b=b, forget_bias=forget_bias,
cell_clip=cell_clip, use_peephole=use_peephole,
name=name)
except (TypeError, ValueError):
result = _dispatch.dispatch(
lstm_block_cell, x=x, cs_prev=cs_prev, h_prev=h_prev, w=w, wci=wci,
wcf=wcf, wco=wco, b=b, forget_bias=forget_bias,
cell_clip=cell_clip, use_peephole=use_peephole,
name=name)
if result is not _dispatch.OpDispatcher.NOT_SUPPORTED:
return result
raise
_result = _op.outputs[:]
_inputs_flat = _op.inputs
_attrs = ("forget_bias", _op.get_attr("forget_bias"), "cell_clip",
_op.get_attr("cell_clip"), "use_peephole",
_op.get_attr("use_peephole"), "T", _op.get_attr("T"))
_execute.record_gradient(
"LSTMBlockCell", _inputs_flat, _attrs, _result, name)
_result = _LSTMBlockCellOutput._make(_result)
return _result
def stochastic_hard_routing_gradient(input_data,
tree_parameters,
tree_biases,
path_probability,
path,
tree_depth,
name=None):
r""" Computes the derivative of the routing loss with respect to each decision
node.
tree_depth: The depth of the decision tree.
input_data: The training batch's features as a 2-d tensor; `input_data[i][j]`
gives the j-th feature of the i-th input
tree_parameters: `tree_parameters[i]` gives the weight of
the logistic regression model that translates from node features to
probabilities.
tree_biases: `tree_biases[i]` gives the bias of the logistic
regression model that translates from node features to
probabilities.
path_probility: `path_probability[i]` gives the probability of reaching each
node in `path[i]`.
path: `path[i][j]` gives the jth node in the path taken by the ith data
instance.
routing_gradient: `routing_gradient` provides du / df, where u is the routing
function and f is the (vector of) decision functions. A decision function
f_i computes the routing decision at node i.
data_gradient: `data_gradient` provides df / dx, where f is the (vector
of) decision functions and x is a batch of data.
parameter_gradient: `parameter_gradient` provides df / dw, where f is the
(vector of) decision functions and w is the matrix of parameters that
determine how instances are routed through a tree.
bias_gradient: `bias_gradient` provides df / db, where f is the
(vector of) decision functions and b is the vector of bias parameters that
determine how instances are routed through a tree.
f_i is parameterized by t_i (parameters) and b_i (bias) and takes data x as
input. This op is called in training_ops.py to compute du / df, and we use
that to compute
du / dx = du / df * df / dx,
du / dt = du / df * df / dt, and
du / db = du / df * df / db.
Args:
input_data: A `Tensor` of type `float32`.
tree_parameters: A `Tensor` of type `float32`.
tree_biases: A `Tensor` of type `float32`.
path_probability: A `Tensor` of type `float32`.
path: A `Tensor` of type `int32`.
tree_depth: An `int`.
name: A name for the operation (optional).
Returns:
A tuple of `Tensor` objects (routing_gradient, data_gradient, parameter_gradient, bias_gradient).
routing_gradient: A `Tensor` of type `float32`.
data_gradient: A `Tensor` of type `float32`.
parameter_gradient: A `Tensor` of type `float32`.
bias_gradient: A `Tensor` of type `float32`.
"""
_ctx = _context._context
if _ctx is None or not _ctx._eager_context.is_eager:
tree_depth = _execute.make_int(tree_depth, "tree_depth")
_, _, _op = _op_def_lib._apply_op_helper(
"StochasticHardRoutingGradient",
input_data=input_data,
tree_parameters=tree_parameters,
tree_biases=tree_biases,
path_probability=path_probability,
path=path,
tree_depth=tree_depth,
name=name)
_result = _op.outputs[:]
_inputs_flat = _op.inputs
_attrs = ("tree_depth", _op.get_attr("tree_depth"))
_execute.record_gradient("StochasticHardRoutingGradient", _inputs_flat,
_attrs, _result, name)
_result = _StochasticHardRoutingGradientOutput._make(_result)
return _result
else:
try:
_result = _pywrap_tensorflow.TFE_Py_FastPathExecute(
_ctx._context_handle, _ctx._eager_context.device_name,
"StochasticHardRoutingGradient", name,
_ctx._post_execution_callbacks, input_data, tree_parameters,
tree_biases, path_probability, path, "tree_depth", tree_depth)
_result = _StochasticHardRoutingGradientOutput._make(_result)
return _result
except _core._FallbackException:
return stochastic_hard_routing_gradient_eager_fallback(
input_data,
tree_parameters,
tree_biases,
path_probability,
path,
tree_depth=tree_depth,
name=name,
ctx=_ctx)
except _core._NotOkStatusException as e:
if name is not None:
message = e.message + " name: " + name
else:
message = e.message
_six.raise_from(_core._status_to_exception(e.code, message), None)
def roll(input, shift, axis, name=None):
r"""Rolls the elements of a tensor along an axis.
The elements are shifted positively (towards larger indices) by the offset of
`shift` along the dimension of `axis`. Negative `shift` values will shift
elements in the opposite direction. Elements that roll passed the last position
will wrap around to the first and vice versa. Multiple shifts along multiple
axes may be specified.
For example:
```
# 't' is [0, 1, 2, 3, 4]
roll(t, shift=2, axis=0) ==> [3, 4, 0, 1, 2]
# shifting along multiple dimensions
# 't' is [[0, 1, 2, 3, 4], [5, 6, 7, 8, 9]]
roll(t, shift=[1, -2], axis=[0, 1]) ==> [[7, 8, 9, 5, 6], [2, 3, 4, 0, 1]]
# shifting along the same axis multiple times
# 't' is [[0, 1, 2, 3, 4], [5, 6, 7, 8, 9]]
roll(t, shift=[2, -3], axis=[1, 1]) ==> [[1, 2, 3, 4, 0], [6, 7, 8, 9, 5]]
```
Args:
input: A `Tensor`.
shift: A `Tensor`. Must be one of the following types: `int32`, `int64`.
Dimension must be 0-D or 1-D. `shift[i]` specifies the number of places by which
elements are shifted positively (towards larger indices) along the dimension
specified by `axis[i]`. Negative shifts will roll the elements in the opposite
direction.
axis: A `Tensor`. Must be one of the following types: `int32`, `int64`.
Dimension must be 0-D or 1-D. `axis[i]` specifies the dimension that the shift
`shift[i]` should occur. If the same axis is referenced more than once, the
total shift for that axis will be the sum of all the shifts that belong to that
axis.
name: A name for the operation (optional).
Returns:
A `Tensor`. Has the same type as `input`.
"""
_ctx = _context._context or _context.context()
if _ctx is not None and _ctx._thread_local_data.is_eager:
try:
_result = _pywrap_tensorflow.TFE_Py_FastPathExecute(
_ctx._context_handle, _ctx._thread_local_data.device_name,
"Roll", name, _ctx._post_execution_callbacks, input, shift,
axis)
return _result
except _core._FallbackException:
try:
return roll_eager_fallback(input,
shift,
axis,
name=name,
ctx=_ctx)
except _core._SymbolicException:
pass # Add nodes to the TensorFlow graph.
except _core._NotOkStatusException as e:
if name is not None:
message = e.message + " name: " + name
else:
message = e.message
_six.raise_from(_core._status_to_exception(e.code, message), None)
# Add nodes to the TensorFlow graph.
_, _, _op = _op_def_lib._apply_op_helper("Roll",
input=input,
shift=shift,
axis=axis,
name=name)
_result = _op.outputs[:]
_inputs_flat = _op.inputs
_attrs = ("T", _op.get_attr("T"), "Tshift", _op.get_attr("Tshift"),
"Taxis", _op.get_attr("Taxis"))
_execute.record_gradient("Roll", _inputs_flat, _attrs, _result, name)
_result, = _result
return _result
def routing_gradient(input_data,
tree_parameters,
tree_biases,
routes,
max_nodes,
name=None):
r""" Computes the derivative of the routing loss with respect to each decision
node.
max_nodes: The number of nodes in the tree.
tree_parameters: `tree_parameters[i]` gives the weight of
the logistic regression model that translates from node features to
probabilities.
tree_biases: `tree_biases[i]` gives the bias of the logistic
regression model that translates from node features to
probabilities.
routes: The routes computed by routing_function_op.
routing_gradient: `routing_gradient` provides du / df, where u is the routing
function and f is the (vector of) decision functions. A decision function
f_i computes the routing decision at node i.
f_i is parameterized by t_i (parameters) and b_i (bias) and takes data x as
input. This op is called in training_ops.py to compute du / df, and we use
that to compute
du / dx = du / df * df / dx,
du / dt = du / df * df / dt, and
du / db = du / df * df / db.
Args:
input_data: A `Tensor` of type `float32`.
tree_parameters: A `Tensor` of type `float32`.
tree_biases: A `Tensor` of type `float32`.
routes: A `Tensor` of type `float32`.
max_nodes: An `int`.
name: A name for the operation (optional).
Returns:
A `Tensor` of type `float32`.
"""
_ctx = _context._context
if _ctx is None or not _ctx._eager_context.is_eager:
max_nodes = _execute.make_int(max_nodes, "max_nodes")
_, _, _op = _op_def_lib._apply_op_helper(
"RoutingGradient",
input_data=input_data,
tree_parameters=tree_parameters,
tree_biases=tree_biases,
routes=routes,
max_nodes=max_nodes,
name=name)
_result = _op.outputs[:]
_inputs_flat = _op.inputs
_attrs = ("max_nodes", _op.get_attr("max_nodes"))
_execute.record_gradient("RoutingGradient", _inputs_flat, _attrs,
_result, name)
_result, = _result
return _result
else:
try:
_result = _pywrap_tensorflow.TFE_Py_FastPathExecute(
_ctx._context_handle, _ctx._eager_context.device_name,
"RoutingGradient", name, _ctx._post_execution_callbacks,
input_data, tree_parameters, tree_biases, routes, "max_nodes",
max_nodes)
return _result
except _core._FallbackException:
return routing_gradient_eager_fallback(input_data,
tree_parameters,
tree_biases,
routes,
max_nodes=max_nodes,
name=name,
ctx=_ctx)
except _core._NotOkStatusException as e:
if name is not None:
message = e.message + " name: " + name
else:
message = e.message
_six.raise_from(_core._status_to_exception(e.code, message), None)
def adjust_hsv_in_yiq(images, delta_h, scale_s, scale_v, name=None):
r"""Adjust the YIQ hue of one or more images.
`images` is a tensor of at least 3 dimensions. The last dimension is
interpreted as channels, and must be three.
We used linear transformation described in:
beesbuzz.biz/code/hsv_color_transforms.php
The input image is considered in the RGB colorspace. Conceptually, the RGB
colors are first mapped into YIQ space, rotated around the Y channel by
delta_h in radians, multiplying the chrominance channels (I, Q) by scale_s,
multiplying all channels (Y, I, Q) by scale_v, and then remapped back to RGB
colorspace. Each operation described above is a linear transformation.
Args:
images: A `Tensor`. Must be one of the following types: `uint8`, `int8`, `int16`, `int32`, `int64`, `half`, `float32`, `float64`.
Images to adjust. At least 3-D.
delta_h: A `Tensor` of type `float32`.
A float scale that represents the hue rotation amount, in radians.
Although delta_h can be any float value.
scale_s: A `Tensor` of type `float32`.
A float scale that represents the factor to multiply the saturation by.
scale_s needs to be non-negative.
scale_v: A `Tensor` of type `float32`.
A float scale that represents the factor to multiply the value by.
scale_v needs to be non-negative.
name: A name for the operation (optional).
Returns:
A `Tensor`. Has the same type as `images`.
The hsv-adjusted image or images. No clipping will be done in this op.
The client can clip them using additional ops in their graph.
"""
_ctx = _context._context
if _ctx is not None and _ctx._eager_context.is_eager:
try:
_result = _pywrap_tensorflow.TFE_Py_FastPathExecute(
_ctx._context_handle, _ctx._eager_context.device_name,
"AdjustHsvInYiq", name, _ctx._post_execution_callbacks, images,
delta_h, scale_s, scale_v)
return _result
except _core._FallbackException:
try:
return adjust_hsv_in_yiq_eager_fallback(
images, delta_h, scale_s, scale_v, name=name, ctx=_ctx)
except _core._SymbolicException:
pass # Add nodes to the TensorFlow graph.
except (TypeError, ValueError):
result = _dispatch.dispatch(
adjust_hsv_in_yiq, images=images, delta_h=delta_h,
scale_s=scale_s, scale_v=scale_v, name=name)
if result is not _dispatch.OpDispatcher.NOT_SUPPORTED:
return result
raise
except _core._NotOkStatusException as e:
if name is not None:
message = e.message + " name: " + name
else:
message = e.message
_six.raise_from(_core._status_to_exception(e.code, message), None)
# Add nodes to the TensorFlow graph.
try:
_, _, _op = _op_def_lib._apply_op_helper(
"AdjustHsvInYiq", images=images, delta_h=delta_h, scale_s=scale_s,
scale_v=scale_v, name=name)
except (TypeError, ValueError):
result = _dispatch.dispatch(
adjust_hsv_in_yiq, images=images, delta_h=delta_h, scale_s=scale_s,
scale_v=scale_v, name=name)
if result is not _dispatch.OpDispatcher.NOT_SUPPORTED:
return result
raise
_result = _op.outputs[:]
_inputs_flat = _op.inputs
_attrs = ("T", _op.get_attr("T"))
_execute.record_gradient(
"AdjustHsvInYiq", _inputs_flat, _attrs, _result, name)
_result, = _result
return _result
def lookup(self, name, visibility=None, scope_limit=None):
'''Look up a symbol in the symbol table. The lookup can be limited
by visibility (e.g. just show public methods) or by scope_limit (e.g.
just show symbols up to a certain scope).
:param str name: name of the symbol.
:param visibilty: the visibility or list of visibilities that the \
symbol must have.
:type visibility: [list of] :py:class:`psyclone.symbols.Visibility`
:param scope_limit: optional Node which limits the symbol \
search space to the symbol tables of the nodes within the \
given scope. If it is None (the default), the whole \
scope (all symbol tables in ancestor nodes) is searched \
otherwise ancestors of the scope_limit node are not \
searched.
:type scope_limit: :py:class:`psyclone.psyir.nodes.Node` or \
`NoneType`
:returns: the symbol with the given name and, if specified, visibility.
:rtype: :py:class:`psyclone.psyir.symbols.Symbol`
:raises TypeError: if the name argument is not a string.
:raises SymbolError: if the name exists in the Symbol Table but does \
not have the specified visibility.
:raises TypeError: if the visibility argument has the wrong type.
:raises KeyError: if the given name is not in the Symbol Table.
'''
if not isinstance(name, six.string_types):
raise TypeError(
"Expected the name argument to the lookup() method to be "
"a str but found '{0}'."
"".format(type(name).__name__))
try:
symbol = self.get_symbols(scope_limit)[self._normalize(name)]
if visibility:
if not isinstance(visibility, list):
vis_list = [visibility]
else:
vis_list = visibility
if symbol.visibility not in vis_list:
vis_names = []
# Take care here in case the 'visibility' argument
# is of the wrong type
for vis in vis_list:
if not isinstance(vis, Symbol.Visibility):
raise TypeError(
"the 'visibility' argument to lookup() must be"
" an instance (or list of instances) of "
"Symbol.Visibility but got '{0}' when "
"searching for symbol '{1}'".format(
type(vis).__name__, name))
vis_names.append(vis.name)
raise SymbolError(
"Symbol '{0}' exists in the Symbol Table but has "
"visibility '{1}' which does not match with the "
"requested visibility: {2}".format(
name, symbol.visibility.name, vis_names))
return symbol
except KeyError as err:
six.raise_from(
KeyError("Could not find '{0}' in the Symbol Table."
"".format(name)), err)
def stochastic_hard_routing_function(input_data,
tree_parameters,
tree_biases,
tree_depth,
random_seed,
name=None):
r""" Samples a path for each instance in `input_data` and returns the
probability of the path and the path taken.
tree_depth: The depth of the decision tree.
random_seed: The base random seed.
input_data: The training batch's features as a 2-d tensor; `input_data[i][j]`
gives the j-th feature of the i-th input.
tree_parameters: `tree_parameters[i]` gives the weight of
the logistic regression model that translates from node features to
probabilities.
tree_biases: `tree_biases[i]` gives the bias of the logistic
regression model that translates from node features to
probabilities.
path_probility: `path_probability[i]` gives the probability of reaching each
node in `path[i]`.
path: `path[i][j]` gives the jth node in the path taken by the ith data
instance.
Args:
input_data: A `Tensor` of type `float32`.
tree_parameters: A `Tensor` of type `float32`.
tree_biases: A `Tensor` of type `float32`.
tree_depth: An `int`.
random_seed: An `int`.
name: A name for the operation (optional).
Returns:
A tuple of `Tensor` objects (path_probability, path).
path_probability: A `Tensor` of type `float32`.
path: A `Tensor` of type `int32`.
"""
_ctx = _context._context
if _ctx is None or not _ctx._eager_context.is_eager:
tree_depth = _execute.make_int(tree_depth, "tree_depth")
random_seed = _execute.make_int(random_seed, "random_seed")
_, _, _op = _op_def_lib._apply_op_helper(
"StochasticHardRoutingFunction",
input_data=input_data,
tree_parameters=tree_parameters,
tree_biases=tree_biases,
tree_depth=tree_depth,
random_seed=random_seed,
name=name)
_result = _op.outputs[:]
_inputs_flat = _op.inputs
_attrs = ("tree_depth", _op.get_attr("tree_depth"), "random_seed",
_op.get_attr("random_seed"))
_execute.record_gradient("StochasticHardRoutingFunction", _inputs_flat,
_attrs, _result, name)
_result = _StochasticHardRoutingFunctionOutput._make(_result)
return _result
else:
try:
_result = _pywrap_tensorflow.TFE_Py_FastPathExecute(
_ctx._context_handle, _ctx._eager_context.device_name,
"StochasticHardRoutingFunction", name,
_ctx._post_execution_callbacks, input_data, tree_parameters,
tree_biases, "tree_depth", tree_depth, "random_seed",
random_seed)
_result = _StochasticHardRoutingFunctionOutput._make(_result)
return _result
except _core._FallbackException:
return stochastic_hard_routing_function_eager_fallback(
input_data,
tree_parameters,
tree_biases,
tree_depth=tree_depth,
random_seed=random_seed,
name=name,
ctx=_ctx)
except _core._NotOkStatusException as e:
if name is not None:
message = e.message + " name: " + name
else:
message = e.message
_six.raise_from(_core._status_to_exception(e.code, message), None)
def resampler_grad(data, warp, grad_output, name=None):
r"""Resampler Grad op.
Args:
data: A `Tensor`. Must be one of the following types: `half`, `bfloat16`, `float32`, `float64`.
warp: A `Tensor`. Must have the same type as `data`.
grad_output: A `Tensor`. Must have the same type as `data`.
name: A name for the operation (optional).
Returns:
A tuple of `Tensor` objects (grad_data, grad_warp).
grad_data: A `Tensor`. Has the same type as `data`.
grad_warp: A `Tensor`. Has the same type as `data`.
"""
_ctx = _context._context or _context.context()
if _ctx is not None and _ctx._thread_local_data.is_eager:
try:
_result = _pywrap_tensorflow.TFE_Py_FastPathExecute(
_ctx._context_handle, _ctx._thread_local_data.device_name,
"ResamplerGrad", name, _ctx._post_execution_callbacks, data, warp,
grad_output)
_result = _ResamplerGradOutput._make(_result)
return _result
except _core._FallbackException:
try:
return resampler_grad_eager_fallback(
data, warp, grad_output, name=name, ctx=_ctx)
except _core._SymbolicException:
pass # Add nodes to the TensorFlow graph.
except (TypeError, ValueError):
result = _dispatch.dispatch(
resampler_grad, data=data, warp=warp, grad_output=grad_output,
name=name)
if result is not _dispatch.OpDispatcher.NOT_SUPPORTED:
return result
raise
except _core._NotOkStatusException as e:
if name is not None:
message = e.message + " name: " + name
else:
message = e.message
_six.raise_from(_core._status_to_exception(e.code, message), None)
# Add nodes to the TensorFlow graph.
try:
_, _, _op = _op_def_lib._apply_op_helper(
"ResamplerGrad", data=data, warp=warp, grad_output=grad_output,
name=name)
except (TypeError, ValueError):
result = _dispatch.dispatch(
resampler_grad, data=data, warp=warp, grad_output=grad_output,
name=name)
if result is not _dispatch.OpDispatcher.NOT_SUPPORTED:
return result
raise
_result = _op.outputs[:]
_inputs_flat = _op.inputs
_attrs = ("T", _op.get_attr("T"))
_execute.record_gradient(
"ResamplerGrad", _inputs_flat, _attrs, _result, name)
_result = _ResamplerGradOutput._make(_result)
return _result
def parse(self):
"""
Parse the POST data and break it into a FILES MultiValueDict and a POST
MultiValueDict.
Return a tuple containing the POST and FILES dictionary, respectively.
"""
from request_parser.http.request import QueryDict
encoding = self._encoding
handlers = self._upload_handlers
# HTTP spec says that Content-Length >= 0 is valid
# handling content-length == 0 before continuing
if self._content_length == 0:
return QueryDict(self.settings,
encoding=self._encoding), MultiValueDict()
# See if any of the handlers take care of the parsing.
# This allows overriding everything if need be.
for handler in handlers:
result = handler.handle_raw_input(
self._input_data,
self._meta,
self._content_length,
self._boundary,
self.settings,
encoding,
)
# Check to see if it was handled
if result is not None:
#If we're returning as soon as the result is not None, then
#it means there can be only one handler that is allowed to parse
#body and files - technically there can be more than one but the
#first one is picked
return result[0], result[1]
# Create the data structures to be used later.
self._post = QueryDict(self.settings, mutable=True)
self._files = MultiValueDict()
# Instantiate the stream:
stream = LazyStream(ChunkIter(self._input_data, self._chunk_size))
# Whether or not to signal a file-completion at the beginning of the loop.
old_field_name = None
counters = [0] * len(handlers)
# Number of bytes that have been read.
num_bytes_read = 0
# To count the number of keys in the request.
num_post_keys = 0
# To limit the amount of data read from the request.
read_size = None
try:
for item_type, meta_data, field_stream in Parser(
stream, self._boundary):
if old_field_name:
# We run this at the beginning of the next loop
# since we cannot be sure a file is complete until
# we hit the next boundary/part of the multipart content.
self.handle_file_complete(old_field_name, counters)
old_field_name = None
try:
disposition = meta_data['content-disposition'][1]
field_name = disposition['name'].strip()
except (KeyError, IndexError, AttributeError):
continue
transfer_encoding = meta_data.get('content-transfer-encoding')
if transfer_encoding is not None:
transfer_encoding = transfer_encoding[0].strip()
transfer_encoding = force_text(transfer_encoding,
encoding,
errors='replace')
field_name = force_text(field_name, encoding, errors='replace')
if item_type == FIELD:
# Avoid storing more than DATA_UPLOAD_MAX_NUMBER_FIELDS.
num_post_keys += 1
if (self.settings.DATA_UPLOAD_MAX_NUMBER_FIELDS is not None
and self.settings.DATA_UPLOAD_MAX_NUMBER_FIELDS <
num_post_keys):
raise TooManyFieldsSent(
'The number of GET/POST parameters exceeded '
'settings.DATA_UPLOAD_MAX_NUMBER_FIELDS.')
# Avoid reading more than DATA_UPLOAD_MAX_MEMORY_SIZE.
if self.settings.DATA_UPLOAD_MAX_MEMORY_SIZE is not None:
read_size = self.settings.DATA_UPLOAD_MAX_MEMORY_SIZE - num_bytes_read
# This is a post field, we can just set it in the post
if transfer_encoding == 'base64':
#read only for the remaining size
raw_data = field_stream.read(size=read_size)
num_bytes_read += len(raw_data)
try:
#decode the data read
data = base64.b64decode(raw_data)
except binascii.Error:
data = raw_data
else:
data = field_stream.read(size=read_size)
num_bytes_read += len(data)
# Add two here to make the check consistent with the
# x-www-form-urlencoded check that includes '&='.
#QUESTION: We can't implement buffered reading if we
#have only read part of a stream. Can we?
num_bytes_read += len(field_name) + 2
if (self.settings.DATA_UPLOAD_MAX_MEMORY_SIZE is not None
and num_bytes_read >
self.settings.DATA_UPLOAD_MAX_MEMORY_SIZE):
raise RequestDataTooBig(
'Request body exceeded settings.DATA_UPLOAD_MAX_MEMORY_SIZE.'
)
#force_text(data, encoding, errors='replace')
if transfer_encoding is None:
transfer_encoding = ''
content_type, content_type_extra = meta_data.get(
'content-type', ('', {}))
content_type = content_type.strip()
#print "data: "+data+"\r\ntype: "+content_type+"\r\nt.encoding: "+transfer_encoding
#self._post.appendlist(field_name, force_text(data, encoding, errors='replace'))
self._post.appendlist(
field_name, {
'data':
force_text(data, encoding, errors='replace'),
'content-type':
force_text(
content_type, encoding, errors='replace'),
'transfer-encoding':
transfer_encoding,
'content-type-extra':
content_type_extra
})
elif item_type == FILE:
# This is a file, use the handler...
file_name = disposition.get('filename')
if file_name:
file_name = force_text(file_name,
encoding,
errors='replace')
file_name = self.IE_sanitize(
unescape_entities(file_name))
if not file_name:
continue
content_type, content_type_extra = meta_data.get(
'content-type', ('', {}))
content_type = content_type.strip()
charset = content_type_extra.get('charset')
try:
content_length = int(
meta_data.get('content-length')[0])
except (IndexError, TypeError, ValueError):
content_length = None
counters = [0] * len(handlers)
try:
for handler in handlers:
try:
handler.new_file(field_name, file_name,
content_type, content_length,
charset, content_type_extra,
transfer_encoding)
#if a handler is handling a new file, it raises StopFutureHandlers
#to prevent others from handling it
except StopFutureHandlers:
break
for chunk in field_stream:
if transfer_encoding == 'base64':
# We only special-case base64 transfer encoding
# We should always decode base64 chunks by multiple of 4,
# ignoring whitespace.
stripped_chunk = b"".join(chunk.split())
remaining = len(stripped_chunk) % 4
while remaining != 0:
over_chunk = field_stream.read(4 -
remaining)
stripped_chunk += b"".join(
over_chunk.split())
remaining = len(stripped_chunk) % 4
try:
chunk = base64.b64decode(stripped_chunk)
except Exception as exc:
# Since this is only a chunk, any error is an unfixable error.
raise_from(
MultiPartParserError(
"Could not decode base64 data."),
exc)
for i, handler in enumerate(handlers):
chunk_length = len(chunk)
#stream data into the temp file
chunk = handler.receive_data_chunk(
chunk, counters[i])
counters[i] += chunk_length
#None means no errors, which means we break
if chunk is None:
# Don't continue if the chunk received by
# the handler is None.
break
#QUESTION: When does this occur?
except SkipFile:
self._close_files()
# Just use up the rest of this file...
exhaust(field_stream)
else:
# Handle file upload completions on next iteration.
old_field_name = field_name
else:
# If this is neither a FIELD or a FILE, just exhaust the stream.
exhaust(stream)
#QUESTION: When does this occur?
#ANSWER: This is used when one of the file handler on line 257 signals to
#stop any more further file handline. This means any further file upload
#request needs to be abruptly stopped.
#See django's tests/file_uploads/uploadhandler.py for more details.
#TODO: Repurpose this so that we can stop file upload parsing whenver needed.
except StopUpload as e:
self._close_files()
if not e.connection_reset:
exhaust(self._input_data)
else:
# Make sure that the request data is all fed
exhaust(self._input_data)
# Signal that the upload has completed.
# any() shortcircuits if a handler's upload_complete() returns a value.
#Perform any clean up after file upload is complete
any(handler.upload_complete() for handler in handlers)
self._post._mutable = False
return self._post, self._files
