def get_gtdbtk_params(input_params: Dict[str, object]) -> GTDBTKParams:
'''
Process input parameters supplied to the GTDB-tk run method and parse them into
a named tuple. The expected fields are:
input_object_ref: a workspace reference to the input object that will be processed in the
x/y/z form.
workspace_id: the integer ID of the workspace where the results will be saved.
min_perc_aa: the minimum sequence alignment as a percent.
:param input_params: the input parameters passed to the method.
:returns: the parsed parameters.
:raises ValueError: if any of the parameters are invalid.
'''
ref = input_params.get('input_object_ref')
if not ref:
# for backwards compatibility
ref = input_params.get('inputObjectRef')
if type(ref) != str:
raise ValueError('input_object_ref is required and must be a string')
# could check ref format, but the ws will do that for us. YAGNI.
min_perc_aa = input_params.get('min_perc_aa', 10)
if type(min_perc_aa) != float and type(min_perc_aa) != int:
raise ValueError('min_perc_aa must be a float')
# TODO check 0 <= min_perc_aa <= 1
wsid = input_params.get('workspace_id')
if type(wsid) != int or _cast(int, wsid) < 1:
raise ValueError('workspace_id is required and must be an integer > 0')
return GTDBTKParams(_cast(str, ref), _cast(int, wsid),
_cast(float, min_perc_aa) * 1.0)
def get_static_key_metadata_params(
params: Dict[str, Any]) -> Tuple[List[str], Optional[bool]]:
'''
Given a dict, extract the parameters to interrogate metadata key static metadata.
:param params: The parameters.
:returns: A tuple consisting of, firstly, the list of keys to interrogate, and secondly,
a trinary value where False indicates that standard keys should be interrogated, None
indicates that prefix keys should be interrogated but only exact matches should be
considered, and True indicates that prefix keys should be interrogated but prefix matches
should be included in the results.
:raises IllegalParameterError: if any of the arguments are illegal.
'''
_check_params(params)
keys = params.get('keys')
if type(keys) != list:
raise _IllegalParameterError('keys must be a list')
for i, k in enumerate(_cast(List[Any], keys)):
if type(k) != str:
raise _IllegalParameterError(f'index {i} of keys is not a string')
prefix = params.get('prefix')
pre: Optional[bool]
if not prefix:
pre = False
elif prefix == 1:
pre = None
elif prefix == 2:
pre = True
else:
raise _IllegalParameterError(f'Unexpected value for prefix: {prefix}')
return _cast(List[str], keys), pre
def create_data_link_params(
params: Dict[str, Any]) -> Tuple[DataUnitID, SampleNodeAddress, bool]:
'''
Given a dict, extract the parameters to create parameters for creating a data link.
Expected keys:
id - sample id
version - sample version
node - sample node
upa - workspace object UPA
dataid - ID of the data within the workspace object
update - whether the link should be updated
:param params: the parameters.
:returns: a tuple consisting of:
1) The data unit ID that is the target of the link,
2) The sample node that is the target of the link,
3) A boolean that indicates whether the link should be updated if it already exists.
:raises MissingParameterError: if any of the required arguments are missing.
:raises IllegalParameterError: if any of the arguments are illegal.
'''
_check_params(params)
sna = SampleNodeAddress(
_SampleAddress(
_cast(UUID, get_id_from_object(params, ID, required=True)),
_cast(int, get_version_from_object(params, required=True))),
_cast(str, _check_string_int(params, 'node', True)))
duid = get_data_unit_id_from_object(params)
return (duid, sna, bool(params.get('update')))
def _check_metadata_value(key: str, value: Dict[str, PrimitiveType],
name: str) -> Dict[str, PrimitiveType]:
if not value:
raise IllegalParameterError(
f'{name} metadata value associated with metadata key {key} is null or empty'
)
for vk in value:
cc = _control_char_first_pos(vk)
if cc >= 0:
raise IllegalParameterError(
f"{name} metadata value key {vk} associated with metadata key {key} has a "
+ f'character at index {cc} that is a control character.')
if len(vk) > _META_MAX_KEY_SIZE:
raise IllegalParameterError(
f'{name} metadata has a value key associated with metadata key {key} starting '
+
f'with {vk[:_META_MAX_KEY_SIZE]} that exceeds maximum length of '
+ f'{_META_MAX_KEY_SIZE}')
val = value[vk]
if type(val) == str:
cc = _control_char_first_pos(_cast(str, val),
allow_tabs_and_lf=True)
if cc >= 0:
raise IllegalParameterError(
f"{name} metadata value associated with metadata key {key} and "
+
f'value key {vk} has a character at index {cc} that is a control character.'
)
if len(_cast(str, val)) > _META_MAX_VALUE_SIZE:
raise IllegalParameterError(
f'{name} metadata has a value associated with metadata key {key} '
+
f'and value key {vk} starting with {_cast(str, val)[:_META_MAX_KEY_SIZE]} '
+ f'that exceeds maximum length of {_META_MAX_VALUE_SIZE}')
return value
def strlen(key: str, d1: Dict[str, PrimitiveType]) -> Optional[str]:
for k, v in d1.items():
if len(k) > _cast(int, maxlen):
return f'Metadata contains key longer than max length of {maxlen}'
if type(v) == str:
if len(_cast(str, v)) > _cast(int, maxlen):
return f'Metadata value at key {k} is longer than max length of {maxlen}'
return None
def deserialize_world(serialized: "SerializedWorldType",
name: "Optional[str]" = None) -> "_World":
"""
Deserialize the output of `serialize_world` into a new `World`.
If any of the component types that were registered in the serialized World
are not registered when you call `deserialize_world`, or any of them
have been renamed, this will fail and raise a ValueError.
If you want to deserialize into the default World, you'll want to call
`move_world` afterwards::
move_world(deserialize_world(serialized_data))
:param serialized: A serialized world, as output by `serialize_world`.
:type serialized: `SerializedWorldType`
:param name: An optional name to use for the new World. See `World`.
:type name: Optional[str]
:return: A new World with the data from the serialized one.
:rtype: `World`
"""
world = _World(name=name)
serialized_names = _cast("List[str]",
serialized[SERIALIZED_COMPONENTS_KEY])
component_types: "List[Type[Component]]" = [
_component_names[name] for name in serialized_names
]
serialized_entities = _cast("Dict[_EntityID, Dict[int, str]]",
serialized[SERIALIZED_ENTITIES_KEY])
for ent_id, components in serialized_entities.items():
ent_id = _EntityID(ent_id)
cmp_instances = [
component_types[int(i)].deserialize(serialized)
for i, serialized in components.items()
]
if ent_id > world._entity_counter:
world._entity_counter = ent_id
# This is pretty directly copied over from `add_component`
bitmask = _ZERO
entdict = {}
entcache = world._entity_cache
for component in cmp_instances:
cc = component.__class__
bitmask |= component._bitmask
entdict[cc] = component
entcache.setdefault(cc, set()).add(ent_id)
world._entities[ent_id] = entdict
world._entity_bitmasks[ent_id] = bitmask
return world
def _to_tuple(self, seq, name):
# dict.fromkeys removes dupes
return tuple(
dict.fromkeys(
_cast(Sequence[UserID],
_not_falsy_in_iterable([] if seq is None else seq,
name))))
def NamedTemporaryFile(mode: str = 'w+b', buffering: int = -1,
encoding: str = None, newline: str = None,
suffix: str = "", prefix: str = template,
dir: str = None, delete: bool = True) -> _IO[_Any]:
"""Create and return a temporary file.
Arguments:
'prefix', 'suffix', 'dir' -- as for mkstemp.
'mode' -- the mode argument to io.open (default "w+b").
'buffering' -- the buffer size argument to io.open (default -1).
'encoding' -- the encoding argument to io.open (default None)
'newline' -- the newline argument to io.open (default None)
'delete' -- whether the file is deleted on close (default True).
The file is created as mkstemp() would do it.
Returns an object with a file-like interface; the name of the file
is accessible as file.name. The file will be automatically deleted
when it is closed unless the 'delete' argument is set to False.
"""
if dir is None:
dir = gettempdir()
flags = _bin_openflags
# Setting O_TEMPORARY in the flags causes the OS to delete
# the file when it is closed. This is only supported by Windows.
if _os.name == 'nt' and delete:
flags |= _os.O_TEMPORARY
(fd, name) = _mkstemp_inner(dir, prefix, suffix, flags)
file = _io.open(fd, mode, buffering=buffering,
newline=newline, encoding=encoding)
return _cast(_IO[_Any], _TemporaryFileWrapper(file, name, delete))
def _check_nodes(s):
nodes = []
for i, n in enumerate(s['node_tree']):
if type(n) != dict:
raise _IllegalParameterError(
f'Node at index {i} is not a structure')
if type(n.get('id')) != str:
raise _IllegalParameterError(
f'Node at index {i} must have an id key that maps to a string')
try:
type_ = _SubSampleType(n.get('type'))
except ValueError:
raise _IllegalParameterError(
f'Node at index {i} has an invalid sample type: {n.get("type")}'
)
if n.get('parent') and type(n.get('parent')) != str:
raise _IllegalParameterError(
f'Node at index {i} has a parent entry that is not a string')
mc = _check_meta(n.get('meta_controlled'), i, 'controlled metadata')
mu = _check_meta(n.get('meta_user'), i, 'user metadata')
sm = _check_source_meta(n.get('source_meta'), i)
try:
nodes.append(
_SampleNode(n.get('id'), type_, n.get('parent'), mc, mu, sm))
# already checked for the missing param error above, for id
except _IllegalParameterError as e:
raise _IllegalParameterError(f'Error for node at index {i}: ' +
_cast(str, e.message)) from e
return nodes
def update_sample_acls(
self,
id_: UUID,
user: UserID,
update: SampleACLDelta,
as_admin: bool = False) -> None:
'''
Completely replace a sample's ACLs.
:param id_: the sample's ID.
:param user: the user changing the ACLs.
:param update: the ACL update. Note the update time is ignored. If the sample owner is
in any of the lists in the update, the update will fail.
:param as_admin: Skip ACL checks.
:raises NoSuchUserError: if any of the users in the ACLs do not exist.
:raises NoSuchSampleError: if the sample does not exist.
:raises UnauthorizedError: if the user does not have admin permission for the sample or
the request attempts to alter the owner.
:raises SampleStorageError: if the sample could not be retrieved.
'''
# could make yet another ACL class that's a delta w/o an update time - probably not
# worth it. If people get confused do it.
_not_falsy(id_, 'id_')
_not_falsy(user, 'user')
_not_falsy(update, 'update')
self._check_for_bad_users(_cast(List[UserID], []) + list(update.admin) +
list(update.write) + list(update.read) + list(update.remove))
self._check_perms(id_, user, _SampleAccessType.ADMIN, as_admin=as_admin)
self._storage.update_sample_acls(id_, update, self._now())
if self._kafka:
self._kafka.notify_sample_acl_change(id_)
def create_sample_params(
params: Dict[str,
Any]) -> Tuple[Sample, Optional[UUID], Optional[int]]:
'''
Process the input from the create_sample API call and translate it into standard types.
:param params: The unmarshalled JSON recieved from the API as part of the create_sample
call.
:returns: A tuple of the sample to save, the UUID of the sample for which a new version should
be created or None if an entirely new sample should be created, and the previous version
of the sample expected when saving a new version.
:raises IllegalParameterError: if any of the arguments are illegal.
'''
_check_params(params)
if type(params.get('sample')) != dict:
raise _IllegalParameterError(
'params must contain sample key that maps to a structure')
s = params['sample']
if type(s.get('node_tree')) != list:
raise _IllegalParameterError(
'sample node tree must be present and a list')
if s.get('name') is not None and type(s.get('name')) != str:
raise _IllegalParameterError('sample name must be omitted or a string')
nodes = []
for i, n in enumerate(s['node_tree']):
if type(n) != dict:
raise _IllegalParameterError(
f'Node at index {i} is not a structure')
if type(n.get('id')) != str:
raise _IllegalParameterError(
f'Node at index {i} must have an id key that maps to a string')
try:
type_ = _SubSampleType(n.get('type'))
except ValueError:
raise _IllegalParameterError(
f'Node at index {i} has an invalid sample type: {n.get("type")}'
)
if n.get('parent') and type(n.get('parent')) != str:
raise _IllegalParameterError(
f'Node at index {i} has a parent entry that is not a string')
mc = _check_meta(n.get('meta_controlled'), i, 'controlled metadata')
mu = _check_meta(n.get('meta_user'), i, 'user metadata')
try:
nodes.append(
_SampleNode(n.get('id'), type_, n.get('parent'), mc, mu))
# already checked for the missing param error above, for id
except _IllegalParameterError as e:
raise _IllegalParameterError(f'Error for node at index {i}: ' +
_cast(str, e.message)) from e
id_ = get_id_from_object(s)
pv = params.get('prior_version')
if pv is not None and type(pv) != int:
raise _IllegalParameterError(
'prior_version must be an integer if supplied')
s = Sample(nodes, s.get('name'))
return (s, id_, pv)
def __getattr__(self, name: str) -> _Any:
# Attribute lookups are delegated to the underlying file
# and cached for non-numeric results
# (i.e. methods are cached, closed and friends are not)
file = _cast(_Any, self).__dict__['file'] # type: _IO[_Any]
a = getattr(file, name)
if not isinstance(a, int):
setattr(self, name, a)
return a
def __getattr__(self, name: str) -> _Any:
# Attribute lookups are delegated to the underlying file
# and cached for non-numeric results
# (i.e. methods are cached, closed and friends are not)
file = _cast(_Any, self).__dict__['file'] # type: _IO[_Any]
a = getattr(file, name)
if not isinstance(a, int):
setattr(self, name, a)
return a
def _to_tuple(seq, name) -> _Tuple[UserID, ...]:
# dict.fromkeys removes dupes
return tuple(
dict.fromkeys(
sorted( # sort to make equals and hash consistent
_cast(Sequence[UserID],
_not_falsy_in_iterable([] if seq is None else seq,
name)),
key=lambda u: u.id))) # add comparison methods to user?
def strlen(key: str, d1: Dict[str, PrimitiveType]) -> Optional[str]:
for k in keys:
if required and k not in d1:
return f'Required key {k} is missing'
v = d1.get(k)
if v is not None and type(v) != str:
return f'Metadata value at key {k} is not a string'
if v and maxlen and len(_cast(str, v)) > maxlen:
return f'Metadata value at key {k} is longer than max length of {maxlen}'
return None
def strlen(key: str,
d1: Dict[str, PrimitiveType]) -> Optional[ValidatorMessage]:
for k, v in d1.items():
if len(k) > _cast(int, maxlen):
return {
'subkey':
str(k),
'message':
f'Metadata contains key longer than max length of {maxlen}'
}
if type(v) == str:
if len(_cast(str, v)) > _cast(int, maxlen):
return {
'subkey':
str(k),
'message':
f'Metadata value at key {k} is longer than max length of {maxlen}'
}
return None
def _check_string_int(params: Dict[str, Any], key: str, required=False) -> Optional[str]:
v = params.get(key)
if v is None:
if required:
raise _MissingParameterError(key)
else:
return None
if type(v) != str:
raise _IllegalParameterError(f'{key} key is not a string as required')
return _cast(str, v)
def _check_meta(m: Dict[str, Dict[str, PrimitiveType]], controlled: bool):
c = 'Controlled' if controlled else 'User'
for k in m:
if len(k) > _META_MAX_KEY_SIZE:
raise IllegalParameterError(
f'{c} metadata has key starting with {k[:_META_MAX_KEY_SIZE]} that '
+ f'exceeds maximum length of {_META_MAX_KEY_SIZE}')
cc = _control_char_first_pos(k)
if cc:
raise IllegalParameterError(
f"{c} metadata key {k}'s character at index {cc} is a control character."
)
for vk in m[k]:
if len(vk) > _META_MAX_KEY_SIZE:
raise IllegalParameterError(
f'{c} metadata has value key under root key {k} starting with '
+
f'{vk[:_META_MAX_KEY_SIZE]} that exceeds maximum length of '
+ f'{_META_MAX_KEY_SIZE}')
cc = _control_char_first_pos(vk)
if cc:
raise IllegalParameterError(
f"{c} metadata value key {vk} under key {k}'s character at index {cc} "
+ 'is a control character.')
val = m[k][vk]
if type(val) == str:
if len(_cast(str, val)) > _META_MAX_VALUE_SIZE:
raise IllegalParameterError(
f'{c} metadata has value under root key {k} and value key {vk} '
+
f'starting with {_cast(str, val)[:_META_MAX_KEY_SIZE]} that '
+ f'exceeds maximum length of {_META_MAX_VALUE_SIZE}')
cc = _control_char_first_pos(_cast(str, val),
allow_tabs_and_lf=True)
if cc:
raise IllegalParameterError(
f"{c} metadata value under root key {k} and value key {vk}'s "
+ f'character at index {cc} is a control character.')
if len(_json.dumps(m,
ensure_ascii=False).encode('utf-8')) > _META_MAX_SIZE_B:
# would be nice if that could be streamed so we don't make a new byte array
raise IllegalParameterError(
f'{c} metadata is larger than maximum of {_META_MAX_SIZE_B}B')
def get_upa_from_object(params: Dict[str, Any]) -> UPA:
'''
Get an UPA from a parameter object. Expects the UPA in the key 'upa'.
:param params: the parameters.
:returns: the UPA.
:raises MissingParameterError: if the UPA is missing.
:raises IllegalParameterError: if the UPA is illegal.
'''
_check_params(params)
return UPA(_cast(str, _check_string_int(params, 'upa', True)))
def replace_sample_acls(self,
id_: UUID,
user: UserID,
new_acls: SampleACLOwnerless,
as_admin: bool = False) -> None:
'''
Completely replace a sample's ACLs. The owner cannot be changed.
:param id_: the sample's ID.
:param user: the user changing the ACLs.
:param new_acls: the new ACLs.
:param as_admin: Skip ACL checks.
:raises NoSuchUserError: if any of the users in the ACLs do not exist.
:raises NoSuchSampleError: if the sample does not exist.
:raises UnauthorizedError: if the user does not have admin permission for the sample or
the request attempts to change the owner.
:raises SampleStorageError: if the sample could not be retrieved.
'''
_not_falsy(id_, 'id_')
_not_falsy(user, 'user')
_not_falsy(new_acls, 'new_acls')
try:
bad_users = self._user_lookup.invalid_users(
_cast(List[UserID], []) + list(new_acls.admin) +
list(new_acls.write) + list(new_acls.read))
# let authentication errors propagate, not much to do
# could add retries to the client
except _user_lookup_mod.InvalidUserError as e:
raise _NoSuchUserError(e.args[0]) from e
except _user_lookup_mod.InvalidTokenError:
raise ValueError(
'user lookup token for KBase auth server is invalid, cannot continue'
)
if bad_users:
raise _NoSuchUserError(', '.join([u.id for u in bad_users[:5]]))
count = 0
while count >= 0:
if count >= 5:
raise ValueError(
f'Failed setting ACLs after 5 attempts for sample {id_}')
acls = self._storage.get_sample_acls(id_)
self._check_perms(id_,
user,
_SampleAccessType.ADMIN,
acls,
as_admin=as_admin)
new_acls = SampleACL(acls.owner, new_acls.admin, new_acls.write,
new_acls.read)
try:
self._storage.replace_sample_acls(id_, new_acls)
count = -1
except _OwnerChangedError:
count += 1
def __init__(self, sample: SampleAddress, node: str):
'''
Create the address.
:param sample: The sample address.
:param node: The ID of the sample node.
'''
self.sampleid = _not_falsy(sample, 'sample').sampleid
self.version = sample.version
self.node = _cast(
str, _check_string(node, 'node', max_len=_MAX_SAMPLE_NAME_LEN))
def resolve_string_type_hint(tpe: str) -> Optional[Type]:
import pyspark.pandas as ps
from pyspark.pandas import DataFrame, Series
locs = {
"ps": ps,
"pyspark.pandas": ps,
"DataFrame": DataFrame,
"Series": Series,
}
# This is a hack to resolve the forward reference string.
exec("def func() -> %s: pass\narg_spec = getfullargspec(func)" % tpe, globals(), locs)
return _cast(FullArgSpec, locs["arg_spec"]).annotations.get("return", None)
def __init__(
self,
name: str,
type_: SubSampleType = SubSampleType.BIOLOGICAL_REPLICATE,
parent: Optional[str] = None,
controlled_metadata: Optional[Dict[str,
Dict[str,
PrimitiveType]]] = None,
user_metadata: Optional[Dict[str, Dict[str,
PrimitiveType]]] = None,
source_metadata: Optional[List[SourceMetadata]] = None):
'''
Create a sample node.
:param name: The name of the sample node.
:param type_: The type of this sample nde.
:param parent: The parent SampleNode of this node. BIOLOGICAL_REPLICATEs, and only
BIOLOGICAL_REPLICATEs, cannot have parents.
:param controlled_metadata: Sample metadata that has been checked against a controlled
vocabulary.
:param user_metadata: Unrestricted sample metadata.
:param source_metadata: Information about the controlled metadata as it existed at the data
source, prior to any possible transformations for ingest.
:raises MissingParameterError: if the name is None or whitespace only.
:raises IllegalParameterError: if the name or parent is too long or contains illegal
characters, the parent is missing and the node type is not BIOLOGICAL_REPLICATE,
or basic metadata constraints are violated.
'''
# could make a bioreplicate class... meh for now
self.name = _cast(
str,
_check_string(name, 'subsample name',
max_len=_MAX_SAMPLE_NAME_LEN))
self.type = _not_falsy(type_, 'type')
self.parent = _check_string(parent,
'parent',
max_len=_MAX_SAMPLE_NAME_LEN,
optional=True)
cm = controlled_metadata if controlled_metadata else {}
_check_meta(cm, True)
self.controlled_metadata = _fz(cm)
um = user_metadata if user_metadata else {}
_check_meta(um, False)
self.user_metadata = _fz(um)
sm = source_metadata if source_metadata else []
_check_source_meta(sm, self.controlled_metadata)
self.source_metadata = tuple(sm)
isbiorep = type_ == SubSampleType.BIOLOGICAL_REPLICATE
if not _xor(bool(parent), isbiorep):
raise IllegalParameterError(
f'Node {self.name} is of type {type_.value} and therefore ' +
f'{"cannot" if isbiorep else "must"} have a parent')
def get_sample_address_from_object(
params: Dict[str, Any], version_required: bool = False) -> Tuple[UUID, Optional[int]]:
'''
Given a dict, get a sample ID and version from the dict. The sample ID is required but
the version is not. The keys 'id' and 'version' are used.
:param params: the unmarshalled JSON recieved from the API as part of the API call.
:param version_required: require the version as well as the ID.
:returns: a tuple containing the ID and the version or None if no version was provided.
:raises MissingParameterError: if the ID is missing or the version is required and not present.
:raises IllegalParameterError: if the ID is malformed or if the version is not an
integer or < 1.
'''
return (_cast(UUID, get_id_from_object(params, ID, required=True)),
get_version_from_object(params, version_required))
def get_user_from_object(params: Dict[str, Any], key: str) -> Optional[UserID]:
'''
Get a user ID from a key in an object.
:param params: the dict containing the user.
:param key: the key in the dict where the value is the username.
:returns: the user ID or None if the user is not present.
:raises IllegalParameterError: if the user if invalid.
'''
_check_params(params)
u = params.get(_cast(str, _check_string(key, 'key')))
if u is None:
return None
if type(u) is not str:
raise _IllegalParameterError(f'{key} must be a string if present')
else:
return UserID(u)
def replace_sample_acls(self,
id_: UUID,
user: UserID,
new_acls: SampleACLOwnerless,
as_admin: bool = False) -> None:
'''
Completely replace a sample's ACLs. The owner cannot be changed.
:param id_: the sample's ID.
:param user: the user changing the ACLs.
:param new_acls: the new ACLs.
:param as_admin: Skip ACL checks.
:raises NoSuchUserError: if any of the users in the ACLs do not exist.
:raises NoSuchSampleError: if the sample does not exist.
:raises UnauthorizedError: if the user does not have admin permission for the sample or
the request attempts to change the owner.
:raises SampleStorageError: if the sample could not be retrieved.
'''
_not_falsy(id_, 'id_')
_not_falsy(user, 'user')
_not_falsy(new_acls, 'new_acls')
self._check_for_bad_users(
_cast(List[UserID], []) + list(new_acls.admin) +
list(new_acls.write) + list(new_acls.read))
count = 0
while count >= 0:
if count >= 5:
raise ValueError(
f'Failed setting ACLs after 5 attempts for sample {id_}')
acls = self._storage.get_sample_acls(id_)
self._check_perms(id_,
user,
_SampleAccessType.ADMIN,
acls,
as_admin=as_admin)
new_acls = SampleACL(acls.owner, self._now(), new_acls.admin,
new_acls.write, new_acls.read,
new_acls.public_read)
try:
self._storage.replace_sample_acls(id_, new_acls)
count = -1
except _OwnerChangedError:
count += 1
if self._kafka:
self._kafka.notify_sample_acl_change(id_)
def unitval(key: str, d1: Dict[str, PrimitiveType]) -> Optional[str]:
unitstr = d1.get(_cast(str, k))
if not unitstr:
return f'metadata value key {k} is required'
if type(unitstr) != str:
return f'metadata value key {k} must be a string'
try:
units = _UNIT_REG.parse_expression(unitstr)
except _UndefinedUnitError as e:
return f"unable to parse units '{u}' at key {k}: undefined unit: {e.args[0]}"
except _DefinitionSyntaxError as e:
return f"unable to parse units '{u}' at key {k}: syntax error: {e.args[0]}"
try:
(1 * units).ito(req_units)
except _DimensionalityError as e:
return (f"Units at key {k}, '{unitstr}', are not equivalent to " +
f"required units, '{u}': {e}")
return None
def NamedTemporaryFile(mode: str = 'w+b',
buffering: int = -1,
encoding: str = None,
newline: str = None,
suffix: str = "",
prefix: str = template,
dir: str = None,
delete: bool = True) -> _IO[_Any]:
"""Create and return a temporary file.
Arguments:
'prefix', 'suffix', 'dir' -- as for mkstemp.
'mode' -- the mode argument to io.open (default "w+b").
'buffering' -- the buffer size argument to io.open (default -1).
'encoding' -- the encoding argument to io.open (default None)
'newline' -- the newline argument to io.open (default None)
'delete' -- whether the file is deleted on close (default True).
The file is created as mkstemp() would do it.
Returns an object with a file-like interface; the name of the file
is accessible as file.name. The file will be automatically deleted
when it is closed unless the 'delete' argument is set to False.
"""
if dir is None:
dir = gettempdir()
flags = _bin_openflags
# Setting O_TEMPORARY in the flags causes the OS to delete
# the file when it is closed. This is only supported by Windows.
if _os.name == 'nt' and delete:
flags |= _os.O_TEMPORARY
(fd, name) = _mkstemp_inner(dir, prefix, suffix, flags)
file = _io.open(fd,
mode,
buffering=buffering,
newline=newline,
encoding=encoding)
return _cast(_IO[_Any], _TemporaryFileWrapper(file, name, delete))
def get_admin_request_from_object(
params: Dict[str, Any], as_admin: str, as_user: str) -> Tuple[bool, Optional[UserID]]:
'''
Get information about a request for administration mode from an object.
:param params: the dict containing the information.
:param as_admin: the name of the key containing a truish value that indicates that the user
wishes to be recognized as an administrator.
:param as_user: the name of the key containing a string that indicates the user the admin
wishes to impersonate, or None if the the admin does not wish to impersonate a user.
:returns: A tuple where the first element is a boolean denoting whether the user wishes
to be recognized as an administrator and the second element is the user that admin wishes
to impersonate, if any. The user is always None if the boolean is False.
'''
_check_params(params)
as_ad = bool(params.get(_cast(str, _check_string(as_admin, 'as_admin'))))
_check_string(as_user, 'as_user')
if not as_ad:
return (as_ad, None)
user = get_user_from_object(params, as_user)
return (as_ad, user)
def unitval(key: str,
d1: Dict[str, PrimitiveType]) -> Optional[ValidatorMessage]:
unitstr = d1.get(_cast(str, k))
if not unitstr:
return {
'subkey': str(k),
'message': f'metadata value key {k} is required'
}
if type(unitstr) != str:
return {
'subkey': str(k),
'message': f'metadata value key {k} must be a string'
}
try:
units = _UNIT_REG.parse_expression(unitstr)
except _UndefinedUnitError as e:
return {
'subkey':
str(k),
'message':
f'unable to parse units \'{u}\' at key {k}: undefined unit: {e.args[0]}'
}
except _DefinitionSyntaxError as e:
return {
'subkey':
str(k),
'message':
f'unable to parse units \'{u}\' at key {k}: syntax error: {e.args[0]}'
}
try:
# Here we attempt to convert a quantity of "1" in the provided unit to
# the canonical (also referred to as "example") unit provided in the
# validation spec.
pint.quantity.Quantity(1, units).to(req_units)
except _DimensionalityError as e:
msg = (f"Units at key {k}, '{unitstr}', are not equivalent to " +
f"required units, '{u}': {e}")
return {'subkey': str(k), 'message': msg}
return None
def __init__(self, bootstrap_servers: str, topic: str):
"""
Create the notifier.
:param bootstrap_servers: the Kafka bootstrap servers parameter.
:param topic: the topic where messages will be sent. The notifier requires the topic
name to consist of ASCII alphanumeric values and the hyphen to avoid Kafka issues
around ambiguity between period and underscore values.
"""
_check_string(bootstrap_servers, 'bootstrap_servers')
self._topic = _check_string(topic, 'topic', max_len=249)
match = self._KAFKA_TOPIC_ILLEGAL_CHARS_RE.search(
_cast(str, self._topic))
if match:
raise ValueError(
f'Illegal character in Kafka topic {self._topic}: {match.group()}'
)
# TODO LATER KAFKA support delivery.timeout.ms when the client supports it
# https://github.com/dpkp/kafka-python/issues/1723
# since not supported, we half ass it with a retry count.
# See https://kafka.apache.org/documentation/#producerconfigs
# this will fail if it can't connect
self._prod = _KafkaProducer(
# can't test multiple servers without a massive PITA
bootstrap_servers=bootstrap_servers.split(','),
acks='all',
# retries can occur from 100-30000 ms by default. If we allow 300 retries, that means
# the send can take from 30s to 150m. Presumably another server timeout will kill
# the request before then.
retries=300,
retry_backoff_ms=100, # default is 100
# low timeouts can cause message loss, apparently:
# https://github.com/dpkp/kafka-python/issues/1723
request_timeout_ms=30000, # default is 30000
# presumably this can be removed once idempotence is supported
max_in_flight_requests_per_connection=1,
)
self._closed = False
def strlen(key: str,
d1: Dict[str, PrimitiveType]) -> Optional[ValidatorMessage]:
for k in keys:
if required and k not in d1:
return {
'subkey': str(k),
'message': f'Required key {k} is missing'
}
v = d1.get(k)
if v is not None and type(v) != str:
return {
'subkey': str(k),
'message': f'Metadata value at key {k} is not a string'
}
if v and maxlen and len(_cast(str, v)) > maxlen:
return {
'subkey':
str(k),
'message':
f'Metadata value at key {k} is longer than max length of {maxlen}'
}
return None
