def save_from(self, source: typing.Iterable, rechunk=True, executor=None):
"""Iterate over source and save the results under key
along with metadata
"""
if rechunk and self.allow_rechunk:
source = strax.fixed_size_chunks(source)
pending = []
try:
for chunk_i, s in enumerate(source):
new_f = self.save(data=s, chunk_i=chunk_i, executor=executor)
if new_f is not None:
pending = [f for f in pending + [new_f] if not f.done()]
except strax.MailboxKilled:
# Write exception (with close), but exit gracefully.
# One traceback on screen is enough
self.close(wait_for=pending)
pass
except Exception as e:
# log exception for the final check
self.got_exception = e
# Throw the exception back into the mailbox
# (hoping that it is still listening...)
source.throw(e)
raise e
finally:
if not self.closed:
self.close(wait_for=pending)
def process_images(
df: pd.DataFrame,
rescaled_dim: Iterable(int, int) = None,
depth: int = 3) -> Iterable(np.ndarray, np.ndarray, np.ndarray):
"""
Function to process all images and store them in one 4 dimensional numpy array, with dimensions
[n, w, h, d] -> n = number of images, w = width, h =height, d = depth
:param df: DataFrame containing the file_paths to each image
:param rescaled_dim: iterable containing the final dimensions of each image
:param depth: the image depth
:return: 4D numpy array
"""
if not rescaled_dim:
rescaled_dim = [256, 256]
images_array = np.zeros([len(df), rescaled_dim[0], rescaled_dim[1], depth])
types = []
file_names = []
assert "file_path" in df, "column 'file_path' not in DataFrame"
for i, f in enumerate(df["file_path"]):
file_names.append(os.path.split(f)[1])
types.append(df.loc[df["file_path"] == f, "type"].squeeze())
img = ip.get_image_data(file_path=f)
images_array[i, :, :, :] = transform_image(img=img,
rescaled_dim=rescaled_dim)
# imgs_mat = np.array(imgages_array).squeeze()
# df_img_mat = pd.DataFrame(imgs_mat)
# df_img_mat.insert(loc=0, column="image", value=file_names)
# df_img_mat.insert(loc=1, column="type", value=types)
return images_array, np.array(types), np.array(file_names)
def _find_shapes(self, label: str) -> Iterable(BaseShape):
""" Finds all shapes that match the label
Args:
label (str): label of the placeholder (without curly braces)
"""
pass
def __init__(self, iterable: typing.Iterable = (), metadata: typing.Dict[str, typing.Any] = None, *,
generate_metadata: bool = False, check: bool = True, source: typing.Any = None,
timestamp: datetime.datetime = None) -> None:
if isinstance(iterable, pandas.DataFrame):
super().__init__(type(self)(row) for row in iterable.itertuples(index=False, name=None))
else:
if isinstance(iterable, numpy.matrix):
# One cannot iterate over a matrix segment by segment. You always get back
# a matrix (2D structure) and not an array of rows or columns. By converting
# it to an array such iteration segment by segment works.
iterable = numpy.array(iterable)
super().__init__(iterable)
from d3m import types
if isinstance(iterable, types.Container):
if isinstance(iterable, List):
# We made a copy, so we do not have to generate metadata.
self.metadata: metadata_base.DataMetadata = iterable.metadata
else:
self.metadata: metadata_base.DataMetadata = iterable.metadata
if generate_metadata:
self.metadata = self.metadata.generate(self)
if metadata is not None:
self.metadata: metadata_base.DataMetadata = self.metadata.update((), metadata)
else:
self.metadata: metadata_base.DataMetadata = metadata_base.DataMetadata(metadata)
if generate_metadata:
self.metadata = self.metadata.generate(self)
def any_starts_with(iterable: typing.Iterable, char: str) -> bool:
"""Return True if any string(s) in nested iterable starts with a given character.
:param iterable iterable: Nested iterable of strings to check
:param str char: Char to check each string
:return bool: True if any string in nested iterable starts with char, else False
"""
if isinstance(iterable, str):
return iterable.startswith(char)
recurse = functools.partial(any_starts_with, char=char)
return any(map(recurse, iterable))
def get_starting_ability_bonus(self) -> Iterable(AbilityScores):
"""
Gets the possible AbilityScore bonuses for a given race.
Normally, races give the creator of the character a simple bonus to a
subset of AbilityClasses (e.g. +1 to Intelligence).
However, some races give the creator a choice of which AbilityClass
they can choose to increase or decrease certain starting statistics.
As such, this function may instead return the possible AbilityScore
bonuses that may be used when creating this particular race.
:returns All possible AbilityScores for the race's starting bonuses
"""
...
def gen_requests(num: int = DEFAULT_TEST_SIZE, verb: str = 'GET', body_iter: typing.Iterable = None):
"""
Utility function for generating request-type objects
:param num: how many requests to generate
:param verb: idempotent verb (GET, PUT, etc) TODO validate this input
:param body_iter: iterable of JSON body data e.g. a hypothesis start to generate valid json
:return: generator of requsts.Request objects
"""
body_iter = None if body_iter is None else iter(body_iter)
for i in range(num):
if body_iter is not None:
body = body_iter.__next__()
else:
body = None
yield requests.Request(verb, f'https://test-request/{i}', data=body)
def get_starting_skill_bonus(self) -> Iterable(SkillSet):
"""
Gets the possible SkillSet bonuses for a given race.
Normally, races give the creator of the character a simple bonus to a
subset of skills (e.g. +1 to Swim).
However, some races give the creator a choice of which SkillClass
they can choose to increase or decrease certain starting statistics.
As such, this function may instead return the possible SkillSet
bonuses that may be used when creating this particular race.
:returns All possible SkillSets for the race's starting bonuses
"""
...
def random_recipe(s: sa.orm.Session, *, n: int = 1) -> Iterable(Recipe):
"""
Get `n` random recipes.
Parameters
----------
s : sqlalchemy.orm.Session
database session to bind objects
n : int = [default: 1]
number of recipes to return
"""
q = s.query(Recipe)\
.order_by(sa.func.random())\
.limit(n)
return iter(q.all())
def iter(
self, value: typing.Iterable
) -> typing.Generator[typing.Any, None, None]:
"""
Iterator version of :py:meth:`apply`.
"""
if value is not None:
if isinstance(value, typing.Mapping):
for k, v in value.items():
u_key = self.unicodify_key(k)
if ((self.restrict_keys is None)
or (k in self.restrict_keys)):
yield k, self._apply_item(u_key, v, self._filter_chain)
else:
# For consistency with FilterMapper, invalid
# keys are not included in the filtered
# value (hence this statement does not
# ``yield``).
self._invalid_value(
value=v,
reason=self.CODE_EXTRA_KEY,
sub_key=u_key,
)
else:
for i, v in enumerate(value):
u_key = self.unicodify_key(i)
if ((self.restrict_keys is None)
or (i in self.restrict_keys)):
yield self._apply_item(u_key, v, self._filter_chain)
else:
# Unlike in mappings, it is not possible to
# identify a "missing" item in a collection,
# so we have to ensure that something ends up
# in the filtered value at the same position
# as the invalid incoming value.
yield self._invalid_value(
value=v,
reason=self.CODE_EXTRA_KEY,
sub_key=u_key,
)
def head(it: _typ.Iterable) -> _typ.Optional[_typ.Any]:
# Convert list to iterator if necessary
try:
new_it = (x for x in it)
it = new_it
except:
raise
# Try to get next element of iterator
try:
return it.__next__()
except AttributeError:
raise
# NoneType provided
return None
except NameError:
# Not an iterator
return None
except StopIteration:
# Iterator empty
return None
def newRow(self, values: _tp.Iterable = None) -> _MutableRow:
"""
Returns a new MutableRow instance (optionally populated with data).
:param values: Optional ``list`` or ``tuple`` of field values in the correct ``InsertCursor`` field order
or a ``dict`` of key-value pairs, where the keys specify the field names to set.
:raises ValueError: If *values* is a ``list`` or ``tuple`` and the length does not match the number of
cursor fields, or if *values* is a ``dict`` and one of the keys does not match with the
cursor field names.
"""
_vld.raise_if(values and not _vld.is_iterable(values), ValueError,
"newRow() 'values' should be iterable or None")
# Although it would be more efficient to initialize _MutableRow once and simply call it
# to set its values, this might not be what the user expects. Therefore, we (re)initialize it each time.
if isinstance(values, dict):
row = _MutableRow(self._field_map)
for k, v in values.items():
row.setValue(k, v)
else:
row = _MutableRow(self._field_map)(values)
return row
def keys(self) -> Iterable():
if self.__empty__():
return Queue()
return self.keyss(self.min(), self.max())
'''
Remove All Before
Not all of the elements are important. What you need to do here is to remove from the list all of the elements before the given one.
example
For the illustration we have a list [3, 4, 5] and we need to remove all elements that go before 3 - which is 1 and 2.
We have two edge cases here: (1) if a cutting element cannot be found, then the list shoudn't be changed. (2) if the list is empty, then it should remain empty.
Input:
List and the border element.
Output:
Iterable (tuple, list, iterator ...).
Example:
remove_all_before([1, 2, 3, 4, 5], 3) == [3, 4, 5]
remove_all_before([1, 1, 2, 2, 3, 3], 2) == [2, 2, 3, 3]
from typing import Iterable
def remove_all_before(items: list, border: int) -> Iterable:
# your code here
return items
if __name__ == '__main__':