def sorted_diff(rs, ss):
try:
r, rs = peek(rs)
except StopIteration:
return
try:
s, ss = peek(ss)
except StopIteration:
for r in rs:
yield r
return
rtup = (r.start, r.stop)
stup = (s.start, s.stop)
if rtup == stup:
next(rs)
next(ss)
elif rtup < stup:
yield next(rs)
else:
next(ss)
for t in sorted_diff(rs, ss):
yield t
def list(self) -> QueryResult[Vehicle]:
responses = self.store.get("vehicle")
response, responses = toolz.peek(responses)
return QueryResultBuilder() \
.count(response.count) \
.iterator(toolz.mapcat(lambda r: (Vehicle(p) for p in r.results), responses)) \
.build()
def discover_chunks(c, **kwargs):
data = c.data
if isinstance(data, Iterator):
fst, c.data = peek(data)
else:
fst = first(c)
return var * discover(fst).subshape[0]
def discover_chunks(c, **kwargs):
data = c.data
if isinstance(data, Iterator):
fst, c.data = peek(data)
else:
fst = first(c)
return var * discover(fst).subshape[0]
def list(self) -> QueryResult[Starship]:
responses = self.store.get("planets")
response, responses = toolz.peek(responses)
return QueryResultBuilder() \
.count(response.count) \
.iterator(toolz.mapcat(lambda r: (Planet(p) for p in r.results), responses)) \
.build()
def execute(file_name):
categories = ['distinguished', 'removal_reason']
f = load(file_name)
batches = partition_all(200000, f)
df, frames = peek(map(to_df, batches))
castra = Castra('./subreddit_dumps/'+file_name+'.castra',
template = df, categories = categories)
castra.extend_sequence(frames, freq = '3h')
def _is_empty(iterable):
try:
_, iterable = toolz.peek(iterable)
empty = False
except StopIteration:
empty = True
return empty, iterable
def _is_empty(iterable):
try:
_, iterable = toolz.peek(iterable)
empty = False
except StopIteration:
empty = True
return empty, iterable
def template_to_ast(html):
"""
Converts cottonmouth type HTML to an "AST" representation, where there can be two kinds of
structural types - one is {tag, attrs, body} for the normal HTML, and the other is {name, props, children}
for the components.
There are two separate ones on purpose so its easy to tell which should be handled as a subcomponent and which
should be handled as an HTML tag as part of a vdom templating library.
Eg, in Elm this makes things a lot easier.
"""
if isinstance(html, str): return html
if isinstance(html, list):
hasattrs = len(html) > 1 and isinstance(html[1], dict)
tag, idsclasses = parsetag(html[0])
# combine attrs with idsclasses in the tag name. only modifies these two properties explicitly, so that
# other attrs and data for components is untouched.
attrs = html[1] if hasattrs else {}
return assemble_ast(tag, idsclasses, attrs, [
template_to_ast(x)
for x in html[(2 if hasattrs else 1):] if x is not None
])
if istag(html):
tag, html = t.peek(html)
if istag(tag):
return map(template_to_ast, html)
# this looks confusing, but it's to leave the generator at the right place,
# and also extract the tag and attrs.
# because I have to check the type of the next element, and optionally consume it if it's the right one
# I end up assigning next(html) to a particular variable twice.
tag, idsclasses = parsetag(next(html))
try:
attrs, html = t.peek(html)
attrs = next(html) if isinstance(attrs, dict) else {}
except StopIteration:
attrs = {}
html = iter([])
return assemble_ast(
tag, idsclasses, attrs,
[template_to_ast(x) for x in html if x is not None])
else:
raise ValueError(
'template_to_ast: Cannot convert the following type of value: ' +
str(html))
def execute(file_name):
categories = ['distinguished', 'removal_reason']
f = load(file_name)
batches = partition_all(200000, f)
df, frames = peek(map(to_df, batches))
castra = Castra('./subreddit_dumps/' + file_name + '.castra',
template=df,
categories=categories)
castra.extend_sequence(frames, freq='3h')
def to_labelled(collection: Dict[str, Sequence]) -> Iterator[LabelledData]:
seq = flatten_collection(collection)
(_, first), seq = tlz.peek(seq)
labeller: Callable
if isinstance(first, tuple):
labeller = _from_record
else:
labeller = _from_scalar
for label, data in seq:
yield labeller(label, data)
def template_to_ast(html):
"""
Converts cottonmouth type HTML to an "AST" representation, where there can be two kinds of
structural types - one is {tag, attrs, body} for the normal HTML, and the other is {name, props, children}
for the components.
There are two separate ones on purpose so its easy to tell which should be handled as a subcomponent and which
should be handled as an HTML tag as part of a vdom templating library.
Eg, in Elm this makes things a lot easier.
"""
if isinstance(html, str): return html
if isinstance(html, list):
hasattrs = len(html) > 1 and isinstance(html[1], dict)
tag, idsclasses = parsetag(html[0])
# combine attrs with idsclasses in the tag name. only modifies these two properties explicitly, so that
# other attrs and data for components is untouched.
attrs = html[1] if hasattrs else {}
return assemble_ast(tag, idsclasses, attrs, [template_to_ast(x) for x in html[(2 if hasattrs else 1):] if x is not None])
if istag(html):
tag,html = t.peek(html)
if istag(tag):
return map(template_to_ast, html)
# this looks confusing, but it's to leave the generator at the right place,
# and also extract the tag and attrs.
# because I have to check the type of the next element, and optionally consume it if it's the right one
# I end up assigning next(html) to a particular variable twice.
tag, idsclasses = parsetag(next(html))
try:
attrs, html = t.peek(html)
attrs = next(html) if isinstance(attrs, dict) else {}
except StopIteration:
attrs = {}
html = iter([])
return assemble_ast(tag, idsclasses, attrs, [template_to_ast(x) for x in html if x is not None])
else: raise ValueError('template_to_ast: Cannot convert the following type of value: ' + str(html))
def get(self, resource) -> Iterator[APIResponse]:
responses = self.store.get(resource)
try:
# We need to peek in order to know if we got something or not.
# An Iterator will be returned or an StopIteration will be raised.
_, responses = toolz.peek(responses)
except StopIteration:
responses = None
if responses:
for r in responses:
yield r
else:
for r in self._fetch_from_swapi(resource):
yield r
def _combine(n, rs):
"""helper for ``_group_ranges``"""
try:
r, rs = peek(rs)
except StopIteration:
yield n
return
if overlap(n, r):
yield merge(n, r)
next(rs)
for r in rs:
yield r
else:
yield n
for r in rs:
yield r
def _combine(n, rs):
"""helper for ``_group_ranges``
"""
try:
r, rs = peek(rs)
except StopIteration:
yield n
return
if overlap(n, r):
yield merge(n, r)
next(rs)
for r in rs:
yield r
else:
yield n
for r in rs:
yield r
