def test_repr_doesnt_get_called_when_not_necessary(self):
class Something(object):
def __repr__(self):
assert False, "__repr__ called when not necessary"
foreach(Something())
unless(Exception, Something())
def test_repr_doesnt_get_called_when_not_necessary(self):
class Something(object):
def __repr__(self):
assert False, "__repr__ called when not necessary"
foreach(Something())
unless(Exception, Something())
def get(self, request, data):
bounds = data > maybe | X.get('in_polygon')
return ((data or {}) > as_kwargs(get_car_position_data)
| where(X['car'] | self.filter)
| group_by(X['location'] | (grouping_precision, X, bounds))
| X.iteritems()
| foreach({
'location': X[1][0]['location'],
'cars': X[1] | foreach(X['car']) | self.get_car_data,
})
| tuple)
def create_journeys(start, end, car, user=None, split_by=None, **kwargs):
from metrocar.car_unit_api.models import CarUnit
unit = CarUnit.objects.get_for(car)
if not unit:
raise ImproperlyConfigured("No CarUnit associated with %s" % car)
return (get_journey_data(start, end, unit.unit_id, split_by=split_by) > pipe
| foreach(X['entries'])
| foreach(create_journey, car=car, user=user, **kwargs)
| tuple)
def get(self, request, data):
bounds = data > maybe | X.get('in_polygon')
return ((data or {}) > as_kwargs(get_car_position_data)
| where(X['car'] | self.filter)
| group_by(X['location'] | (grouping_precision, X, bounds))
| X.iteritems()
| foreach({
'location': X[1][0]['location'],
'cars': X[1] | foreach(X['car']) | self.get_car_data,
})
| tuple)
def _get_search_fields(self):
return [
self._get_fields(lambda f: type(f) in self.search_field_types),
# if there are any ForeignKeys to User, we'd like to be able to
# search by the user's last_name, username and email
(self.all_fields > pipe
| where(isinstance, X, ForeignKey)
| where(X.related.parent_model | (issubclass, X, User))
| foreach(X.name)
| foreach(['{0}__last_name', '{0}__username', '{0}__email']))
] > flatten | tuple
def test_pipe_exception():
"How an exception in a pipe should look like"
f = pipe | str | foreach(X.upper()) | foreach(~X.lower() | int) | list
with pytest.raises(ValueError) as excinfo:
f('adfasfa')
assert excinfo.value.message == (
"invalid literal for int() with base 10: 'a'\n"
" in pipe | X.lower | X() | int\n"
" in pipe | str | foreach(X.upper | X()) | foreach(X.lower | X() | int) | list"
)
def query_last_position(**kwargs):
return query(model=LastKnownPosition, **kwargs) > foreach([
X.unit_id, {
'timestamp': X.timestamp,
'location': X.location | decode_location,
}
]) | dict
def f(seq):
if max > len(seq):
return seq
take_every_xth = len(seq) / float(max)
indexes = xrange(r_int(max) - 1) > foreach((X * take_every_xth)
| r_int)
return [seq[i] for i in indexes] + [seq[-1]]
def fetch_problem_list():
url = 'https://leetcode-cn.com/api/problems/all/'
r = requests.get(url)
r.raise_for_status()
data = r.json()
def item_to_problem(item):
id = item['stat']['question_id']
title = item['stat']['question__title']
title_slug = item['stat']['question__title_slug']
difficulty = item['difficulty']['level']
paid_only = item['paid_only']
problem = Problem(id=id,
title=title,
title_slug=title_slug,
difficulty=Difficulty(difficulty),
paid_only=paid_only)
return problem
problems = list(data['stat_status_pairs'] > pipe
| foreach(item_to_problem)
| where(X.id < 10000))
return problems
def import_csv(unit_id, filename, make_entry):
(filename > pipe
| open
| DictReader
| foreach(make_entry)
| tuple
| (store, unit_id))
def import_query(potential_module_names):
query_module = first_of(potential_module_names >
foreach(unless(ImportError, import_module)))
try:
return query_module and query_module.execute
except AttributeError:
raise NotImplementedError('The "%s" module does not provide the '
'execute method.' % query_module)
def process_raw_results(data, max_items):
return (data > pipe
| split_entry_data_to_journeys
| (prune_journeys_to, max_items)
| foreach({
'entries': X['entries'] | group_by_location,
'events': X['events'],
})
)
def fetch_problem_submissions(title_slug, offset=0, limit=50):
query = '''query Submissions($offset: Int!, $limit: Int!, $lastKey: String, $questionSlug: String!) {
submissionList(offset: $offset, limit: $limit, lastKey: $lastKey, questionSlug: $questionSlug) {
lastKey
hasNext
submissions {
id
statusDisplay
lang
timestamp
url
}
}
}
'''
payload = {
"operationName": "Submissions",
"variables": {
'offset': offset,
'limit': limit,
"questionSlug": title_slug,
},
"query": query,
}
headers = {
'user-agent': user_agent,
'cookie': cookie,
'referer': problem_url,
'content-type': "application/json",
}
data = execute_graphql(api_url, payload=payload, headers=headers)
def item_to_submission(item):
id = item['id']
status = Status(item['statusDisplay'])
language = Language(item['lang'])
timestamp = item['timestamp']
url = urljoin(base_url, item['url'])
return Submission(id=id,
status=status,
language=language,
timestamp=timestamp,
url=url)
def fill_submission_code(submission):
submission.code = fetch_submission_code(submission.url)
return submission
submissions = list(data['submissionList']['submissions'] > pipe
| foreach(item_to_submission)
| where(X.status == Status.AC)
# | foreach(fill_submission_code)
)
return submissions
def update_car_status(unit_id, entries):
"""
Updates Car model from `entries` received through the API.
"""
car = CarUnit.objects.get(unit_id=unit_id).car
car and (entries > pipe
| foreach(with_parsed_timestamp)
| sort_by(X['timestamp'])
| first_of
| (update_car, car))
def update_car_status(unit_id, entries):
"""
Updates Car model from `entries` received through the API.
"""
car = CarUnit.objects.get(unit_id=unit_id).car
car and (entries > pipe
| foreach(with_parsed_timestamp)
| sort_by(X['timestamp'])
| first_of
| (update_car, car))
def validate_each(*rules):
"""
Like :func:`validate`, but checks each item in `iterable` separately
against `rules`.
Also the first encountered error is returned without checking the rest
of the items.
"""
return lambda val: (
(val > foreach(validate(*rules)) | select_first(X != OK) or OK)
if isinstance(val, Iterable) and not isinstance(val, basestring)
else (False, '"{0}" is not iterable'.format(val)))
def group_by_location(entries):
"""
Some entries could be on the same location, especially ones containing
interesting events, like ENGINE_OFF, LOCKED, etc. Grouping will make
displaying them on a map much easier.
This function assumes that `entries` are sorted by timestamp.
"""
return groupby(entries, ~X['location']) > foreach({
'location': X[0],
'entries': X[1] | tuple
}) | tuple
def prune_journeys_to(max_items, journeys):
"""
For each journey we need first and last entry and overall at most
`max_items` worth of entries.
"""
entry_count = X['entries'] | len
total_entries = journeys > foreach(entry_count) | sum
for j in journeys:
journey_max_items = max(2,
max_items * (entry_count(j) / float(total_entries)))
yield dict(j, entries=prune_to(journey_max_items)(j['entries']))
def link_to_solutions(problem: Problem):
solution_dir = Path('solutions')
problem_dir = solution_dir / problem.dir_name()
if not problem_dir.exists():
return []
links = (
problem_dir.iterdir() > pipe
| foreach(X.name)
| foreach(SubmissionFileName.from_str)
# Select one newest submission from each language
| group_by(X.language)
| foreach(X[1])
| foreach(lambda submissions: max(submissions,
key=lambda s: s.timestamp))
# Sort
| sort_by(X.language.order())
# Format to links
| foreach(
lambda file:
f'[{file.language.display()}]({problem_dir / file.to_str()})'))
links = list(links)
return links
def _current_position_data(geotrack_data):
"""
Add Cars to geotrack_data for current car positions.
"""
unit_data = geotrack_data.items()
unit_ids = geotrack_data.keys()
units = CarUnit.objects.filter(unit_id__in=unit_ids).select_related('car')
unit_dict = units > foreach([X.unit_id, X]) | dict
for unit_id, data in unit_data:
# in case a CarUnit was deleted...
if unit_id in unit_dict:
yield {
'location': data['location'],
'timestamp': data['timestamp'],
'car': unit_dict[unit_id].car,
}
def render_readme():
problems = fetch_problem_list()
problems_for_render = (problems > pipe
| sort_by(X.id)
| foreach(ProblemForRender.from_problem)
| list)
env = Environment(loader=PackageLoader('coder', 'templates'),
trim_blocks=True)
template = env.get_template('README.md')
rendered = template.render(problems=problems_for_render)
with open('README.md', 'w') as f:
print(rendered, file=f)
def _current_position_data(geotrack_data):
"""
Add Cars to geotrack_data for current car positions.
"""
unit_data = geotrack_data.items()
unit_ids = geotrack_data.keys()
units = CarUnit.objects.filter(unit_id__in=unit_ids).select_related('car')
unit_dict = units > foreach([X.unit_id, X]) | dict
for unit_id, data in unit_data:
# in case a CarUnit was deleted...
if unit_id in unit_dict:
yield {
'location': data['location'],
'timestamp': data['timestamp'],
'car': unit_dict[unit_id].car,
}
def validate(*rules):
"""
Creates a validation function that will check if it's input satisfies
`rules`.
`rules` should be an (arbitrarily nested) sequence of functions
that take one argument and return a tuple of:
``(valid: bool, error: string or None)``
where `valid` says if the argument satisfies the rule and `error` says
why not if it doesn't.
The rules are checked sequentially and when an error is encountered, it is
returned immediately from the function, without checking the rest of the
rules. The returned value is the one returned from the rule, i.e.
``(False, "error message")``
If no error is encountered the function returns ``(True, None)``.
(Note that the validation function itself is a rule.)
"""
rules = rules > flatten | tuple
return lambda val: rules > foreach(X(val)) | select_first(X != OK) or OK
def test_with_exception(self):
f = unless(AttributeError, foreach(X.lower()) | list)
assert f(['A', 'B', 37]) is None
def test_ok(self):
f = unless(AttributeError, foreach(X.lower())) | list
assert f("ABC") == ['a', 'b', 'c']
def test_foreach_format(self):
result = [1, 2] > foreach("Number {0}") | list
assert result == ['Number 1', 'Number 2']
def test_make_list(self):
result = [1, 2, 3] > foreach([X, X % 2]) | list
assert result == [[1, 1], [2, 0], [3, 1]]
def f(seq):
if max > len(seq):
return seq
take_every_xth = len(seq) / float(max)
indexes = xrange(r_int(max) - 1) > foreach((X * take_every_xth) | r_int)
return [seq[i] for i in indexes] + [seq[-1]]
See the `full documentation <{0}#contents>`_.
""".format(DOC_ROOT)
link_template = u"`{text} <%s{url}>`_" % DOC_ROOT
link_replacements = (
# :doc:`pipe-utils' documentation<pipeutils>`.
(r":doc:`([^<]*)<([^>]*)>`", {'url': r'\2.html', 'text': r'\1'}),
# :func:`~pipetools.utils.where`
(r":func:`~pipetools\.utils\.([^`]*)`",
{'url': r'pipeutils.html#pipetools.utils.\1', 'text': r'\1()'}),
) > foreach([X[0] | re.compile, X[1] | link_template])
def create_readme():
with codecs.open('docs/source/overview.rst', 'r', 'utf-8') as overview:
with codecs.open('README.rst', 'w+', 'utf-8') as readme:
overview.read() > pipe | fix_links | readme_template | readme.write
def fix_links(string):
for pattern, replacement in link_replacements:
string = pattern.sub(replacement, string)
return string
commit_readme = """
def test_partially_applied(self):
f = foreach(my_func, 42, kwarg=2)
assert repr(f) == 'foreach(my_func, 42, kwarg=2)'
def __repr__(self):
return (self.iteritems() >
foreach(' {0}={1!r}') | sort | '\n'.join | '<Bunch\n{0}>')
def test_with_exception(self):
f = unless(AttributeError, foreach(X.lower()) | list)
assert f(['A', 'B', 37]) is None
def test_ok(self):
f = unless(AttributeError, foreach(X.lower())) | list
assert f("ABC") == ['a', 'b', 'c']
def test_foreach_format(self):
result = [1, 2] > foreach("Number {0}") | list
assert result == ['Number 1', 'Number 2']
def test_pipability(self):
f = range | foreach(X) | sum
result = f(4)
assert result == 6
def test_with_exception_in_foreach(self):
f = foreach(unless(AttributeError, X.lower())) | list
assert f(['A', 'B', 37]) == ['a', 'b', None]
def test_with_exception_in_foreach(self):
f = foreach(unless(AttributeError, X.lower())) | list
assert f(['A', 'B', 37]) == ['a', 'b', None]
def test_basic(self):
f = foreach(my_func)
assert repr(f) == 'foreach(my_func)'
def _get_position_data_from_cars():
return Car.objects.all() > foreach(car_position_data)
def test_string_formatting(self):
f = foreach("{0} asdf {1} jk;l")
assert repr(f) == "foreach('{0} asdf {1} jk;l')"
def test_make_dict(self):
result = [1, 2] > foreach({'num': X, 'str': str}) | list
assert result == [{'num': 1, 'str': '1'}, {'num': 2, 'str': '2'}]
def test_pipe_util_xpartial():
f = range | foreach(range, X, 0, -1) | foreach(list) | list
assert f(3, 5) == [[3, 2, 1], [4, 3, 2, 1]]
def test_pipability(self):
f = range | foreach(X) | sum
result = f(4)
assert result == 6
def execute(backend, query, max_items=50, **kwargs):
return (query(**kwargs)
> group_by(X['unit_id'])
| X.iteritems()
| foreach([KEY, VALUE | prune_to(max_items)])
| dict)
def test_make_tuple(self):
result = [1, 2, 3] > foreach((X, X % 2)) | list
assert result == [(1, 1), (2, 0), (3, 1)]
def test_make_dict(self):
result = [1, 2] > foreach({'num': X, 'str': str}) | list
assert result == [{'num': 1, 'str': '1'}, {'num': 2, 'str': '2'}]
def test_make_list(self):
result = [1, 2, 3] > foreach([X, X % 2]) | list
assert result == [[1, 1], [2, 0], [3, 1]]
