def test_repr_doesnt_get_called_when_not_necessary(self): class Something(object): def __repr__(self): assert False, "__repr__ called when not necessary" foreach(Something()) unless(Exception, Something())
def get(self, request, data): bounds = data > maybe | X.get('in_polygon') return ((data or {}) > as_kwargs(get_car_position_data) | where(X['car'] | self.filter) | group_by(X['location'] | (grouping_precision, X, bounds)) | X.iteritems() | foreach({ 'location': X[1][0]['location'], 'cars': X[1] | foreach(X['car']) | self.get_car_data, }) | tuple)
def create_journeys(start, end, car, user=None, split_by=None, **kwargs): from metrocar.car_unit_api.models import CarUnit unit = CarUnit.objects.get_for(car) if not unit: raise ImproperlyConfigured("No CarUnit associated with %s" % car) return (get_journey_data(start, end, unit.unit_id, split_by=split_by) > pipe | foreach(X['entries']) | foreach(create_journey, car=car, user=user, **kwargs) | tuple)
def _get_search_fields(self): return [ self._get_fields(lambda f: type(f) in self.search_field_types), # if there are any ForeignKeys to User, we'd like to be able to # search by the user's last_name, username and email (self.all_fields > pipe | where(isinstance, X, ForeignKey) | where(X.related.parent_model | (issubclass, X, User)) | foreach(X.name) | foreach(['{0}__last_name', '{0}__username', '{0}__email'])) ] > flatten | tuple
def test_pipe_exception(): "How an exception in a pipe should look like" f = pipe | str | foreach(X.upper()) | foreach(~X.lower() | int) | list with pytest.raises(ValueError) as excinfo: f('adfasfa') assert excinfo.value.message == ( "invalid literal for int() with base 10: 'a'\n" " in pipe | X.lower | X() | int\n" " in pipe | str | foreach(X.upper | X()) | foreach(X.lower | X() | int) | list" )
def query_last_position(**kwargs): return query(model=LastKnownPosition, **kwargs) > foreach([ X.unit_id, { 'timestamp': X.timestamp, 'location': X.location | decode_location, } ]) | dict
def f(seq): if max > len(seq): return seq take_every_xth = len(seq) / float(max) indexes = xrange(r_int(max) - 1) > foreach((X * take_every_xth) | r_int) return [seq[i] for i in indexes] + [seq[-1]]
def fetch_problem_list(): url = 'https://leetcode-cn.com/api/problems/all/' r = requests.get(url) r.raise_for_status() data = r.json() def item_to_problem(item): id = item['stat']['question_id'] title = item['stat']['question__title'] title_slug = item['stat']['question__title_slug'] difficulty = item['difficulty']['level'] paid_only = item['paid_only'] problem = Problem(id=id, title=title, title_slug=title_slug, difficulty=Difficulty(difficulty), paid_only=paid_only) return problem problems = list(data['stat_status_pairs'] > pipe | foreach(item_to_problem) | where(X.id < 10000)) return problems
def import_csv(unit_id, filename, make_entry): (filename > pipe | open | DictReader | foreach(make_entry) | tuple | (store, unit_id))
def import_query(potential_module_names): query_module = first_of(potential_module_names > foreach(unless(ImportError, import_module))) try: return query_module and query_module.execute except AttributeError: raise NotImplementedError('The "%s" module does not provide the ' 'execute method.' % query_module)
def process_raw_results(data, max_items): return (data > pipe | split_entry_data_to_journeys | (prune_journeys_to, max_items) | foreach({ 'entries': X['entries'] | group_by_location, 'events': X['events'], }) )
def fetch_problem_submissions(title_slug, offset=0, limit=50): query = '''query Submissions($offset: Int!, $limit: Int!, $lastKey: String, $questionSlug: String!) { submissionList(offset: $offset, limit: $limit, lastKey: $lastKey, questionSlug: $questionSlug) { lastKey hasNext submissions { id statusDisplay lang timestamp url } } } ''' payload = { "operationName": "Submissions", "variables": { 'offset': offset, 'limit': limit, "questionSlug": title_slug, }, "query": query, } headers = { 'user-agent': user_agent, 'cookie': cookie, 'referer': problem_url, 'content-type': "application/json", } data = execute_graphql(api_url, payload=payload, headers=headers) def item_to_submission(item): id = item['id'] status = Status(item['statusDisplay']) language = Language(item['lang']) timestamp = item['timestamp'] url = urljoin(base_url, item['url']) return Submission(id=id, status=status, language=language, timestamp=timestamp, url=url) def fill_submission_code(submission): submission.code = fetch_submission_code(submission.url) return submission submissions = list(data['submissionList']['submissions'] > pipe | foreach(item_to_submission) | where(X.status == Status.AC) # | foreach(fill_submission_code) ) return submissions
def update_car_status(unit_id, entries): """ Updates Car model from `entries` received through the API. """ car = CarUnit.objects.get(unit_id=unit_id).car car and (entries > pipe | foreach(with_parsed_timestamp) | sort_by(X['timestamp']) | first_of | (update_car, car))
def validate_each(*rules): """ Like :func:`validate`, but checks each item in `iterable` separately against `rules`. Also the first encountered error is returned without checking the rest of the items. """ return lambda val: ( (val > foreach(validate(*rules)) | select_first(X != OK) or OK) if isinstance(val, Iterable) and not isinstance(val, basestring) else (False, '"{0}" is not iterable'.format(val)))
def group_by_location(entries): """ Some entries could be on the same location, especially ones containing interesting events, like ENGINE_OFF, LOCKED, etc. Grouping will make displaying them on a map much easier. This function assumes that `entries` are sorted by timestamp. """ return groupby(entries, ~X['location']) > foreach({ 'location': X[0], 'entries': X[1] | tuple }) | tuple
def prune_journeys_to(max_items, journeys): """ For each journey we need first and last entry and overall at most `max_items` worth of entries. """ entry_count = X['entries'] | len total_entries = journeys > foreach(entry_count) | sum for j in journeys: journey_max_items = max(2, max_items * (entry_count(j) / float(total_entries))) yield dict(j, entries=prune_to(journey_max_items)(j['entries']))
def link_to_solutions(problem: Problem): solution_dir = Path('solutions') problem_dir = solution_dir / problem.dir_name() if not problem_dir.exists(): return [] links = ( problem_dir.iterdir() > pipe | foreach(X.name) | foreach(SubmissionFileName.from_str) # Select one newest submission from each language | group_by(X.language) | foreach(X[1]) | foreach(lambda submissions: max(submissions, key=lambda s: s.timestamp)) # Sort | sort_by(X.language.order()) # Format to links | foreach( lambda file: f'[{file.language.display()}]({problem_dir / file.to_str()})')) links = list(links) return links
def _current_position_data(geotrack_data): """ Add Cars to geotrack_data for current car positions. """ unit_data = geotrack_data.items() unit_ids = geotrack_data.keys() units = CarUnit.objects.filter(unit_id__in=unit_ids).select_related('car') unit_dict = units > foreach([X.unit_id, X]) | dict for unit_id, data in unit_data: # in case a CarUnit was deleted... if unit_id in unit_dict: yield { 'location': data['location'], 'timestamp': data['timestamp'], 'car': unit_dict[unit_id].car, }
def render_readme(): problems = fetch_problem_list() problems_for_render = (problems > pipe | sort_by(X.id) | foreach(ProblemForRender.from_problem) | list) env = Environment(loader=PackageLoader('coder', 'templates'), trim_blocks=True) template = env.get_template('README.md') rendered = template.render(problems=problems_for_render) with open('README.md', 'w') as f: print(rendered, file=f)
def validate(*rules): """ Creates a validation function that will check if it's input satisfies `rules`. `rules` should be an (arbitrarily nested) sequence of functions that take one argument and return a tuple of: ``(valid: bool, error: string or None)`` where `valid` says if the argument satisfies the rule and `error` says why not if it doesn't. The rules are checked sequentially and when an error is encountered, it is returned immediately from the function, without checking the rest of the rules. The returned value is the one returned from the rule, i.e. ``(False, "error message")`` If no error is encountered the function returns ``(True, None)``. (Note that the validation function itself is a rule.) """ rules = rules > flatten | tuple return lambda val: rules > foreach(X(val)) | select_first(X != OK) or OK
def test_with_exception(self): f = unless(AttributeError, foreach(X.lower()) | list) assert f(['A', 'B', 37]) is None
def test_ok(self): f = unless(AttributeError, foreach(X.lower())) | list assert f("ABC") == ['a', 'b', 'c']
def test_foreach_format(self): result = [1, 2] > foreach("Number {0}") | list assert result == ['Number 1', 'Number 2']
def test_make_list(self): result = [1, 2, 3] > foreach([X, X % 2]) | list assert result == [[1, 1], [2, 0], [3, 1]]
See the `full documentation <{0}#contents>`_. """.format(DOC_ROOT) link_template = u"`{text} <%s{url}>`_" % DOC_ROOT link_replacements = ( # :doc:`pipe-utils' documentation<pipeutils>`. (r":doc:`([^<]*)<([^>]*)>`", {'url': r'\2.html', 'text': r'\1'}), # :func:`~pipetools.utils.where` (r":func:`~pipetools\.utils\.([^`]*)`", {'url': r'pipeutils.html#pipetools.utils.\1', 'text': r'\1()'}), ) > foreach([X[0] | re.compile, X[1] | link_template]) def create_readme(): with codecs.open('docs/source/overview.rst', 'r', 'utf-8') as overview: with codecs.open('README.rst', 'w+', 'utf-8') as readme: overview.read() > pipe | fix_links | readme_template | readme.write def fix_links(string): for pattern, replacement in link_replacements: string = pattern.sub(replacement, string) return string commit_readme = """
def test_partially_applied(self): f = foreach(my_func, 42, kwarg=2) assert repr(f) == 'foreach(my_func, 42, kwarg=2)'
def __repr__(self): return (self.iteritems() > foreach(' {0}={1!r}') | sort | '\n'.join | '<Bunch\n{0}>')
def test_pipability(self): f = range | foreach(X) | sum result = f(4) assert result == 6
def test_with_exception_in_foreach(self): f = foreach(unless(AttributeError, X.lower())) | list assert f(['A', 'B', 37]) == ['a', 'b', None]
def test_basic(self): f = foreach(my_func) assert repr(f) == 'foreach(my_func)'
def _get_position_data_from_cars(): return Car.objects.all() > foreach(car_position_data)
def test_string_formatting(self): f = foreach("{0} asdf {1} jk;l") assert repr(f) == "foreach('{0} asdf {1} jk;l')"
def test_make_dict(self): result = [1, 2] > foreach({'num': X, 'str': str}) | list assert result == [{'num': 1, 'str': '1'}, {'num': 2, 'str': '2'}]
def test_pipe_util_xpartial(): f = range | foreach(range, X, 0, -1) | foreach(list) | list assert f(3, 5) == [[3, 2, 1], [4, 3, 2, 1]]
def execute(backend, query, max_items=50, **kwargs): return (query(**kwargs) > group_by(X['unit_id']) | X.iteritems() | foreach([KEY, VALUE | prune_to(max_items)]) | dict)
def test_make_tuple(self): result = [1, 2, 3] > foreach((X, X % 2)) | list assert result == [(1, 1), (2, 0), (3, 1)]