def main(data_fol, data_file): """Extract and write air traffic information between countries Parameters: ----------- data_fol : partial path.join function Partial path function that links to the data directory data_file : str File name of the input file """ yr_range = tuple(map(str, range(2006, 2016))) fname = data_fol(data_file) air_traffic = defaultdict(float) mean_func = compose(np.mean, list, cfilter(lambda x: x is not None), cmap(catch_int_except)) with open(fname, 'r') as fid: header_finfo, *header_tinfo = next(fid).split('\t') header_tinfo = [d.strip() for d in header_tinfo] yr_inds = tuple( filter(lambda x: header_tinfo[x] in yr_range, range(len(header_tinfo)))) for row in fid: location, *flow_rate = row.split('\t') flow_rate = [d.strip() for d in flow_rate] unit, code, source, dest = location.split(',') if unit != 'PAS' or code != 'PAS_CRD' or \ any(map(lambda x: 'EU' in x or 'EA' in x, (source, dest))): continue air_traffic[(source, dest)] += mean_func(flow_rate[i] for i in yr_inds) print(source, dest, air_traffic[(source, dest)]) write_file(air_traffic, data_fol) return air_traffic
def _word_skip_grams(self, tokens, stop_words=None): # handle stop words if stop_words is not None: tokens = [w for w in tokens if w not in stop_words] return compose(cmap(' '.join), pluck([0, 2]), sliding_window(3))(tokens)
def paths_to_major_minors(self, device_paths): """ Create a list of device major minors for a list of device paths from _path_to_major_minor dict. If any of the paths come back as None, continue to the next. :param device_paths: The list of paths to get the list of major minors for. :return: list of dev_major_minors, or an empty list if any device_path is not found. """ return pipe(device_paths, cmap(self.path_to_major_minor), cfilter(None), list)
def paths_to_major_minors(node_block_devices, ndt, device_paths): """ Create a list of device major minors for a list of device paths from _path_to_major_minor dict. If any of the paths come back as None, continue to the next. :param node_block_devices: dict of major-minor ids keyed on path :param ndt: normalised device table :param device_paths: The list of paths to get the list of major minors for. :return: list of dev_major_minors, or an empty list if any device_path is not found. """ c_path_to_major_minor = path_to_major_minor(node_block_devices, ndt) return pipe(device_paths, cmap(c_path_to_major_minor), cfilter(None), list)
def check(self, skip_keys, expect, result, x): from toolz import pipe from toolz.curried import map as cmap, filter as cfilter def cmpval(key): expected = expect[x][key] actual = result[x][key] if type(expected) is dict: self.check(skip_keys, expect[x], result[x], key) else: self.assertEqual( actual, expected, "item {} ({}) in {} does not match expected ({})".format( key, actual, x, expected), ) pipe(expect[x].keys(), cfilter(lambda y: y not in skip_keys), cmap(cmpval), list)
def from_figures(cls, figures): "return the Entry (tuple of Lines) that contains the given Figures" lines = cmap(curry(cls._line_by_index, figures)) return pipe(settings.lines_per_entry, range, lines, tuple)
def with_invalid_strokes(cls): "return tuple of Lines that each include an invalid stroke" return pipe(Strokes.invalid(), cmap(cls._by_invalid_stroke), tuple)
import os import copy import random from toolz import pipe, curry from toolz.curried import nth from toolz.curried import map as cmap from parse import settings import parse from common_tools import get_one_or_more, adulterate_iterable, bad_length_duplicator import fixture_constants _first_elements = cmap(curry(nth, 0)) _second_elements = cmap(curry(nth, 1)) _third_elements = cmap(curry(nth, 2)) _fourth_elements = cmap(curry(nth, 3)) _fifth_elements = cmap(curry(nth, 4)) class Numerals: "methods that provide Numerals for testing" @classmethod def get_random(cls, count=None): "return random valid Numeral[s]" getter = lambda: random.choice(cls.valid()) return get_one_or_more(getter, count) @staticmethod
def create_device_list(device_dict): return pipe(device_dict.itervalues(), cmap(as_device), cfilter(filter_device), list)
def valid(cls): "return ordered Figures representing Numerals 0-9" return cmap(cls.from_numeral, Numerals.valid())
def of_example_accounts(cls): "return lists of Superpositions made from example accounts" return pipe(fixture_constants.example_accounts, _first_elements, cmap(cls.from_account))
import os import copy import random from toolz import pipe, curry from toolz.curried import nth from toolz.curried import map as cmap from parse import settings import parse from common_tools import get_one_or_more, adulterate_iterable, bad_length_duplicator import fixture_constants _first_elements = cmap(curry(nth, 0)) _second_elements = cmap(curry(nth, 1)) _third_elements = cmap(curry(nth, 2)) _fourth_elements = cmap(curry(nth, 3)) _fifth_elements = cmap(curry(nth, 4)) class Numerals: "methods that provide Numerals for testing" @classmethod def get_random(cls, count=None): "return random valid Numeral[s]" getter = lambda: random.choice(cls.valid()) return get_one_or_more(getter, count)
def from_account(cls, account): "return the Figures that represent the given Account" return pipe(account, tuple, cmap(cls.from_numeral), tuple)
def of_basic_input_file(cls): "return Entries from basic input file" return pipe(Accounts.of_basic_input_file(), cmap(cls.from_account), tuple)
def _line_by_index(cls, figures, line_index): "return a Line composed of Figures Substrings" substrings = cmap(curry(cls._substring_by_index, line_index)) return pipe(figures, substrings, ''.join)
def fetch_device_list(): AgentShell.run(["udevadm", "settle"]) info = scanner_cmd("info") return pipe(info.itervalues(), cmap(as_device), cfilter(filter_device), list)
def custom_analyser(self, token, n, k): return compose(cmap(' '.join), skipgrams)(token, n, k)
def _invalid_or_illegible_account(superpositions): "return the invalid or illegible Account represented by Superpositions" return pipe(superpositions, cmap(_numeral), ''.join)
def _substrings_in_line(line): "return list of Substrings within a single Line" splitter = curry(partition, settings.strokes_per_substring) return pipe(line, splitter, cmap(''.join))