def main(data_fol, data_file):
"""Extract and write air traffic information between countries
Parameters:
-----------
data_fol : partial path.join function
Partial path function that links to the data directory
data_file : str
File name of the input file
"""
yr_range = tuple(map(str, range(2006, 2016)))
fname = data_fol(data_file)
air_traffic = defaultdict(float)
mean_func = compose(np.mean, list, cfilter(lambda x: x is not None),
cmap(catch_int_except))
with open(fname, 'r') as fid:
header_finfo, *header_tinfo = next(fid).split('\t')
header_tinfo = [d.strip() for d in header_tinfo]
yr_inds = tuple(
filter(lambda x: header_tinfo[x] in yr_range,
range(len(header_tinfo))))
for row in fid:
location, *flow_rate = row.split('\t')
flow_rate = [d.strip() for d in flow_rate]
unit, code, source, dest = location.split(',')
if unit != 'PAS' or code != 'PAS_CRD' or \
any(map(lambda x: 'EU' in x or 'EA' in x, (source, dest))):
continue
air_traffic[(source, dest)] += mean_func(flow_rate[i]
for i in yr_inds)
print(source, dest, air_traffic[(source, dest)])
write_file(air_traffic, data_fol)
return air_traffic
def _word_skip_grams(self, tokens, stop_words=None):
# handle stop words
if stop_words is not None:
tokens = [w for w in tokens if w not in stop_words]
return compose(cmap(' '.join), pluck([0, 2]),
sliding_window(3))(tokens)
def paths_to_major_minors(self, device_paths):
"""
Create a list of device major minors for a list of
device paths from _path_to_major_minor dict.
If any of the paths come back as None, continue to the next.
:param device_paths: The list of paths to get
the list of major minors for.
:return: list of dev_major_minors, or an empty
list if any device_path is not found.
"""
return pipe(device_paths, cmap(self.path_to_major_minor),
cfilter(None), list)
def paths_to_major_minors(node_block_devices, ndt, device_paths):
"""
Create a list of device major minors for a list of
device paths from _path_to_major_minor dict.
If any of the paths come back as None, continue to the next.
:param node_block_devices: dict of major-minor ids keyed on path
:param ndt: normalised device table
:param device_paths: The list of paths to get
the list of major minors for.
:return: list of dev_major_minors, or an empty
list if any device_path is not found.
"""
c_path_to_major_minor = path_to_major_minor(node_block_devices, ndt)
return pipe(device_paths, cmap(c_path_to_major_minor), cfilter(None), list)
def check(self, skip_keys, expect, result, x):
from toolz import pipe
from toolz.curried import map as cmap, filter as cfilter
def cmpval(key):
expected = expect[x][key]
actual = result[x][key]
if type(expected) is dict:
self.check(skip_keys, expect[x], result[x], key)
else:
self.assertEqual(
actual,
expected,
"item {} ({}) in {} does not match expected ({})".format(
key, actual, x, expected),
)
pipe(expect[x].keys(), cfilter(lambda y: y not in skip_keys),
cmap(cmpval), list)
def from_figures(cls, figures):
"return the Entry (tuple of Lines) that contains the given Figures"
lines = cmap(curry(cls._line_by_index, figures))
return pipe(settings.lines_per_entry, range, lines, tuple)
def with_invalid_strokes(cls):
"return tuple of Lines that each include an invalid stroke"
return pipe(Strokes.invalid(), cmap(cls._by_invalid_stroke), tuple)
import os
import copy
import random
from toolz import pipe, curry
from toolz.curried import nth
from toolz.curried import map as cmap
from parse import settings
import parse
from common_tools import get_one_or_more, adulterate_iterable, bad_length_duplicator
import fixture_constants
_first_elements = cmap(curry(nth, 0))
_second_elements = cmap(curry(nth, 1))
_third_elements = cmap(curry(nth, 2))
_fourth_elements = cmap(curry(nth, 3))
_fifth_elements = cmap(curry(nth, 4))
class Numerals:
"methods that provide Numerals for testing"
@classmethod
def get_random(cls, count=None):
"return random valid Numeral[s]"
getter = lambda: random.choice(cls.valid())
return get_one_or_more(getter, count)
@staticmethod
def from_figures(cls, figures):
"return the Entry (tuple of Lines) that contains the given Figures"
lines = cmap(curry(cls._line_by_index, figures))
return pipe(settings.lines_per_entry, range, lines, tuple)
def create_device_list(device_dict):
return pipe(device_dict.itervalues(), cmap(as_device),
cfilter(filter_device), list)
def valid(cls):
"return ordered Figures representing Numerals 0-9"
return cmap(cls.from_numeral, Numerals.valid())
def of_example_accounts(cls):
"return lists of Superpositions made from example accounts"
return pipe(fixture_constants.example_accounts, _first_elements, cmap(cls.from_account))
import os
import copy
import random
from toolz import pipe, curry
from toolz.curried import nth
from toolz.curried import map as cmap
from parse import settings
import parse
from common_tools import get_one_or_more, adulterate_iterable, bad_length_duplicator
import fixture_constants
_first_elements = cmap(curry(nth, 0))
_second_elements = cmap(curry(nth, 1))
_third_elements = cmap(curry(nth, 2))
_fourth_elements = cmap(curry(nth, 3))
_fifth_elements = cmap(curry(nth, 4))
class Numerals:
"methods that provide Numerals for testing"
@classmethod
def get_random(cls, count=None):
"return random valid Numeral[s]"
getter = lambda: random.choice(cls.valid())
return get_one_or_more(getter, count)
def from_account(cls, account):
"return the Figures that represent the given Account"
return pipe(account, tuple, cmap(cls.from_numeral), tuple)
def valid(cls):
"return ordered Figures representing Numerals 0-9"
return cmap(cls.from_numeral, Numerals.valid())
def of_example_accounts(cls):
"return lists of Superpositions made from example accounts"
return pipe(fixture_constants.example_accounts, _first_elements,
cmap(cls.from_account))
def of_basic_input_file(cls):
"return Entries from basic input file"
return pipe(Accounts.of_basic_input_file(), cmap(cls.from_account),
tuple)
def _line_by_index(cls, figures, line_index):
"return a Line composed of Figures Substrings"
substrings = cmap(curry(cls._substring_by_index, line_index))
return pipe(figures, substrings, ''.join)
def _line_by_index(cls, figures, line_index):
"return a Line composed of Figures Substrings"
substrings = cmap(curry(cls._substring_by_index, line_index))
return pipe(figures, substrings, ''.join)
def fetch_device_list():
AgentShell.run(["udevadm", "settle"])
info = scanner_cmd("info")
return pipe(info.itervalues(), cmap(as_device), cfilter(filter_device),
list)
def of_basic_input_file(cls):
"return Entries from basic input file"
return pipe(Accounts.of_basic_input_file(), cmap(cls.from_account), tuple)
def custom_analyser(self, token, n, k):
return compose(cmap(' '.join), skipgrams)(token, n, k)
def from_account(cls, account):
"return the Figures that represent the given Account"
return pipe(account, tuple, cmap(cls.from_numeral), tuple)
def _invalid_or_illegible_account(superpositions):
"return the invalid or illegible Account represented by Superpositions"
return pipe(superpositions, cmap(_numeral), ''.join)
def _invalid_or_illegible_account(superpositions):
"return the invalid or illegible Account represented by Superpositions"
return pipe(superpositions, cmap(_numeral), ''.join)
def _substrings_in_line(line):
"return list of Substrings within a single Line"
splitter = curry(partition, settings.strokes_per_substring)
return pipe(line, splitter, cmap(''.join))
def _substrings_in_line(line):
"return list of Substrings within a single Line"
splitter = curry(partition, settings.strokes_per_substring)
return pipe(line, splitter, cmap(''.join))
def with_invalid_strokes(cls):
"return tuple of Lines that each include an invalid stroke"
return pipe(Strokes.invalid(), cmap(cls._by_invalid_stroke), tuple)
