def download_and_extract(self, url_or_urls):
output = super().extract(super().download(url_or_urls))
dummy_output = self.mock_download_manager.download(url_or_urls)
map_nested(self.downloaded_dummy_paths.append, output, map_tuple=True)
map_nested(self.expected_dummy_paths.append,
dummy_output,
map_tuple=True)
return output
def test_map_nested(self):
s1 = {}
s2 = []
s3 = 1
s4 = [1, 2]
s5 = {"a": 1, "b": 2}
s6 = {"a": [1, 2], "b": [3, 4]}
s7 = {"a": {"1": 1}, "b": 2}
s8 = {"a": 1, "b": 2, "c": 3, "d": 4}
expected_map_nested_s1 = {}
expected_map_nested_s2 = []
expected_map_nested_s3 = 2
expected_map_nested_s4 = [2, 3]
expected_map_nested_s5 = {"a": 2, "b": 3}
expected_map_nested_s6 = {"a": [2, 3], "b": [4, 5]}
expected_map_nested_s7 = {"a": {"1": 2}, "b": 3}
expected_map_nested_s8 = {"a": 2, "b": 3, "c": 4, "d": 5}
self.assertEqual(map_nested(add_one, s1), expected_map_nested_s1)
self.assertEqual(map_nested(add_one, s2), expected_map_nested_s2)
self.assertEqual(map_nested(add_one, s3), expected_map_nested_s3)
self.assertEqual(map_nested(add_one, s4), expected_map_nested_s4)
self.assertEqual(map_nested(add_one, s5), expected_map_nested_s5)
self.assertEqual(map_nested(add_one, s6), expected_map_nested_s6)
self.assertEqual(map_nested(add_one, s7), expected_map_nested_s7)
self.assertEqual(map_nested(add_one, s8), expected_map_nested_s8)
num_proc = 2
self.assertEqual(map_nested(add_one, s1, num_proc=num_proc), expected_map_nested_s1)
self.assertEqual(map_nested(add_one, s2, num_proc=num_proc), expected_map_nested_s2)
self.assertEqual(map_nested(add_one, s3, num_proc=num_proc), expected_map_nested_s3)
self.assertEqual(map_nested(add_one, s4, num_proc=num_proc), expected_map_nested_s4)
self.assertEqual(map_nested(add_one, s5, num_proc=num_proc), expected_map_nested_s5)
self.assertEqual(map_nested(add_one, s6, num_proc=num_proc), expected_map_nested_s6)
self.assertEqual(map_nested(add_one, s7, num_proc=num_proc), expected_map_nested_s7)
self.assertEqual(map_nested(add_one, s8, num_proc=num_proc), expected_map_nested_s8)
sn1 = {"a": np.eye(2), "b": np.zeros(3), "c": np.ones(2)}
expected_map_nested_sn1_sum = {"a": 2, "b": 0, "c": 2}
expected_map_nested_sn1_int = {
"a": np.eye(2).astype(int),
"b": np.zeros(3).astype(int),
"c": np.ones(2).astype(int),
}
self.assertEqual(map_nested(np_sum, sn1, map_numpy=False), expected_map_nested_sn1_sum)
self.assertEqual(
{k: v.tolist() for k, v in map_nested(int, sn1, map_numpy=True).items()},
{k: v.tolist() for k, v in expected_map_nested_sn1_int.items()},
)
self.assertEqual(map_nested(np_sum, sn1, map_numpy=False, num_proc=num_proc), expected_map_nested_sn1_sum)
self.assertEqual(
{k: v.tolist() for k, v in map_nested(int, sn1, map_numpy=True, num_proc=num_proc).items()},
{k: v.tolist() for k, v in expected_map_nested_sn1_int.items()},
)
with self.assertRaises(AttributeError): # can't pickle a local lambda
map_nested(lambda x: x + 1, sn1, num_proc=num_proc)
