def load(self, deck_data, global_options=None):
if self.id is None:
# This happens when you load a deck not from a filepath so we
# use the first 16 characters of the SHA256 hexdigest as an ID
self.id = hashlib.sha256(json.dumps(deck_data)).hexdigest()[:16]
if global_options is not None:
self.global_options = normalize_options(global_options)
if isinstance(deck_data, list):
deck_data = convert_legacy_deck(deck_data)
self.name = deck_data.pop("name", "Un-named Deck")
self.description = deck_data.pop("description", "No description")
self.icon = deck_data.pop("icon", "fa-gears")
bouncer_address = self.global_options.get('bouncer',
deck_data.pop("bouncer", None))
if bouncer_address is None:
self.bouncer = get_preferred_bouncer()
elif isinstance(bouncer_address, dict):
self.bouncer = BouncerClient(settings=bouncer_address)
else:
self.bouncer = BouncerClient(bouncer_address)
self.schedule = deck_data.pop("schedule", None)
tasks_data = deck_data.pop("tasks", [])
for key, metadata in deck_data.items():
self.metadata[key] = metadata
# We override the task metadata with the global options if present
self.metadata.update(self.global_options)
for task_data in tasks_data:
deck_task = DeckTask(
data=task_data,
parent_metadata=self.metadata,
global_options=self.global_options,
cwd=self.deck_directory,
arbitrary_paths=self._arbitrary_paths
)
if deck_task.requires_tor:
self.requires_tor = True
if (deck_task.requires_bouncer and
self.bouncer.backend_type == "onion"):
self.requires_tor = True
self._tasks.append(deck_task)
if self.metadata.get('no_collector', False):
self.no_collector = True
if (self.no_collector is False and
self.bouncer.backend_type == "onion"):
self.requires_tor = True
def load(self, deck_data, global_options=None):
if self.id is None:
# This happens when you load a deck not from a filepath so we
# use the first 16 characters of the SHA256 hexdigest as an ID
self.id = hashlib.sha256(json.dumps(deck_data)).hexdigest()[:16]
if global_options is not None:
self.global_options = normalize_options(global_options)
if isinstance(deck_data, list):
deck_data = convert_legacy_deck(deck_data)
self.name = deck_data.pop("name", "Un-named Deck")
self.description = deck_data.pop("description", "No description")
self.icon = deck_data.pop("icon", "fa-gears")
bouncer_address = self.global_options.get(
'bouncer', deck_data.pop("bouncer", None))
if bouncer_address is None:
self.bouncer = get_preferred_bouncer()
elif isinstance(bouncer_address, dict):
self.bouncer = BouncerClient(settings=bouncer_address)
else:
self.bouncer = BouncerClient(bouncer_address)
self.schedule = deck_data.pop("schedule", None)
tasks_data = deck_data.pop("tasks", [])
for key, metadata in deck_data.items():
self.metadata[key] = metadata
# We override the task metadata with the global options if present
self.metadata.update(self.global_options)
for task_data in tasks_data:
deck_task = DeckTask(data=task_data,
parent_metadata=self.metadata,
global_options=self.global_options,
cwd=self.deck_directory,
arbitrary_paths=self._arbitrary_paths)
if deck_task.requires_tor:
self.requires_tor = True
if (deck_task.requires_bouncer
and self.bouncer.backend_type == "onion"):
self.requires_tor = True
self._tasks.append(deck_task)
if self.metadata.get('no_collector', False):
self.no_collector = True
if (self.no_collector is False
and self.bouncer.backend_type == "onion"):
self.requires_tor = True
def test_convert_legacy_deck(self):
legacy_deck = yaml.safe_load(StringIO(LEGACY_DECK))
ng_deck = convert_legacy_deck(legacy_deck)
self.assertEqual(len(ng_deck['tasks']), 3)
task_names = map(lambda task: task['ooni']['test_name'],
ng_deck['tasks'])
self.assertItemsEqual(task_names, [
"manipulation/http_invalid_request_line",
"manipulation/http_header_field_manipulation",
"blocking/web_connectivity"
])
tasks = map(lambda task: task['ooni'], ng_deck['tasks'])
self.assertEqual(tasks[2]['f'], '/path/to/citizenlab-urls-global.txt')
def load(self, deck_data, global_options=None):
if global_options is not None:
self.global_options = normalize_options(global_options)
if isinstance(deck_data, list):
deck_data = convert_legacy_deck(deck_data)
self.name = deck_data.pop("name", "Un-named Deck")
self.description = deck_data.pop("description", "No description")
bouncer_address = self.global_options.get('bouncer',
deck_data.pop("bouncer", None))
if bouncer_address is None:
self.bouncer = get_preferred_bouncer()
elif isinstance(bouncer_address, dict):
self.bouncer = BouncerClient(settings=bouncer_address)
else:
self.bouncer = BouncerClient(bouncer_address)
self.schedule = deck_data.pop("schedule", None)
tasks_data = deck_data.pop("tasks", [])
for key, metadata in deck_data.items():
self.metadata[key] = metadata
# We override the task metadata with the global options if present
self.metadata.update(self.global_options)
for task_data in tasks_data:
deck_task = DeckTask(
data=task_data,
parent_metadata=self.metadata,
global_options=self.global_options,
cwd=self.deck_directory,
arbitrary_paths=self._arbitrary_paths
)
if deck_task.requires_tor:
self.requires_tor = True
if (deck_task.requires_bouncer and
self.bouncer.backend_type == "onion"):
self.requires_tor = True
self._tasks.append(deck_task)
if self.metadata.get('no_collector', False):
self.no_collector = True
if (self.no_collector is False and
self.bouncer.backend_type == "onion"):
self.requires_tor = True
def test_convert_legacy_deck(self):
legacy_deck = yaml.safe_load(StringIO(LEGACY_DECK))
ng_deck = convert_legacy_deck(legacy_deck)
self.assertEqual(len(ng_deck['tasks']), 3)
task_names = map(lambda task: task['ooni']['test_name'],
ng_deck['tasks'])
self.assertItemsEqual(task_names, [
"manipulation/http_invalid_request_line",
"manipulation/http_header_field_manipulation",
"blocking/web_connectivity"
])
tasks = map(lambda task: task['ooni'], ng_deck['tasks'])
self.assertEqual(
tasks[2]['f'],
'/path/to/citizenlab-urls-global.txt')
