def load(self, deck_data, global_options=None): if self.id is None: # This happens when you load a deck not from a filepath so we # use the first 16 characters of the SHA256 hexdigest as an ID self.id = hashlib.sha256(json.dumps(deck_data)).hexdigest()[:16] if global_options is not None: self.global_options = normalize_options(global_options) if isinstance(deck_data, list): deck_data = convert_legacy_deck(deck_data) self.name = deck_data.pop("name", "Un-named Deck") self.description = deck_data.pop("description", "No description") self.icon = deck_data.pop("icon", "fa-gears") bouncer_address = self.global_options.get('bouncer', deck_data.pop("bouncer", None)) if bouncer_address is None: self.bouncer = get_preferred_bouncer() elif isinstance(bouncer_address, dict): self.bouncer = BouncerClient(settings=bouncer_address) else: self.bouncer = BouncerClient(bouncer_address) self.schedule = deck_data.pop("schedule", None) tasks_data = deck_data.pop("tasks", []) for key, metadata in deck_data.items(): self.metadata[key] = metadata # We override the task metadata with the global options if present self.metadata.update(self.global_options) for task_data in tasks_data: deck_task = DeckTask( data=task_data, parent_metadata=self.metadata, global_options=self.global_options, cwd=self.deck_directory, arbitrary_paths=self._arbitrary_paths ) if deck_task.requires_tor: self.requires_tor = True if (deck_task.requires_bouncer and self.bouncer.backend_type == "onion"): self.requires_tor = True self._tasks.append(deck_task) if self.metadata.get('no_collector', False): self.no_collector = True if (self.no_collector is False and self.bouncer.backend_type == "onion"): self.requires_tor = True
def load(self, deck_data, global_options=None): if self.id is None: # This happens when you load a deck not from a filepath so we # use the first 16 characters of the SHA256 hexdigest as an ID self.id = hashlib.sha256(json.dumps(deck_data)).hexdigest()[:16] if global_options is not None: self.global_options = normalize_options(global_options) if isinstance(deck_data, list): deck_data = convert_legacy_deck(deck_data) self.name = deck_data.pop("name", "Un-named Deck") self.description = deck_data.pop("description", "No description") self.icon = deck_data.pop("icon", "fa-gears") bouncer_address = self.global_options.get( 'bouncer', deck_data.pop("bouncer", None)) if bouncer_address is None: self.bouncer = get_preferred_bouncer() elif isinstance(bouncer_address, dict): self.bouncer = BouncerClient(settings=bouncer_address) else: self.bouncer = BouncerClient(bouncer_address) self.schedule = deck_data.pop("schedule", None) tasks_data = deck_data.pop("tasks", []) for key, metadata in deck_data.items(): self.metadata[key] = metadata # We override the task metadata with the global options if present self.metadata.update(self.global_options) for task_data in tasks_data: deck_task = DeckTask(data=task_data, parent_metadata=self.metadata, global_options=self.global_options, cwd=self.deck_directory, arbitrary_paths=self._arbitrary_paths) if deck_task.requires_tor: self.requires_tor = True if (deck_task.requires_bouncer and self.bouncer.backend_type == "onion"): self.requires_tor = True self._tasks.append(deck_task) if self.metadata.get('no_collector', False): self.no_collector = True if (self.no_collector is False and self.bouncer.backend_type == "onion"): self.requires_tor = True
def test_convert_legacy_deck(self): legacy_deck = yaml.safe_load(StringIO(LEGACY_DECK)) ng_deck = convert_legacy_deck(legacy_deck) self.assertEqual(len(ng_deck['tasks']), 3) task_names = map(lambda task: task['ooni']['test_name'], ng_deck['tasks']) self.assertItemsEqual(task_names, [ "manipulation/http_invalid_request_line", "manipulation/http_header_field_manipulation", "blocking/web_connectivity" ]) tasks = map(lambda task: task['ooni'], ng_deck['tasks']) self.assertEqual(tasks[2]['f'], '/path/to/citizenlab-urls-global.txt')
def load(self, deck_data, global_options=None): if global_options is not None: self.global_options = normalize_options(global_options) if isinstance(deck_data, list): deck_data = convert_legacy_deck(deck_data) self.name = deck_data.pop("name", "Un-named Deck") self.description = deck_data.pop("description", "No description") bouncer_address = self.global_options.get('bouncer', deck_data.pop("bouncer", None)) if bouncer_address is None: self.bouncer = get_preferred_bouncer() elif isinstance(bouncer_address, dict): self.bouncer = BouncerClient(settings=bouncer_address) else: self.bouncer = BouncerClient(bouncer_address) self.schedule = deck_data.pop("schedule", None) tasks_data = deck_data.pop("tasks", []) for key, metadata in deck_data.items(): self.metadata[key] = metadata # We override the task metadata with the global options if present self.metadata.update(self.global_options) for task_data in tasks_data: deck_task = DeckTask( data=task_data, parent_metadata=self.metadata, global_options=self.global_options, cwd=self.deck_directory, arbitrary_paths=self._arbitrary_paths ) if deck_task.requires_tor: self.requires_tor = True if (deck_task.requires_bouncer and self.bouncer.backend_type == "onion"): self.requires_tor = True self._tasks.append(deck_task) if self.metadata.get('no_collector', False): self.no_collector = True if (self.no_collector is False and self.bouncer.backend_type == "onion"): self.requires_tor = True
def test_convert_legacy_deck(self): legacy_deck = yaml.safe_load(StringIO(LEGACY_DECK)) ng_deck = convert_legacy_deck(legacy_deck) self.assertEqual(len(ng_deck['tasks']), 3) task_names = map(lambda task: task['ooni']['test_name'], ng_deck['tasks']) self.assertItemsEqual(task_names, [ "manipulation/http_invalid_request_line", "manipulation/http_header_field_manipulation", "blocking/web_connectivity" ]) tasks = map(lambda task: task['ooni'], ng_deck['tasks']) self.assertEqual( tasks[2]['f'], '/path/to/citizenlab-urls-global.txt')