def test_valid_json(self):
read_data = json.dumps({'a': 1, 'b': 2, 'c': 3})
mock_open_method = mock_open(read_data=read_data)
test_filename = 'test.json'
with patch('builtins.open', mock_open_method):
result = load_config_file(test_filename)
self.assertEqual({'a': 1, 'b': 2, 'c': 3}, result)
def __init__(self):
storage_config = load_config_file("storage_config.json")
self.channel_blob_path = storage_config['channel_blob_path']
self.channels_file_blob_path = storage_config[
'channels_file_blob_path']
self.bucket = storage_config['bucket']
self.archive_blob_path = storage_config['archive_blob_path']
self.scraped_data_blob_path = storage_config['scraped_data_blob_path']
self.archive_file_name = "archive.txt"
self.token_file_name = 'token.txt'
def __init__(self, youtube):
self.youtube = youtube
self.storage_util = StorageUtil()
self.TYPE = "channel"
self.MAX_PAGE_RESULT = 50
config = load_config_file("youtube_api_config.json")
num_pages, num_results = self.__calculate_pages(config["max_results"])
self.max_results = num_results
self.pages = num_pages
self.rel_language = config["language_code"]
self.__set_keywords(config)
self.pages_exhausted = False
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
StorageUtil().set_gcs_creds(str(kwargs["my_setting"]).replace("'", ""))
self.total_duration_in_seconds = 0
config = load_config_file('web_crawl_config.json')
self.language = config["language"]
self.language_code = config["language_code"]
self.max_seconds = config["max_hours"] * 3600
self.word_to_ignore = config["word_to_ignore"]
self.extensions_to_include = config["extensions_to_include"]
self.extensions_to_ignore = config["extensions_to_ignore"]
self.enable_hours_restriction = config[
"enable_hours_restriction"].lower() == "yes"
self.depth = config["depth"]
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
StorageUtil().set_gcs_creds(
str(kwargs["my_setting"]).replace("\'", ""))
self.total_duration_in_seconds = 0
self.web_crawl_config = os.path.dirname(
os.path.realpath(__file__)) + "/../configs/web_crawl_config.json"
config = load_config_file('web_crawl_config.json')
self.config = config
self.language = config["language"]
self.language_code = config["language_code"]
self.max_seconds = config["max_hours"] * 3600
self.depth = config["depth"]
self.pages = config["pages"]
self.max_hours = config["max_hours"]
self.extensions_to_include = config["extensions_to_include"]
self.word_to_ignore = config["word_to_ignore"]
self.extensions_to_ignore = config["extensions_to_ignore"]
self.is_continued = config["continue_page"].lower() == "yes"
self.enable_hours_restriction = config[
"enable_hours_restriction"].lower() == "yes"
def __init__(self):
self.config_json = load_config_file('config.json')['downloader']
def test_file_not_exists(self):
with self.assertRaises(IOError) as context:
load_config_file('null')
self.assertEqual('file null does not exist.', str(context.exception))
def test_invalid_json(self):
read_data = ''
mock_open_method = mock_open(read_data=read_data)
with patch("builtins.open", mock_open_method):
with self.assertRaises(ValueError):
load_config_file('filename')