def update_dependencies(options):
max_age = 14 * 24 * 3600
TLDEXTRACT_CACHE_FILE.parent.mkdir(parents=True, exist_ok=True)
if file_is_outdated(TLDEXTRACT_CACHE_FILE, max_age):
parse_domain.update(fetch_now=True)
for extractor_class in EXTRACTOR_CLASSES:
if hasattr(extractor_class, 'update_dependencies'):
extractor_class.update_dependencies(options)
def update_dependencies(self):
max_age = 14 * 24 * 3600
cache_file = Path(parse_domain.cache_file)
cache_file.parent.mkdir(parents=True, exist_ok=True)
if file_is_outdated(cache_file, max_age):
parse_domain.update(fetch_now=True)
for extractor_class in EXTRACTOR_CLASSES:
if hasattr(extractor_class, 'update_dependencies'):
extractor_class.update_dependencies(self.options)
def update_dependencies(options):
if not file_is_outdated(GEOIP_DATABASE_PATH, GEOIP_MAX_AGE):
return
GEOIP_DATABASE_PATH.parent.mkdir(parents=True, exist_ok=True)
FILES = ['COPYRIGHT.txt', 'LICENSE.txt', 'GeoLite2-Country.mmdb']
with tempfile.NamedTemporaryFile() as f:
download_file(GEOIP_DOWNLOAD_URL, f)
archive = tarfile.open(f.name)
for member in archive.getmembers():
base_name = os.path.basename(member.name)
if base_name in FILES and member.isfile():
with (GEOIP_DATABASE_PATH.parent / base_name).open('wb') as f:
copy_to(archive.extractfile(member), f)
def update_dependencies(cls, options):
lookup_file = options['storage_path'] / 'hsts.json'
if not file_is_outdated(lookup_file, 3600 * 24 * 7):
return
buf = io.BytesIO()
download_url = options.get('hsts_preload_url', HSTS_PRELOAD_URL)
download_file(download_url, buf)
plain_json = ''.join(line for line in buf.getvalue().decode().splitlines()
if not line.lstrip().startswith('//'))
hsts_data = json.loads(plain_json)
lookup = {entry['name']: entry.get('include_subdomains')
for entry in hsts_data['entries']}
with lookup_file.open('w') as f:
json.dump(lookup, f)
def update_dependencies(self):
geoip_database_path = self.options['geoip_database_path']
geoip_max_age = self.options['geoip_max_age']
if not file_is_outdated(geoip_database_path, geoip_max_age):
return
geoip_database_path.parent.mkdir(parents=True, exist_ok=True)
FILES = ['COPYRIGHT.txt', 'LICENSE.txt', 'GeoLite2-Country.mmdb']
with tempfile.NamedTemporaryFile() as f:
download_file(GEOIP_DOWNLOAD_URL, f)
archive = tarfile.open(f.name)
for member in archive.getmembers():
base_name = Path(member.name).name
if base_name in FILES and member.isfile():
with (geoip_database_path.parent /
base_name).open('wb') as f:
copy_to(archive.extractfile(member), f)