def __init__(self, metadata, resolver, logger, config, args):
super().__init__(metadata, resolver, logger, config, args)
self.url = 'url' in metadata and metadata[
'url'] or CLOUDFLARE_IP_RANGES_DEFAULT_JSON_URL
self.cache_file = 'cache_file' in metadata and metadata[
'cache_file'] or None
self.file_retriever = FileRetriever(self.logger, self.url,
self.cache_file)
def __init__(self, metadata, resolver, logger, config, args):
super().__init__(metadata, resolver, logger, config, args)
self.url = 'url' in metadata and metadata[
'url'] or GITHUB_IP_RANGES_DEFAULT_JSON_URL
self.ip_types = 'ip_types' in metadata and metadata[
'ip_types'] or IP_TYPES_DEFAULT
self.cache_file = 'cache_file' in metadata and metadata[
'cache_file'] or None
self.file_retriever = FileRetriever(self.logger, self.url,
self.cache_file)
def __init__(self, metadata, resolver, logger, config, args):
super().__init__(metadata, resolver, logger, config, args)
self.client_guid = 'client_guid' in metadata and metadata[
'client_guid'] or self.generate_client_guid()
self.url = 'url' in metadata and metadata[
'url'] or '%s?clientrequestid=%s' % (
MICROSOFT_IP_RANGES_DEFAULT_JSON_URL, self.client_guid)
self.hostname_filters = 'hostname_filters' in metadata and set(
metadata['hostname_filters']) or None
self.cache_file = 'cache_file' in metadata and metadata[
'cache_file'] or None
self.file_retriever = FileRetriever(self.logger, self.url,
self.cache_file)
class GetElements(Plugin):
def __init__(self, metadata, resolver, logger, config, args):
super().__init__(metadata, resolver, logger, config, args)
self.cloud_default_domains_only = 'cloud_default_domains_only' in metadata and metadata[
'cloud_default_domains_only'] or False
self.goog_url = 'goog_url' in metadata and metadata[
'goog_url'] or GOOG_DEFAULT_JSON_URL
self.cloud_url = 'cloud_url' in metadata and metadata[
'cloud_url'] or CLOUD_DEFAULT_JSON_URL
self.goog_cache_file = 'goog_cache_file' in metadata and metadata[
'goog_cache_file'] or None
self.cloud_cache_file = 'cloud_cache_file' in metadata and metadata[
'cloud_cache_file'] or None
self.goog_file_retriever = FileRetriever(self.logger, self.goog_url,
self.goog_cache_file)
self.cloud_file_retriever = FileRetriever(self.logger, self.cloud_url,
self.cloud_cache_file)
def get_elements(self):
google_ips = self.get_google_ips()
return google_ips
def get_google_ips(self):
goog_json = self.goog_file_retriever.get_json()
cloud_json = self.cloud_file_retriever.get_json()
if goog_json and cloud_json:
self.logger.debug('%s published: %s' %
(self.goog_url, goog_json.get('creationTime')))
self.logger.debug('%s published: %s' %
(self.cloud_url, cloud_json.get('creationTime')))
goog_cidrs = set()
for e in goog_json['prefixes']:
cidr = e.get('ipv4Prefix')
if cidr:
goog_cidrs.add(cidr)
cloud_cidrs = set()
for e in cloud_json['prefixes']:
cidr = e.get('ipv4Prefix')
if cidr:
cloud_cidrs.add(cidr)
if self.cloud_default_domains_only:
ip_ranges = list(goog_cidrs.difference(cloud_cidrs))
else:
ip_ranges = list(goog_cidrs)
self.logger.debug(
"IP ranges for Google APIs and services default domains: %s" %
json.dumps(ip_ranges))
return ip_ranges
else:
self.logger.error("JSON data not properly loaded")
return False
def __init__(self, metadata, resolver, logger, config, args):
super().__init__(metadata, resolver, logger, config, args)
# self.metadata: Any metadata from the configuration for the set.
# self.resolver: Configured instance of berserker_resolver.
# If args.berserk is set, then berserker_ips will
# be used as well to resolve hostnames.
# self.logger: The logger instantiated for this plugin.
# self.config: Loaded configuration file.
# self.args: Any parsed command line args.
# An example of how to instantiate FileRetriever class.
self.url = 'url' in metadata and metadata['url'] or DEFAULT_JSON_URL
self.cache_file = 'cache_file' in metadata and metadata['cache_file'] or None
self.file_retriever = FileRetriever(self.logger, self.url, self.cache_file)
def setUp(self):
"""Call before every test case."""
self.ret = FileRetriever()
self._out_dir = os.getcwd() + '/tests-profiles'
self.arg_dict = {
'profile': 'jim',
'provider': None,
'output': self._out_dir,
'tspan': 'latest'
}
try:
os.lstat(self._out_dir)
except FileNotFoundError:
os.mkdir(self._out_dir)
print(f'Test files download dir : {self._out_dir}')
def __init__(self, metadata, resolver, logger, config, args):
super().__init__(metadata, resolver, logger, config, args)
self.cloud_default_domains_only = 'cloud_default_domains_only' in metadata and metadata[
'cloud_default_domains_only'] or False
self.goog_url = 'goog_url' in metadata and metadata[
'goog_url'] or GOOG_DEFAULT_JSON_URL
self.cloud_url = 'cloud_url' in metadata and metadata[
'cloud_url'] or CLOUD_DEFAULT_JSON_URL
self.goog_cache_file = 'goog_cache_file' in metadata and metadata[
'goog_cache_file'] or None
self.cloud_cache_file = 'cloud_cache_file' in metadata and metadata[
'cloud_cache_file'] or None
self.goog_file_retriever = FileRetriever(self.logger, self.goog_url,
self.goog_cache_file)
self.cloud_file_retriever = FileRetriever(self.logger, self.cloud_url,
self.cloud_cache_file)
class GetElements(Plugin):
def __init__(self, metadata, resolver, logger, config, args):
super().__init__(metadata, resolver, logger, config, args)
self.url = 'url' in metadata and metadata[
'url'] or GITHUB_IP_RANGES_DEFAULT_JSON_URL
self.ip_types = 'ip_types' in metadata and metadata[
'ip_types'] or IP_TYPES_DEFAULT
self.cache_file = 'cache_file' in metadata and metadata[
'cache_file'] or None
self.file_retriever = FileRetriever(self.logger, self.url,
self.cache_file)
def get_elements(self):
github_ips = self.get_github_ips_for_types()
return github_ips
def get_github_ips_for_types(self):
data = self.file_retriever.get_json()
if data:
return self.build_elements(data)
return False
def build_elements(self, data):
elements = []
for ip_type in data:
if ip_type in self.ip_types:
self.logger.debug("Adding elements for IP type: %s" % ip_type)
for element in data[ip_type]:
elements.append(element)
else:
self.logger.debug("Skipping IP type: %s" % ip_type)
return elements
class GetElements(Plugin):
def __init__(self, metadata, resolver, logger, config, args):
super().__init__(metadata, resolver, logger, config, args)
self.url = 'url' in metadata and metadata[
'url'] or S3_IP_RANGES_DEFAULT_JSON_URL
self.regions = 'regions' in metadata and metadata[
'regions'] or REGIONS_DEFAULT
self.cache_file = 'cache_file' in metadata and metadata[
'cache_file'] or None
self.file_retriever = FileRetriever(self.logger, self.url,
self.cache_file)
def get_elements(self):
s3_ips = self.get_s3_ips_for_regions()
return s3_ips
def get_s3_ips_for_regions(self):
data = self.file_retriever.get_json()
if data:
return self.build_elements(data)
return False
def build_elements(self, data):
self.logger.debug("Adding elements from regions: %s" %
json.dumps(self.regions))
return [
obj['ip_prefix'] for obj in data['prefixes']
if not self.regions or obj['region'] in self.regions
]
class GetElements(Plugin):
def __init__(self, metadata, resolver, logger, config, args):
super().__init__(metadata, resolver, logger, config, args)
# self.metadata: Any metadata from the configuration for the set.
# self.resolver: Configured instance of berserker_resolver.
# If args.berserk is set, then berserker_ips will
# be used as well to resolve hostnames.
# self.logger: The logger instantiated for this plugin.
# self.config: Loaded configuration file.
# self.args: Any parsed command line args.
# An example of how to instantiate FileRetriever class.
self.url = 'url' in metadata and metadata['url'] or DEFAULT_JSON_URL
self.cache_file = 'cache_file' in metadata and metadata['cache_file'] or None
self.file_retriever = FileRetriever(self.logger, self.url, self.cache_file)
# The one required method, must return the list of elements generated by
# the call to the plugin.
def get_elements(self):
# Also has a 'get_json()' method.
data = self.file_retriever.get()
if data:
elements = data.split()
return elements
return False
class GetElements(Plugin):
def __init__(self, metadata, resolver, logger, config, args):
super().__init__(metadata, resolver, logger, config, args)
self.url = 'url' in metadata and metadata[
'url'] or CLOUDFRONT_IP_RANGES_DEFAULT_JSON_URL
self.ip_lists = 'ip_lists' in metadata and metadata[
'ip_lists'] or IP_LISTS_DEFAULT
self.cache_file = 'cache_file' in metadata and metadata[
'cache_file'] or None
self.file_retriever = FileRetriever(self.logger, self.url,
self.cache_file)
def get_elements(self):
cloudfront_ips = self.get_cloudfront_ips()
return cloudfront_ips
def get_cloudfront_ips(self):
data = self.file_retriever.get_json()
if data:
return self.build_elements(data)
return False
def build_elements(self, data):
elements = []
for ip_list in self.ip_lists:
if ip_list in data:
self.logger.debug("Adding elements from list %s: %s" %
(ip_list, json.dumps(data[ip_list])))
elements.extend(data[ip_list])
else:
self.logger.warn("List %s does not exist, skipping" % ip_list)
return elements
class GetElements(Plugin):
def __init__(self, metadata, resolver, logger, config, args):
super().__init__(metadata, resolver, logger, config, args)
self.url = 'url' in metadata and metadata[
'url'] or CLOUDFLARE_IP_RANGES_DEFAULT_JSON_URL
self.cache_file = 'cache_file' in metadata and metadata[
'cache_file'] or None
self.file_retriever = FileRetriever(self.logger, self.url,
self.cache_file)
def get_elements(self):
cloudflare_ips = self.get_cloudflare_ips_for_types()
return cloudflare_ips
def get_cloudflare_ips_for_types(self):
data = self.file_retriever.get()
if data:
elements = data.split()
return elements
return False
class TestFileRetriever(unittest.TestCase):
def setUp(self):
"""Call before every test case."""
self.ret = FileRetriever()
self._out_dir = os.getcwd() + '/tests-profiles'
self.arg_dict = {
'profile': 'jim',
'provider': None,
'output': self._out_dir,
'tspan': 'latest'
}
try:
os.lstat(self._out_dir)
except FileNotFoundError:
os.mkdir(self._out_dir)
print(f'Test files download dir : {self._out_dir}')
def tearDown(self):
print('::::::::::: Deleting all downloaded files :::::::::::')
# shutil.rmtree(self._out_dir)
def test_profile_jim(self):
print('Test 1: Download latest file for profile "jim"')
self.ret.retrieve(**self.arg_dict)
def test_profile_mark(self):
print('Test 1: Download latest file for profile "mark"')
opts = self.arg_dict
opts['profile'] = 'mark'
self.ret.retrieve(**opts)
def test_profile_wf(self):
print('Test 1: Download latest file for profile "mark"')
opts = self.arg_dict
opts['profile'] = 'wf'
self.ret.retrieve(**opts)
class GetElements(Plugin):
def __init__(self, metadata, resolver, logger, config, args):
super().__init__(metadata, resolver, logger, config, args)
self.client_guid = 'client_guid' in metadata and metadata[
'client_guid'] or self.generate_client_guid()
self.url = 'url' in metadata and metadata[
'url'] or '%s?clientrequestid=%s' % (
MICROSOFT_IP_RANGES_DEFAULT_JSON_URL, self.client_guid)
self.hostname_filters = 'hostname_filters' in metadata and set(
metadata['hostname_filters']) or None
self.cache_file = 'cache_file' in metadata and metadata[
'cache_file'] or None
self.file_retriever = FileRetriever(self.logger, self.url,
self.cache_file)
def get_elements(self):
microsoft_ips = self.get_microsoft_ips_for_hostnames()
return microsoft_ips
def generate_client_guid(self):
return uuid.uuid4()
def get_microsoft_ips_for_hostnames(self):
data = self.file_retriever.get_json()
if data:
return self.build_elements(data)
return False
def extract_matching_hostnames(self, group):
if self.hostname_filters is None:
return True
# TODO: Anything with no urls key is skipped for now, maybe also
# provide support for serviceArea?
if not 'urls' in group:
return False
common_hostnames = self.hostname_filters.intersection(
set(group['urls']))
for h in common_hostnames:
self.hostname_filters.remove(h)
self.logger.debug("Common hostnames in %s: %s" %
(group['id'], common_hostnames))
return len(common_hostnames) > 0
def no_more_hostname_filters(self):
return self.hostname_filters is not None and len(
self.hostname_filters) == 0
def build_elements(self, data):
elements = []
for group in data:
has_matches = self.extract_matching_hostnames(group)
if has_matches:
self.logger.debug("Adding elements for id: %d" % group['id'])
for element in group['ips']:
# TODO: Add IPv6 option.
if re.match(IPV4_CIDR_REGEX, element):
elements.append(element)
if self.no_more_hostname_filters():
self.logger.debug(
"No more hostname filters, returning elements")
return elements
else:
self.logger.debug("Skipping elements for id: %d" % group['id'])
return elements