def __init__(self):
self.global_db = redis.Redis(port=c.redis_port1,
db=c.redis_db_global,
host=c.redis_hostname1)
self.history_db = redis.Redis(port=c.redis_port2,
db=c.redis_db_history,
host=c.redis_hostname2)
self.config_db = redis.Redis(port=c.redis_port2,
db=c.redis_db_config,
host=c.redis_hostname2)
self.history_db_cache = redis.Redis(port=c.redis_cached_port,
db=c.redis_cached_db_history,
host=c.redis_hostname1)
if c.ptr_host and c.ptr_port:
# PTR record lookup configured
self.ptr_redis = redis.Redis(host=c.ptr_host, port=c.ptr_port)
self.has_ptr = True
else:
self.has_ptr = False
if HAS_IPASN and c.ipasn_host and c.ipasn_port:
# IP ASN history lookup configured
self.ipasn = IPASN(host=c.ipasn_host, port=c.ipasn_port)
self.has_ipasn = True
else:
self.has_ipasn = False
if HAS_ASN_HIST and c.asn_host and c.asn_port:
# ASN Description history configured
self.asnhistory = ASNHistory(host=c.asn_host, port=c.asn_port)
self.has_asnhistory = True
else:
self.has_asnhistory = False
def handler(q=False):
if q is False:
return False
request = json.loads(q)
if request.get('AS'):
toquery = request['AS']
else:
misperrors['error'] = "Unsupported attributes type"
return misperrors
if not request.get('config') and not (request['config'].get('host') and
request['config'].get('port') and
request['config'].get('db')):
misperrors['error'] = 'ASN description history configuration is missing'
return misperrors
asnhistory = ASNHistory(host=request['config'].get('host'),
port=request['config'].get('port'), db=request['config'].get('db'))
values = ['{} {}'.format(date.isoformat(), description) for date, description in asnhistory.get_all_descriptions(toquery)]
if not values:
misperrors['error'] = 'Unable to find descriptions for this ASN'
return misperrors
return {'results': [{'types': mispattributes['output'], 'values': values}]}
def __init__(self):
self.global_db = redis.Redis(port=c.redis_port1, db=c.redis_db_global, host=c.redis_hostname1)
self.history_db = redis.Redis(port=c.redis_port2, db=c.redis_db_history, host=c.redis_hostname2)
self.config_db = redis.Redis(port=c.redis_port2, db=c.redis_db_config, host=c.redis_hostname2)
self.history_db_cache = redis.Redis(port=c.redis_cached_port, db=c.redis_cached_db_history, host=c.redis_hostname1)
if c.ptr_host and c.ptr_port:
# PTR record lookup configured
self.ptr_redis = redis.Redis(host=c.ptr_host, port=c.ptr_port)
self.has_ptr = True
else:
self.has_ptr = False
if HAS_IPASN and c.ipasn_host and c.ipasn_port:
# IP ASN history lookup configured
self.ipasn = IPASN(host=c.ipasn_host, port=c.ipasn_port)
self.has_ipasn = True
else:
self.has_ipasn = False
if HAS_ASN_HIST and c.asn_host and c.asn_port:
# ASN Description history configured
self.asnhistory = ASNHistory(host=c.asn_host, port=c.asn_port)
self.has_asnhistory = True
else:
self.has_asnhistory = False
class BGPRanking(object):
def __init__(self):
self.global_db = redis.Redis(port=c.redis_port1,
db=c.redis_db_global,
host=c.redis_hostname1)
self.history_db = redis.Redis(port=c.redis_port2,
db=c.redis_db_history,
host=c.redis_hostname2)
self.config_db = redis.Redis(port=c.redis_port2,
db=c.redis_db_config,
host=c.redis_hostname2)
self.history_db_cache = redis.Redis(port=c.redis_cached_port,
db=c.redis_cached_db_history,
host=c.redis_hostname1)
if c.ptr_host and c.ptr_port:
# PTR record lookup configured
self.ptr_redis = redis.Redis(host=c.ptr_host, port=c.ptr_port)
self.has_ptr = True
else:
self.has_ptr = False
if HAS_IPASN and c.ipasn_host and c.ipasn_port:
# IP ASN history lookup configured
self.ipasn = IPASN(host=c.ipasn_host, port=c.ipasn_port)
self.has_ipasn = True
else:
self.has_ipasn = False
if HAS_ASN_HIST and c.asn_host and c.asn_port:
# ASN Description history configured
self.asnhistory = ASNHistory(host=c.asn_host, port=c.asn_port)
self.has_asnhistory = True
else:
self.has_asnhistory = False
def get_ptr_record(self, ip):
# Get PTR Record when looking for an IP
if self.has_ptr:
return self.ptr_redis.get(ip)
def get_asn_descriptions(self, asn):
if not self.has_asnhistory:
publisher.debug('ASN History not enabled.')
return [datetime.date.today(), 'ASN History not enabled.']
desc_history = self.asnhistory.get_all_descriptions(asn)
return [(date.astimezone(tz.tzutc()).date(), descr)
for date, descr in desc_history]
def get_ip_info(self, ip, days_limit=None):
"""
Return informations related to an IP address.
:param ip: The IP address
:param days_limit: The number of days we want to check in the past
(default: around 2 years)
:rtype: Dictionary
.. note:: Format of the output:
.. code-block:: python
{
'ip': ip,
'days_limit' : days_limit,
'ptrrecord' : 'ptr.record.com',
'history':
[
{
'asn': asn,
'interval': [first, last],
'block': block,
'timestamp': timestamp,
'descriptions':
[
[date, descr],
...
]
},
...
]
}
"""
if days_limit is None:
days_limit = 750
to_return = {'ip': ip, 'days_limit': days_limit, 'history': []}
if self.has_ptr:
to_return['ptrrecord'] = self.get_ptr_record(ip)
if not self.has_ipasn:
publisher.debug('IPASN not enabled.')
to_return['error'] = 'IPASN not enabled.'
return to_return
if not ip:
to_return['error'] = 'No IP provided.'
return to_return
for first, last, asn, block in self.ipasn.aggregate_history(
ip, days_limit):
first_date = parser.parse(first).replace(tzinfo=tz.tzutc()).date()
last_date = parser.parse(last).replace(tzinfo=tz.tzutc()).date()
if self.has_asnhistory:
desc_history = self.asnhistory.get_all_descriptions(asn)
valid_descriptions = []
for date, descr in desc_history:
date = date.astimezone(tz.tzutc()).date()
test_date = date - datetime.timedelta(days=1)
if last_date < test_date:
# Too new
continue
elif last_date >= test_date and first_date <= test_date:
# Changes within the interval
valid_descriptions.append([date.isoformat(), descr])
elif first_date > test_date:
# get the most recent change befrore the interval
valid_descriptions.append([date.isoformat(), descr])
break
else:
publisher.debug('ASN History not enabled.')
valid_descriptions = [
datetime.date.today().isoformat(),
'ASN History not enabled.'
]
if len(valid_descriptions) == 0:
if len(desc_history) != 0:
# fallback, use the oldest description.
date = desc_history[-1][0].astimezone(tz.tzutc()).date()
descr = desc_history[-1][1]
valid_descriptions.append([date.isoformat(), descr])
else:
# No history found for this ASN
if last_date > datetime.date(2013, 1, 1):
# ASN has been seen recently, should not happen
# as the asn history module is running since early 2013
publisher.error(
'Unable to find the ASN description of {}. IP address: {}. ASN History might be down.'
.format(asn, ip))
valid_descriptions.append(
['0000-00-00', 'No ASN description has been found.'])
entry = {}
entry['asn'] = asn
entry['interval'] = [first_date.isoformat(), last_date.isoformat()]
entry['block'] = block
entry['timestamp'] = self.get_first_seen(asn, block)
entry['descriptions'] = valid_descriptions
to_return['history'].append(entry)
return to_return
def asn_exists(self, asn):
return self.global_db.exists(asn)
def get_default_date(self, delta_days=1):
return self.get_default_date_raw(delta_days).isoformat()
def get_default_date_raw(self, delta_days=1):
"""
Get the default date displayed on the website.
"""
delta = datetime.timedelta(days=delta_days)
timestamp = self.history_db.get('latest_ranking')
if timestamp:
default_date_raw = parser.parse(
timestamp.split()[0]).date() - delta
else:
default_date_raw = datetime.date.today() - delta
return default_date_raw
def get_last_seen_sources(self, asn, dates_sources):
"""
Return a dictionary conteining the last date a particular ASN
has been seen by a source.
"""
if not self.asn_exists(asn):
return {}
string = '{asn}|{date}|{source}|rankv{ip_version}'
s_dates = dates_sources.keys()
s_dates.sort(reverse=True)
p = self.history_db.pipeline()
for date in s_dates:
sources = dates_sources[date]
if len(sources) > 0:
[
p.exists(
string.format(asn=asn,
date=date,
source=source,
ip_version=c.ip_version))
for source in sources
]
asns_found = p.execute()
i = 0
to_return = {}
for date in s_dates:
sources = dates_sources[date]
if len(sources) > 0:
for source in sources:
if to_return.get(source) is None and asns_found[i]:
to_return[source] = date
i += 1
return to_return
def get_all_ranks_single_asn(self,
asn,
dates_sources,
with_details_sources=False):
"""
Get all the ranks on a timeframe for a single ASN.
:param asn: Autonomous System Number
:param dates_sources: Dictionaries of the dates and sources
.. note:: Format of the dictionary:
.. code-block:: python
{
YYYY-MM-DD: [source1, source2, ...],
YYYY-MM-DD: [source1, source2, ...],
...
}
:param with_details_sources: Returns the details by sources if True
:rype: Dictionary
.. note:: Format of the output:
.. code-block:: python
{
date1:
{
'details':
[
(source1, rank),
(source2, rank),
...
],
'total': sum_of_ranks,
'description': description
}
...
}
The details key is only present if
``with_details_sources`` is True.
"""
to_return = {}
if with_details_sources is None:
with_details_sources = False
if not self.asn_exists(asn):
return to_return
string = '{asn}|{date}|{source}|rankv{ip_version}'
p = self.history_db.pipeline()
for date, sources in dates_sources.iteritems():
if len(sources) > 0:
p.mget([
string.format(asn=asn,
date=date,
source=source,
ip_version=c.ip_version)
for source in sources
])
ranks = p.execute()
i = 0
for date, sources in dates_sources.iteritems():
impacts = self.get_all_weights(date)
if len(sources) > 0:
to_return[date] = {}
zipped_sources_rank = zip(sources, ranks[i])
if with_details_sources:
to_return[date]['details'] = list(zipped_sources_rank)
to_return[date]['description'] = self.asn_desc_via_history(asn)
to_return[date]['total'] = 0
for s, r in zipped_sources_rank:
if r is not None:
to_return[date]['total'] += float(r) * float(
impacts[s])
i += 1
return to_return
def existing_asns_timeframe(self, dates_sources):
"""
Get all the ASNs seen in the lists on a timeframe.
"""
asns_keys = []
for date, sources in dates_sources.iteritems():
asns_keys.extend([
'{date}|{source}|asns'.format(date=date, source=source)
for source in sources
])
p = self.global_db.pipeline(False)
[p.smembers(k) for k in asns_keys]
asns_list = p.execute()
return set.union(*asns_list)
def get_all_ranks_all_asns(self,
dates_sources,
with_details_sources=False):
"""
Get all ranks for all the ASNs on a timeframe.
:param dates_sources: Dictionaries of the dates and sources
.. note:: Format of the dictionary:
.. code-block:: python
{
YYYY-MM-DD: [source1, source2, ...],
YYYY-MM-DD: [source1, source2, ...],
...
}
:param with_details_sources: Also returns the ranks by sources
:rype: Dictionary
.. note:: Format of the dictionary:
.. code-block:: python
{
YYYY-MM-DD:
{
asn1 :
{
'details':
[
(source1, rank),
(source2, rank),
...
],
'total': sum_of_ranks,
'description': description
}
...
}
...
}
The details key is only present if
``with_details_sources`` is True.
"""
asns = self.existing_asns_timeframe(dates_sources)
to_return = {}
for asn in asns:
details = self.get_all_ranks_single_asn(asn, dates_sources,
with_details_sources)
for date, entries in details.iteritems():
if to_return.get(date) is None:
to_return[date] = {}
to_return[date][asn] = entries
return to_return
def get_all_blocks(self, asn):
"""
Expose the list of blocks announced by an AS.
:param asn: Autonomous System Number
:rtype: Set of ip blocks
"""
return self.global_db.smembers(asn)
def get_all_blocks_description(self, asn):
"""
Get the most recent description of all the blocks announced by an AS
:param asn: Autonomous System Number
:rtype: List
.. note:: Format of the list:
.. code-block:: python
[asn, [(block, description), ...]]
"""
blocks = self.get_all_blocks(asn)
if len(blocks) > 0:
block_keys = [
'{asn}|{block}'.format(asn=asn, block=block)
for block in blocks
]
p = self.global_db.pipeline(False)
[p.hgetall(block_key) for block_key in block_keys]
ts_descr = p.execute()
i = 0
block_descr = []
for block_key in block_keys:
if ts_descr[i] is not None:
asn, block = block_key.split('|')
most_recent_ts = sorted(ts_descr[i].keys())[0]
descr = ts_descr[i][most_recent_ts]
block_descr.append((block, descr))
i += 1
return asn, block_descr
else:
return asn, []
def get_first_seen(self, asn, block):
"""
Get the oldest timestamp where the block has been announced by the AS
"""
timestamps = self.global_db.hkeys(asn + '|' + block)
if timestamps is not None and len(timestamps) > 0:
timestamps.sort()
return timestamps[0]
return None
def get_block_descriptions(self, asn, block):
"""
Return all the descriptions of a block, in a listed sorted by
timestamp (new => old)
:param asn: Autonomous System Number
:param block: IP Block you are looking for
:rtype: List
.. note:: Format of the list:
.. code-block:: python
[asn, block, [(ts, descr), ...]]
"""
ts_descr = self.global_db.hgetall('{}|{}'.format(asn, block))
timestamps = sorted(ts_descr.keys(), reverse=True)
descriptions = [(t, ts_descr[t]) for t in timestamps]
return asn, block, descriptions
def get_all_blocks_descriptions(self, asn):
"""
Return all tuples timestamp-description of all the blocs announced
by an AS over time.
:param asn: Autonomous System Number
:rtype: List
.. note:: Format of the list:
.. code-block:: python
[ asn,
{
block:
[
(timestamp, description),
...
],
...
}
]
"""
blocks_descriptions = {}
for block in self.get_all_blocks(asn):
asn, block, descriptions = self.get_block_descriptions(asn, block)
blocks_descriptions[block] = descriptions
return asn, blocks_descriptions
def asn_desc_via_history(self, asn):
if self.has_asnhistory:
asn_descr = self.asnhistory.get_last_description(asn)
if asn_descr is None:
# The ASN has no descripion in the database
# publisher.error(\
# 'Unable to find the ASN description of {}. ASN History might be down.'.\
# format(asn))
asn_descr = 'No ASN description has been found.'
else:
publisher.debug('ASN History not enabled.')
asn_descr = 'ASN History not enabled.'
return asn_descr
def get_asn_descs(self, asn, date=None, sources=None):
"""
Get all what is available in the database about an ASN for
one day
:param asn: Autonomous System Number
:param date: Date of the information (default: last ranked day)
:param sources: List of sources (default: the available sources
at ``date``)
:rtype: Dictionary
.. note:: Format of the dictionary:
.. code-block:: python
{
'date': date,
'sources': [source1, source2, ...],
'asn': asn,
'asn_description': asn_description,
asn:
{
'clean_blocks':
{
block: [[ts, descr], ...],
....
},
'old_blocks':
{
block: [[ts, descr], ...],
....
},
block:
{
'description': description,
'timestamp': ts,
'old_descr': [(timestamp, description), ...]
'nb_of_ips': nb,
'sources': [source1, source2, ...]
'rank': subnet_rank
},
...
}
}
"""
if not self.asn_exists(asn):
# The ASN does not exists in the database
return None
if date is None:
date = self.get_default_date()
day_sources = self.daily_sources([date])
if sources is None:
sources = list(day_sources)
else:
if not isinstance(sources, list):
sources = [sources]
sources = list(day_sources.intersection(set(sources)))
asn_descr = self.asn_desc_via_history(asn)
to_return = {
'date': date,
'sources': sources,
'asn': asn,
'asn_description': asn_descr,
asn: {}
}
key_clean_set = '{asn}|{date}|clean_set'.format(asn=asn, date=date)
for ip_block in self.get_all_blocks(asn):
# Get descriptions
asn, block, ts_descr = self.get_block_descriptions(asn, ip_block)
# Find out if the block has been seen these day
p = self.global_db.pipeline(False)
[
p.scard('{}|{}|{}|{}'.format(asn, ip_block, date, source))
for source in sources
]
ips_by_sources = p.execute()
nb_of_ips = sum(ips_by_sources)
if nb_of_ips == 0:
if ip_block in self.history_db.smembers(key_clean_set):
if to_return[asn].get('clean_blocks') is None:
to_return[asn]['clean_blocks'] = {}
to_return[asn]['clean_blocks'][ip_block] = ts_descr
else:
if to_return[asn].get('old_blocks') is None:
to_return[asn]['old_blocks'] = {}
to_return[asn]['old_blocks'][ip_block] = ts_descr
else:
impacts = self.get_all_weights(date)
# Compute the local ranking: the ranking if this subnet is
# the only one for this AS
i = 0
sum_rank = 0
sources_exists = []
for source in sources:
sum_rank += float(ips_by_sources[i]) * float(
impacts[source])
if ips_by_sources[i] != 0:
sources_exists.append(source)
i += 1
local_rank = sum_rank / IPy.IP(ip_block).len()
to_return[asn][ip_block] = {
'description': ts_descr[0][1],
'timestamp': ts_descr[0][0],
'old_descr': ts_descr[1:],
'nb_of_ips': nb_of_ips,
'sources': sources_exists,
'rank': local_rank
}
return to_return
def get_all_weights(self, date=None):
"""
Get the weights for all the sources.
:param date: Date of the information (default: last ranked day)
:rtype: Dictionary
.. note:: Format of the dictionary:
.. code-block:: python
{
source: date,
...
}
"""
if date is None:
date = self.get_default_date()
sources = self.daily_sources([date])
to_return = {}
if len(sources) > 0:
impacts = self.config_db.mget(sources)
to_return = dict(zip(sources, impacts))
return to_return
def daily_sources(self, dates):
"""
Get the sources parsed during a list of dates
:param dates: List of dates
:rtype: Set of sources for each date
"""
if len(dates) == 1:
return self.global_db.smembers(
'{date}|sources'.format(date=dates[0]))
else:
p = self.global_db.pipeline(False)
[p.smembers('{date}|sources'.format(date=date)) for date in dates]
return p.execute()
def get_ips_descs(self, asn, block, date=None, sources=None):
"""
Get all what is available in the database about an subnet for one day
:param asn: Autonomous System Number
:param asn_timestamp: First the subnet has been seen in the db
(for this ASN)
:param date: Date of the information (default: last ranked day)
:param sources: List of sources (default: the available sources
at ``date``)
:rtype: Dictionary
.. note:: Format of the dictionary:
.. code-block:: python
{
'date': date,
'sources': [source1, source2, ...],
'asn': asn,
'asn_description': asn_description,
'block': block
block:
{
ip:
{
'sources': [source1, source2, ...],
'ptrrecord': 'ptr.record.com'
}
...
}
}
"""
if date is None:
date = self.get_default_date()
day_sources = self.daily_sources([date])
if sources is None:
sources = list(day_sources)
else:
if not isinstance(sources, list):
sources = [sources]
sources = list(day_sources.intersection(set(sources)))
asn_descr = self.asn_desc_via_history(asn)
to_return = {
'date': date,
'sources': sources,
'asn': asn,
'asn_description': asn_descr,
'block': block,
block: {}
}
asn_block_key = '{asn}|{b}|{date}|'.format(asn=asn, b=block, date=date)
pipeline = self.global_db.pipeline(False)
[
pipeline.smembers('{asn_b}{source}'.format(asn_b=asn_block_key,
source=source))
for source in sources
]
ips_by_source = pipeline.execute()
i = 0
for source in sources:
ips = ips_by_source[i]
for ip_details in ips:
ip, timestamp = ip_details.split('|')
if to_return[block].get(ip) is None:
to_return[block][ip] = {'sources': [], 'ptrrecord': None}
to_return[block][ip]['sources'].append(source)
i += 1
if self.has_ptr:
for ip in to_return[block]:
to_return[block][ip]['ptrrecord'] = self.get_ptr_record(ip)
return to_return
def get_stats(self, dates_sources):
"""
Return amount of asn and subnets found by source on a list of days.
:param dates_sources: Dictionaries of the dates and sources
:rtype: Dictionary
.. note:: Format of the Dictionary
.. code-block:: python
{
date:
{
'sources':
{
source : [nb_asns, nb_subnets],
...
},
'total_asn': total_asn
'total_subnets': total_subnets
}
...
}
"""
to_return = {}
p = self.global_db.pipeline(False)
for date, sources in dates_sources.iteritems():
to_return[date] = {}
for source in sources:
current_key = '{date}|{source}|asns'.format(date=date,
source=source)
p.scard(current_key)
p.scard(current_key + '_details')
cards = p.execute()
i = 0
for date, sources in dates_sources.iteritems():
to_return[date] = {'sources': {}}
total_asn = 0
total_subnets = 0
for source in sources:
nb_asns = cards[i]
nb_subnets = cards[i + 1]
total_asn += nb_asns
total_subnets += nb_subnets
to_return[date]['sources'].update(
{source: (nb_asns, nb_subnets)})
i = i + 2
to_return[date]['total_asn'] = total_asn
to_return[date]['total_subnets'] = total_subnets
return to_return
# Need cached database
def cache_get_dates(self):
"""
**From the temporary database**
Get a list of dates. The ranking are available in the cache database.
"""
return sorted(self.history_db_cache.smembers('all_dates'))
def cache_get_stats(self):
"""
Return amount of asn and subnets found by source from the cache.
"""
dates = self.cache_get_dates()
dates_sources = dict(zip(dates, self.daily_sources(dates)))
return self.get_stats(dates_sources)
def cache_get_daily_rank(self, asn, source='global', date=None):
"""
**From the temporary database**
Get a single rank.
:param asn: Autonomous System Number
:param source: Source to use. global is the aggregated view for
all the sources
:param date: Date of the information (default: last ranked day)
:rtype: List
.. note:: Format of the list
.. code-block:: python
[asn, asn_description, date, source, rank]
"""
if source is None:
source = 'global'
if date is None:
date = self.get_default_date()
histo_key = '{date}|{source}|rankv{ip_version}'.format(
date=date, source=source, ip_version=c.ip_version)
asn_descr = self.asn_desc_via_history(asn)
return asn, asn_descr, date, source, self.history_db_cache.zscore(
histo_key, asn)
def cache_get_position(self, asn, source='global', date=None):
"""
**From the temporary database**
Get the position of the ASN in the zrank.
:param asn: Autonomous System Number
:param source: Source to use. global is the aggregated view for
all the sources
:param date: Date of the information (default: last ranked day)
:rtype: Integer, position in the list and size of the list.
.. note:: if None, the zrank does not exists (source or date invalid)
"""
if source is None:
source = 'global'
if date is None:
date = self.get_default_date()
histo_key = '{date}|{source}|rankv{ip_version}'.format(
date=date, source=source, ip_version=c.ip_version)
return self.history_db_cache.zrevrank(
histo_key, asn), self.history_db_cache.zcard(histo_key)
def cache_get_top_asns(self,
source='global',
date=None,
limit=100,
with_sources=True):
"""
**From the temporary database**
Get worse ASNs.
:param source: Source used to rank the ASNs. global is the
aggregated view for all the sources
:param date: Date of the information (default: last ranked day)
:param limit: Number of ASNs to get
:param with_sources: Get the list of sources where each ASN has
been found.
:rtype: Dictionary
.. note:: Format of the Dictionary
.. code-block:: python
{
'source': source,
'date': date,
'size_list': size,
'top_list':
[
(
(asn, asn_description, rank),
set([source1, source2, ...])
),
...
]
}
The set of sources is only presetn if ``with_sources``
is True
"""
if source is None:
source = 'global'
if date is None:
date = self.get_default_date()
if limit is None:
limit = 100
if with_sources is None:
with_sources = True
histo_key = '{date}|{histo_key}|rankv{ip_version}'.format(
date=date, histo_key=source, ip_version=c.ip_version)
to_return = {
'source': source,
'date': date,
'size_list': self.history_db_cache.zcard(histo_key),
'top_list': []
}
ranks = self.history_db_cache.zrevrange(histo_key, 0, limit, True)
if ranks is None:
return to_return
temp_rank = []
for asn, rank in ranks:
if asn == '-1':
continue
asn_descr = self.asn_desc_via_history(asn)
temp_rank.append((asn, asn_descr, rank))
if not with_sources:
to_return['top_list'] = temp_rank
else:
p = self.history_db_cache.pipeline(False)
[p.smembers('{}|{}'.format(date, rank[0])) for rank in temp_rank]
to_return['top_list'] = list(zip(temp_rank, p.execute()))
return to_return
class BGPRanking(object):
def __init__(self):
self.global_db = redis.Redis(port=c.redis_port1, db=c.redis_db_global, host=c.redis_hostname1)
self.history_db = redis.Redis(port=c.redis_port2, db=c.redis_db_history, host=c.redis_hostname2)
self.config_db = redis.Redis(port=c.redis_port2, db=c.redis_db_config, host=c.redis_hostname2)
self.history_db_cache = redis.Redis(port=c.redis_cached_port, db=c.redis_cached_db_history, host=c.redis_hostname1)
if c.ptr_host and c.ptr_port:
# PTR record lookup configured
self.ptr_redis = redis.Redis(host=c.ptr_host, port=c.ptr_port)
self.has_ptr = True
else:
self.has_ptr = False
if HAS_IPASN and c.ipasn_host and c.ipasn_port:
# IP ASN history lookup configured
self.ipasn = IPASN(host=c.ipasn_host, port=c.ipasn_port)
self.has_ipasn = True
else:
self.has_ipasn = False
if HAS_ASN_HIST and c.asn_host and c.asn_port:
# ASN Description history configured
self.asnhistory = ASNHistory(host=c.asn_host, port=c.asn_port)
self.has_asnhistory = True
else:
self.has_asnhistory = False
def get_ptr_record(self, ip):
# Get PTR Record when looking for an IP
if self.has_ptr:
return self.ptr_redis.get(ip)
def get_asn_descriptions(self, asn):
if not self.has_asnhistory:
publisher.debug('ASN History not enabled.')
return [datetime.date.today(), 'ASN History not enabled.']
desc_history = self.asnhistory.get_all_descriptions(asn)
return [(date.astimezone(tz.tzutc()).date(), descr) for date, descr in desc_history]
def get_ip_info(self, ip, days_limit=None):
"""
Return informations related to an IP address.
:param ip: The IP address
:param days_limit: The number of days we want to check in the past
(default: around 2 years)
:rtype: Dictionary
.. note:: Format of the output:
.. code-block:: python
{
'ip': ip,
'days_limit' : days_limit,
'ptrrecord' : 'ptr.record.com',
'history':
[
{
'asn': asn,
'interval': [first, last],
'block': block,
'timestamp': timestamp,
'descriptions':
[
[date, descr],
...
]
},
...
]
}
"""
if days_limit is None:
days_limit = 750
to_return = {'ip': ip, 'days_limit': days_limit, 'history': []}
if self.has_ptr:
to_return['ptrrecord'] = self.get_ptr_record(ip)
if not self.has_ipasn:
publisher.debug('IPASN not enabled.')
to_return['error'] = 'IPASN not enabled.'
return to_return
if not ip:
to_return['error'] = 'No IP provided.'
return to_return
for first, last, asn, block in self.ipasn.aggregate_history(ip, days_limit):
first_date = parser.parse(first).replace(tzinfo=tz.tzutc()).date()
last_date = parser.parse(last).replace(tzinfo=tz.tzutc()).date()
if self.has_asnhistory:
desc_history = self.asnhistory.get_all_descriptions(asn)
valid_descriptions = []
for date, descr in desc_history:
date = date.astimezone(tz.tzutc()).date()
test_date = date - datetime.timedelta(days=1)
if last_date < test_date:
# Too new
continue
elif last_date >= test_date and first_date <= test_date:
# Changes within the interval
valid_descriptions.append([date.isoformat(), descr])
elif first_date > test_date:
# get the most recent change befrore the interval
valid_descriptions.append([date.isoformat(), descr])
break
else:
publisher.debug('ASN History not enabled.')
valid_descriptions = [datetime.date.today().isoformat(), 'ASN History not enabled.']
if len(valid_descriptions) == 0:
if len(desc_history) != 0:
# fallback, use the oldest description.
date = desc_history[-1][0].astimezone(tz.tzutc()).date()
descr = desc_history[-1][1]
valid_descriptions.append([date.isoformat(), descr])
else:
# No history found for this ASN
if last_date > datetime.date(2013, 1, 1):
# ASN has been seen recently, should not happen
# as the asn history module is running since early 2013
publisher.error('Unable to find the ASN description of {}. IP address: {}. ASN History might be down.'.format(asn, ip))
valid_descriptions.append(['0000-00-00', 'No ASN description has been found.'])
entry = {}
entry['asn'] = asn
entry['interval'] = [first_date.isoformat(), last_date.isoformat()]
entry['block'] = block
entry['timestamp'] = self.get_first_seen(asn, block)
entry['descriptions'] = valid_descriptions
to_return['history'].append(entry)
return to_return
def asn_exists(self, asn):
return self.global_db.exists(asn)
def get_default_date(self, delta_days=1):
return self.get_default_date_raw(delta_days).isoformat()
def get_default_date_raw(self, delta_days=1):
"""
Get the default date displayed on the website.
"""
delta = datetime.timedelta(days=delta_days)
timestamp = self.history_db.get('latest_ranking')
if timestamp:
default_date_raw = parser.parse(timestamp.split()[0]).date() - delta
else:
default_date_raw = datetime.date.today() - delta
return default_date_raw
def get_last_seen_sources(self, asn, dates_sources):
"""
Return a dictionary conteining the last date a particular ASN
has been seen by a source.
"""
if not self.asn_exists(asn):
return {}
string = '{asn}|{date}|{source}|rankv{ip_version}'
s_dates = dates_sources.keys()
s_dates.sort(reverse=True)
p = self.history_db.pipeline()
for date in s_dates:
sources = dates_sources[date]
if len(sources) > 0:
[p.exists(string.format(asn=asn, date=date, source=source, ip_version=c.ip_version)) for source in sources]
asns_found = p.execute()
i = 0
to_return = {}
for date in s_dates:
sources = dates_sources[date]
if len(sources) > 0:
for source in sources:
if to_return.get(source) is None and asns_found[i]:
to_return[source] = date
i += 1
return to_return
def get_all_ranks_single_asn(self, asn, dates_sources, with_details_sources=False):
"""
Get all the ranks on a timeframe for a single ASN.
:param asn: Autonomous System Number
:param dates_sources: Dictionaries of the dates and sources
.. note:: Format of the dictionary:
.. code-block:: python
{
YYYY-MM-DD: [source1, source2, ...],
YYYY-MM-DD: [source1, source2, ...],
...
}
:param with_details_sources: Returns the details by sources if True
:rype: Dictionary
.. note:: Format of the output:
.. code-block:: python
{
date1:
{
'details':
[
(source1, rank),
(source2, rank),
...
],
'total': sum_of_ranks,
'description': description
}
...
}
The details key is only present if
``with_details_sources`` is True.
"""
to_return = {}
if with_details_sources is None:
with_details_sources = False
if not self.asn_exists(asn):
return to_return
string = '{asn}|{date}|{source}|rankv{ip_version}'
p = self.history_db.pipeline()
for date, sources in dates_sources.iteritems():
if len(sources) > 0:
p.mget([string.format(asn=asn, date=date, source=source, ip_version=c.ip_version) for source in sources])
ranks = p.execute()
i = 0
for date, sources in dates_sources.iteritems():
impacts = self.get_all_weights(date)
if len(sources) > 0:
to_return[date] = {}
zipped_sources_rank = zip(sources, ranks[i])
if with_details_sources:
to_return[date]['details'] = list(zipped_sources_rank)
to_return[date]['description'] = self.asn_desc_via_history(asn)
to_return[date]['total'] = 0
for s, r in zipped_sources_rank:
if r is not None:
to_return[date]['total'] += float(r) * float(impacts[s])
i += 1
return to_return
def existing_asns_timeframe(self, dates_sources):
"""
Get all the ASNs seen in the lists on a timeframe.
"""
asns_keys = []
for date, sources in dates_sources.iteritems():
asns_keys.extend(['{date}|{source}|asns'.format(date=date, source=source) for source in sources])
p = self.global_db.pipeline(False)
[p.smembers(k) for k in asns_keys]
asns_list = p.execute()
return set.union(*asns_list)
def get_all_ranks_all_asns(self, dates_sources, with_details_sources=False):
"""
Get all ranks for all the ASNs on a timeframe.
:param dates_sources: Dictionaries of the dates and sources
.. note:: Format of the dictionary:
.. code-block:: python
{
YYYY-MM-DD: [source1, source2, ...],
YYYY-MM-DD: [source1, source2, ...],
...
}
:param with_details_sources: Also returns the ranks by sources
:rype: Dictionary
.. note:: Format of the dictionary:
.. code-block:: python
{
YYYY-MM-DD:
{
asn1 :
{
'details':
[
(source1, rank),
(source2, rank),
...
],
'total': sum_of_ranks,
'description': description
}
...
}
...
}
The details key is only present if
``with_details_sources`` is True.
"""
asns = self.existing_asns_timeframe(dates_sources)
to_return = {}
for asn in asns:
details = self.get_all_ranks_single_asn(asn, dates_sources, with_details_sources)
for date, entries in details.iteritems():
if to_return.get(date) is None:
to_return[date] = {}
to_return[date][asn] = entries
return to_return
def get_all_blocks(self, asn):
"""
Expose the list of blocks announced by an AS.
:param asn: Autonomous System Number
:rtype: Set of ip blocks
"""
return self.global_db.smembers(asn)
def get_all_blocks_description(self, asn):
"""
Get the most recent description of all the blocks announced by an AS
:param asn: Autonomous System Number
:rtype: List
.. note:: Format of the list:
.. code-block:: python
[asn, [(block, description), ...]]
"""
blocks = self.get_all_blocks(asn)
if len(blocks) > 0:
block_keys = ['{asn}|{block}'.format(asn=asn, block=block) for block in blocks]
p = self.global_db.pipeline(False)
[p.hgetall(block_key) for block_key in block_keys]
ts_descr = p.execute()
i = 0
block_descr = []
for block_key in block_keys:
if ts_descr[i] is not None:
asn, block = block_key.split('|')
most_recent_ts = sorted(ts_descr[i].keys())[0]
descr = ts_descr[i][most_recent_ts]
block_descr.append((block, descr))
i += 1
return asn, block_descr
else:
return asn, []
def get_first_seen(self, asn, block):
"""
Get the oldest timestamp where the block has been announced by the AS
"""
timestamps = self.global_db.hkeys(asn + '|' + block)
if timestamps is not None and len(timestamps) > 0:
timestamps.sort()
return timestamps[0]
return None
def get_block_descriptions(self, asn, block):
"""
Return all the descriptions of a block, in a listed sorted by
timestamp (new => old)
:param asn: Autonomous System Number
:param block: IP Block you are looking for
:rtype: List
.. note:: Format of the list:
.. code-block:: python
[asn, block, [(ts, descr), ...]]
"""
ts_descr = self.global_db.hgetall('{}|{}'.format(asn, block))
timestamps = sorted(ts_descr.keys(), reverse=True)
descriptions = [(t, ts_descr[t])for t in timestamps]
return asn, block, descriptions
def get_all_blocks_descriptions(self, asn):
"""
Return all tuples timestamp-description of all the blocs announced
by an AS over time.
:param asn: Autonomous System Number
:rtype: List
.. note:: Format of the list:
.. code-block:: python
[ asn,
{
block:
[
(timestamp, description),
...
],
...
}
]
"""
blocks_descriptions = {}
for block in self.get_all_blocks(asn):
asn, block, descriptions = self.get_block_descriptions(asn, block)
blocks_descriptions[block] = descriptions
return asn, blocks_descriptions
def asn_desc_via_history(self, asn):
if self.has_asnhistory:
asn_descr = self.asnhistory.get_last_description(asn)
if asn_descr is None:
# The ASN has no descripion in the database
# publisher.error(\
# 'Unable to find the ASN description of {}. ASN History might be down.'.\
# format(asn))
asn_descr = 'No ASN description has been found.'
else:
publisher.debug('ASN History not enabled.')
asn_descr = 'ASN History not enabled.'
return asn_descr
def get_asn_descs(self, asn, date=None, sources=None):
"""
Get all what is available in the database about an ASN for
one day
:param asn: Autonomous System Number
:param date: Date of the information (default: last ranked day)
:param sources: List of sources (default: the available sources
at ``date``)
:rtype: Dictionary
.. note:: Format of the dictionary:
.. code-block:: python
{
'date': date,
'sources': [source1, source2, ...],
'asn': asn,
'asn_description': asn_description,
asn:
{
'clean_blocks':
{
block: [[ts, descr], ...],
....
},
'old_blocks':
{
block: [[ts, descr], ...],
....
},
block:
{
'description': description,
'timestamp': ts,
'old_descr': [(timestamp, description), ...]
'nb_of_ips': nb,
'sources': [source1, source2, ...]
'rank': subnet_rank
},
...
}
}
"""
if not self.asn_exists(asn):
# The ASN does not exists in the database
return None
if date is None:
date = self.get_default_date()
day_sources = self.daily_sources([date])
if sources is None:
sources = list(day_sources)
else:
if not isinstance(sources, list):
sources = [sources]
sources = list(day_sources.intersection(set(sources)))
asn_descr = self.asn_desc_via_history(asn)
to_return = {'date': date, 'sources': sources, 'asn': asn, 'asn_description': asn_descr, asn: {}}
key_clean_set = '{asn}|{date}|clean_set'.format(asn=asn, date=date)
for ip_block in self.get_all_blocks(asn):
# Get descriptions
asn, block, ts_descr = self.get_block_descriptions(asn, ip_block)
# Find out if the block has been seen these day
p = self.global_db.pipeline(False)
[p.scard('{}|{}|{}|{}'.format(asn, ip_block, date, source)) for source in sources]
ips_by_sources = p.execute()
nb_of_ips = sum(ips_by_sources)
if nb_of_ips == 0:
if ip_block in self.history_db.smembers(key_clean_set):
if to_return[asn].get('clean_blocks') is None:
to_return[asn]['clean_blocks'] = {}
to_return[asn]['clean_blocks'][ip_block] = ts_descr
else:
if to_return[asn].get('old_blocks') is None:
to_return[asn]['old_blocks'] = {}
to_return[asn]['old_blocks'][ip_block] = ts_descr
else:
impacts = self.get_all_weights(date)
# Compute the local ranking: the ranking if this subnet is
# the only one for this AS
i = 0
sum_rank = 0
sources_exists = []
for source in sources:
sum_rank += float(ips_by_sources[i]) * float(impacts[source])
if ips_by_sources[i] != 0:
sources_exists.append(source)
i += 1
local_rank = sum_rank / IPy.IP(ip_block).len()
to_return[asn][ip_block] = {'description': ts_descr[0][1], 'timestamp': ts_descr[0][0],
'old_descr': ts_descr[1:], 'nb_of_ips': nb_of_ips,
'sources': sources_exists, 'rank': local_rank}
return to_return
def get_all_weights(self, date=None):
"""
Get the weights for all the sources.
:param date: Date of the information (default: last ranked day)
:rtype: Dictionary
.. note:: Format of the dictionary:
.. code-block:: python
{
source: date,
...
}
"""
if date is None:
date = self.get_default_date()
sources = self.daily_sources([date])
to_return = {}
if len(sources) > 0:
impacts = self.config_db.mget(sources)
to_return = dict(zip(sources, impacts))
return to_return
def daily_sources(self, dates):
"""
Get the sources parsed during a list of dates
:param dates: List of dates
:rtype: Set of sources for each date
"""
if len(dates) == 1:
return self.global_db.smembers('{date}|sources'.format(date=dates[0]))
else:
p = self.global_db.pipeline(False)
[p.smembers('{date}|sources'.format(date=date)) for date in dates]
return p.execute()
def get_ips_descs(self, asn, block, date=None, sources=None):
"""
Get all what is available in the database about an subnet for one day
:param asn: Autonomous System Number
:param asn_timestamp: First the subnet has been seen in the db
(for this ASN)
:param date: Date of the information (default: last ranked day)
:param sources: List of sources (default: the available sources
at ``date``)
:rtype: Dictionary
.. note:: Format of the dictionary:
.. code-block:: python
{
'date': date,
'sources': [source1, source2, ...],
'asn': asn,
'asn_description': asn_description,
'block': block
block:
{
ip:
{
'sources': [source1, source2, ...],
'ptrrecord': 'ptr.record.com'
}
...
}
}
"""
if date is None:
date = self.get_default_date()
day_sources = self.daily_sources([date])
if sources is None:
sources = list(day_sources)
else:
if not isinstance(sources, list):
sources = [sources]
sources = list(day_sources.intersection(set(sources)))
asn_descr = self.asn_desc_via_history(asn)
to_return = {'date': date, 'sources': sources, 'asn': asn, 'asn_description': asn_descr, 'block': block, block: {}}
asn_block_key = '{asn}|{b}|{date}|'.format(asn=asn, b=block, date=date)
pipeline = self.global_db.pipeline(False)
[pipeline.smembers('{asn_b}{source}'.format(asn_b=asn_block_key, source=source)) for source in sources]
ips_by_source = pipeline.execute()
i = 0
for source in sources:
ips = ips_by_source[i]
for ip_details in ips:
ip, timestamp = ip_details.split('|')
if to_return[block].get(ip) is None:
to_return[block][ip] = {'sources': [], 'ptrrecord': None}
to_return[block][ip]['sources'].append(source)
i += 1
if self.has_ptr:
for ip in to_return[block]:
to_return[block][ip]['ptrrecord'] = self.get_ptr_record(ip)
return to_return
def get_stats(self, dates_sources):
"""
Return amount of asn and subnets found by source on a list of days.
:param dates_sources: Dictionaries of the dates and sources
:rtype: Dictionary
.. note:: Format of the Dictionary
.. code-block:: python
{
date:
{
'sources':
{
source : [nb_asns, nb_subnets],
...
},
'total_asn': total_asn
'total_subnets': total_subnets
}
...
}
"""
to_return = {}
p = self.global_db.pipeline(False)
for date, sources in dates_sources.iteritems():
to_return[date] = {}
for source in sources:
current_key = '{date}|{source}|asns'.format(date=date, source=source)
p.scard(current_key)
p.scard(current_key + '_details')
cards = p.execute()
i = 0
for date, sources in dates_sources.iteritems():
to_return[date] = {'sources': {}}
total_asn = 0
total_subnets = 0
for source in sources:
nb_asns = cards[i]
nb_subnets = cards[i + 1]
total_asn += nb_asns
total_subnets += nb_subnets
to_return[date]['sources'].update({source: (nb_asns, nb_subnets)})
i = i + 2
to_return[date]['total_asn'] = total_asn
to_return[date]['total_subnets'] = total_subnets
return to_return
# Need cached database
def cache_get_dates(self):
"""
**From the temporary database**
Get a list of dates. The ranking are available in the cache database.
"""
return sorted(self.history_db_cache.smembers('all_dates'))
def cache_get_stats(self):
"""
Return amount of asn and subnets found by source from the cache.
"""
dates = self.cache_get_dates()
dates_sources = dict(zip(dates, self.daily_sources(dates)))
return self.get_stats(dates_sources)
def cache_get_daily_rank(self, asn, source='global', date=None):
"""
**From the temporary database**
Get a single rank.
:param asn: Autonomous System Number
:param source: Source to use. global is the aggregated view for
all the sources
:param date: Date of the information (default: last ranked day)
:rtype: List
.. note:: Format of the list
.. code-block:: python
[asn, asn_description, date, source, rank]
"""
if source is None:
source = 'global'
if date is None:
date = self.get_default_date()
histo_key = '{date}|{source}|rankv{ip_version}'.format(date=date, source=source, ip_version=c.ip_version)
asn_descr = self.asn_desc_via_history(asn)
return asn, asn_descr, date, source, self.history_db_cache.zscore(histo_key, asn)
def cache_get_position(self, asn, source='global', date=None):
"""
**From the temporary database**
Get the position of the ASN in the zrank.
:param asn: Autonomous System Number
:param source: Source to use. global is the aggregated view for
all the sources
:param date: Date of the information (default: last ranked day)
:rtype: Integer, position in the list and size of the list.
.. note:: if None, the zrank does not exists (source or date invalid)
"""
if source is None:
source = 'global'
if date is None:
date = self.get_default_date()
histo_key = '{date}|{source}|rankv{ip_version}'.format(date=date, source=source, ip_version=c.ip_version)
return self.history_db_cache.zrevrank(histo_key, asn), self.history_db_cache.zcard(histo_key)
def cache_get_top_asns(self, source='global', date=None, limit=100, with_sources=True):
"""
**From the temporary database**
Get worse ASNs.
:param source: Source used to rank the ASNs. global is the
aggregated view for all the sources
:param date: Date of the information (default: last ranked day)
:param limit: Number of ASNs to get
:param with_sources: Get the list of sources where each ASN has
been found.
:rtype: Dictionary
.. note:: Format of the Dictionary
.. code-block:: python
{
'source': source,
'date': date,
'size_list': size,
'top_list':
[
(
(asn, asn_description, rank),
set([source1, source2, ...])
),
...
]
}
The set of sources is only presetn if ``with_sources``
is True
"""
if source is None:
source = 'global'
if date is None:
date = self.get_default_date()
if limit is None:
limit = 100
if with_sources is None:
with_sources = True
histo_key = '{date}|{histo_key}|rankv{ip_version}'.format(date=date, histo_key=source, ip_version=c.ip_version)
to_return = {'source': source, 'date': date, 'size_list': self.history_db_cache.zcard(histo_key), 'top_list': []}
ranks = self.history_db_cache.zrevrange(histo_key, 0, limit, True)
if ranks is None:
return to_return
temp_rank = []
for asn, rank in ranks:
if asn == '-1':
continue
asn_descr = self.asn_desc_via_history(asn)
temp_rank.append((asn, asn_descr, rank))
if not with_sources:
to_return['top_list'] = temp_rank
else:
p = self.history_db_cache.pipeline(False)
[p.smembers('{}|{}'.format(date, rank[0])) for rank in temp_rank]
to_return['top_list'] = list(zip(temp_rank, p.execute()))
return to_return
