def convert_ribs(config):
for file in config.rib_dir.iterdir():
m = meta_from_filename(file)
if isinstance(m, RIBMeta):
meta = IPASNMeta(m.collector, m.datetime)
dest = config.rib_dir / meta.filename
log.info("%s", meta)
if not dest.exists():
prefixes = mrtx.parse_mrt_file(str(file), print_progress=True)
mrtx.dump_prefixes_to_file(prefixes, dest, file)
def convert_asn_table(fn, target):
from os import path
try:
newfile = path.splitext(fn)[0] + '.dat'
logger.debug("Decompressing bz2 file.\n" + \
"newfile loc: %s\n" + \
"file being decompressed: %s", newfile, fn)
prefixes = mrtx.parse_mrt_file(str(fn),
print_progress=True,
skip_record_on_error=True)
mrtx.dump_prefixes_to_file(prefixes, newfile, str(fn))
except:
logger.warning("Could not download ASN database.")
logger.warning(
"Instantiating a default DAT file. This may be very out of date!")
return newfile
def fetch_rib(force=False):
"""Process the routeview data."""
if not to_download() and not force:
return
logger.debug("Downloading the latest RIB")
meta = gen_request()
response = requests.get(meta['url'])
path = '%s/resources/asn/ribs/%s' % (APP_BASE, meta['filename'])
open(path, 'wb').write(response.content)
logger.debug("RIB file saved")
current = '%s/resources/asn/current' % (APP_BASE)
logger.debug("Converting RIB to database format")
prefixes = mrtx.parse_mrt_file(path, print_progress=False,
skip_record_on_error=True)
mrtx.dump_prefixes_to_file(prefixes, current, path)
logger.debug("Updated the database")
config = json.load(open('%s/resources/config.json' % APP_BASE))
config['asn']['last_rib_file'] = meta['filename']
config['asn']['last_update'] = str_now_time()
json.dump(config, open('%s/resources/config.json' % APP_BASE, 'w'),
indent=4)
def verify_asn_database():
"""
looks for latest version of ASN file in this folder
"""
if any(item == "asn_{}.asn".format(datetime.today().strftime('%Y%m%d'))
for item in os.listdir('.')):
print('latest asn file found')
return [item for item in os.listdir('.') if item.endswith('.asn')][0]
if not any((item.startswith("rib.{}".format(datetime.today().strftime('%Y%m%d'))) \
and item.endswith('.bz2')) for item in os.listdir('.')):
print(
'Today\'s version of DB not found. Will be downloading latest definitions from archive.routeviews.org. Downloading and parsing DB will take around 1 min.'
)
download_rib_file()
rib_file = sorted([item for item in os.listdir('.') if \
(item.startswith('rib.{}'.format(datetime.today().strftime('%Y%m%d')))\
and item.endswith('.bz2'))])[-1]
prefixes = mrtx.parse_mrt_file(rib_file,
print_progress=True,
skip_record_on_error=True)
mrtx.dump_prefixes_to_file(
prefixes, "asn_{}.asn".format(datetime.today().strftime('%Y%m%d'),
rib_file))
return ("asn_{}.asn".format(datetime.today().strftime('%Y%m%d')))
metavar="N",
action="store",
help="start dump from record N (with --dump-screen)")
parser.add_argument("--record-to",
type=int,
metavar="N",
action="store",
help="end dump at record N (with --dump-screen)")
args = parser.parse_args()
if args.version:
print("MRT/RIB converter version %s." % __version__)
if args.single:
prefixes = mrtx.parse_mrt_file(args.single[0],
print_progress=not args.no_progress,
skip_record_on_error=args.skip_on_error)
mrtx.dump_prefixes_to_file(prefixes, args.single[1], args.single[0])
if not args.no_progress:
v6 = sum(1 for x in prefixes if ':' in x)
v4 = len(prefixes) - v6
print('IPASN database saved (%d IPV4 + %d IPV6 prefixes)' % (v4, v6))
if args.compress:
call(['gzip', args.single[1]])
if args.dump_screen:
mrtx.dump_screen_mrt_file(args.dump_screen[0],
record_to=args.record_to,
record_from=args.record_from,
screen=stdout)
print(
'\n This script converts MRT/RIB export (downloadable from RouteViews or RIPE RIS) to IPASN-databases'
)
print(
' For bulk mode, dates should be in yyyy-mm-dd format, and files saved into current folder.'
)
print(
' Use the binary switch to save the output in binary format.')
exit()
binary_output = '--binary' in argv[4:]
if argv[1] == '--single':
f = BZ2File(argv[2], 'rb')
print_progress = '--no-progress' not in argv[4:]
dat = mrtx.parse_mrt_file(f, print_progress=print_progress)
f.close()
if not binary_output:
mrtx.dump_prefixes_to_text_file(dat, argv[3], argv[2])
else:
mrtx.dump_prefixes_to_binary_file(dat, argv[3], argv[2])
print('IPASN database saved (%d prefixes)' % len(dat))
exit()
assert argv[1] == '--bulk'
try:
dt = datetime.strptime(argv[2], '%Y-%m-%d').date()
dt_end = datetime.strptime(argv[3], '%Y-%m-%d').date()
except ValueError:
print('Malformed date. Try yyyy-mm-dd')
exit()
if len(argv) not in (4, 5, 6) or argv[1] not in ('--single', '--bulk'):
# todo: rewrite using argparse
print('Usage: pyasn_convert_rib.py --single ribdump.bz2 ipasn.dat(.bin) [--binary] [--no-progress]')
print(' pyasn_convert_rib.py --bulk START_DATE END_DATE [--binary]')
print('\n This script converts MRT/RIB export (downloadable from RouteViews or RIPE RIS) to IPASN-databases')
print(' For bulk mode, dates should be in yyyy-mm-dd format, and files saved into current folder.')
print(' Use the binary switch to save the output in binary format.')
exit()
binary_output = '--binary' in argv[4:]
if argv[1] == '--single':
f = BZ2File(argv[2], 'rb')
print_progress = '--no-progress' not in argv[4:]
dat = mrtx.parse_mrt_file(f, print_progress=print_progress)
f.close()
if not binary_output:
mrtx.dump_prefixes_to_text_file(dat, argv[3], argv[2])
else:
mrtx.dump_prefixes_to_binary_file(dat, argv[3], argv[2])
print('IPASN database saved (%d prefixes)' % len(dat))
exit()
assert argv[1] == '--bulk'
try:
dt = datetime.strptime(argv[2], '%Y-%m-%d').date()
dt_end = datetime.strptime(argv[3], '%Y-%m-%d').date()
except ValueError:
print('Malformed date. Try yyyy-mm-dd')
exit()
parser.add_argument("--skip-on-error", action="store_true",
help="skip records which fail conversion, instead of stopping (with --single)")
parser.add_argument("--record-from", type=int, metavar="N", action="store",
help="start dump from record N (with --dump-screen)")
parser.add_argument("--record-to", type=int, metavar="N", action="store",
help="end dump at record N (with --dump-screen)")
args = parser.parse_args()
if args.version:
print("MRT/RIB converter version %s." % __version__)
if args.single:
prefixes = mrtx.parse_mrt_file(args.single[0],
print_progress=not args.no_progress,
skip_record_on_error=args.skip_on_error)
mrtx.dump_prefixes_to_file(prefixes, args.single[1], args.single[0])
if not args.no_progress:
v6 = sum(1 for x in prefixes if ':' in x)
v4 = len(prefixes) - v6
print('IPASN database saved (%d IPV4 + %d IPV6 prefixes)' % (v4, v6))
if args.compress:
call(['gzip', args.single[1]])
if args.dump_screen:
mrtx.dump_screen_mrt_file(args.dump_screen[0],
record_to=args.record_to,
record_from=args.record_from,
screen=stdout)
import datetime
import requests
from pyasn import mrtx, __version__
if __name__ == '__main__':
base = "http://archive.routeviews.org//bgpdata/"
now = datetime.datetime.utcnow()
slug = now.strftime('%Y.%m')
fname = now.strftime('rib.%Y%m%d.%H00.bz2')
hour = int(now.strftime('%H'))
if not hour % 2 == 0:
fname = now.strftime('rib.%Y%m%d.')
fname = fname + str(hour - 1) + '00.bz2'
url = "%s/%s/RIBS/%s" % (base, slug, fname)
response = requests.get(url)
path = 'app/resources/ribs/%s' % (fname)
open(path, 'wb').write(response.content)
current = 'app/resources/current'
prefixes = mrtx.parse_mrt_file(path,
print_progress=False,
skip_record_on_error=True)
mrtx.dump_prefixes_to_file(prefixes, current, path)
