def testSkipCommments1(self):
self.islands_tpl = [('name', 'text'), ('pop', 'integer'), ('discovered', 'integer') ]
self.islands_loader = sql3load(self.islands_tpl, self.db_fn, '\t', "islands")
r = self.islands_loader.importFile("test/test-import2.txt")
self.islands_loader2 = sql3load(self.islands_tpl, self.db_fn2, '\t', "islands") #, comments_mark="#")
p = self.islands_loader2.importFile("test/test-import2.txt")
#
a = self.islands_loader.get_rowcount()
b = self.islands_loader2.get_rowcount()
self.assertTrue(a==b, "Sin saltear comentarios, ambos archivos tienen que tener la misma cantidad de lineas x=%s, y=%s" % (a,b))
def testSkipCommments2(self):
self.islands_tpl = [('name', 'text'), ('pop', 'integer'), ('discovered', 'integer') ]
self.islands_loader = sql3load(self.islands_tpl, self.db_fn, '\t', "islands")
r = self.islands_loader.importFile("test/test-import2.txt")
self.islands_loader2 = sql3load(self.islands_tpl, self.db_fn2, '\t', "islands", comments_mark="#")
p = self.islands_loader2.importFile("test/test-import2.txt")
#
a = self.islands_loader.get_rowcount()
b = self.islands_loader2.get_rowcount()
self.assertTrue(a>b, "Salteando comentarios, uno tiene que tener que tener menos lineas, x=%s, y=%s" % (a,b))
def __init__(self, **kwargs):
"""
Class constructor, receives keyword arguments {rir} and {date}
"""
self.drir = kwargs.get('rir', 'lacnic')
self.ddate = kwargs.get('date', 'latest')
self.local_file = kwargs.get('local_file', None)
self.db_filename = kwargs.get('db_filename', get_tmp_fn(".db"))
self.as_cache = kwargs.get('as_cache', False)
# get archivo delegated
if self.local_file == None:
self.dlg_fn_name = self._download_stats_file(rir=self.drir,
date=self.ddate)
else:
self.dlg_fn_name = self.local_file
## load into database
self.s3_template = [('rir', 'text'), ('cc', 'text'), ('type', 'text'),
('block', 'text'), ('length', 'integer'),
('date', 'integer'), ('status', 'text'),
('orgid', 'integer')]
self.s3l = sql3load(self.s3_template,
self.db_filename,
"|",
"numres",
as_cache=self.as_cache,
comments_mark="#")
r = self.s3l.importFile(self.dlg_fn_name)
self.dbh = self.s3l
# Add Meta columns
self._add_columns()
return
def __init__(self, **kwargs):
"""
Class constructor, receives keyword arguments {rir} and {date}
"""
self.drir = kwargs.get('rir','lacnic')
self.ddate = kwargs.get('date', 'latest')
self.local_file = kwargs.get('local_file', None)
self.db_filename = kwargs.get('db_filename', get_tmp_fn(".db") )
self.as_cache = kwargs.get('as_cache', False)
# get archivo delegated
if self.local_file == None:
self.dlg_fn_name = self._download_stats_file(rir=self.drir, date=self.ddate)
else:
self.dlg_fn_name = self.local_file
## load into database
self.s3_template = [ ('rir', 'text'), ('cc', 'text'), ('type', 'text'),
('block', 'text'), ('length', 'integer'), ('date', 'integer'),
('status', 'text'), ('orgid', 'integer')
]
self.s3l = sql3load(self.s3_template, self.db_filename , "|", "numres", as_cache=self.as_cache, comments_mark="#")
r = self.s3l.importFile(self.dlg_fn_name)
self.dbh = self.s3l
# Add Meta columns
self._add_columns()
return
def __init__(self, **kwargs):
"""
Default constructor
:param kwargs: name parameters, see below
:param local_file: pointer to a file in the local filesystem, instead of trying to download the ris whois files
:param db_filename: pointer to the local database file.
"""
# import a local file instead of download
self.local_file = kwargs.get('local_file', None)
# database filename
self.db_filename = kwargs.get('db_filename', None)
# s3_template = [ ('origin_as', 'text'), ('prefix', 'text'), ('viewed_by', 'integer')]
csv_template = [('origin_as', 'text'), ('prefix', 'text'), ('maxlen', 'integer')]
self.dbh = sql3load(csv_template, self.db_filename, ",", "roadata")
self._add_columns() # add meta columns
if not self.local_file:
# download Dumps
self.local_file = self._getRoaCSVExport()
r = self._addEntries(local_file=self.local_file)
else:
r = self._addEntries(local_file=self.local_file)
#
return
def setUp(self):
self.s3_template = [ ('name', 'text'), ('age', 'integer'), ('weigth', 'float') ]
self.db_fn = get_tmp_fn(".db")
self.db_fn2 = get_tmp_fn(".db")
self.s3l = sql3load(self.s3_template, self.db_fn )
r = self.s3l.importFile("test/test-import.txt")
self.s3l.addMetaColumn("cDoubleAge VARCHAR(80)")
def __init__(self, **kwargs):
"""
Default constructor
:param kwargs: name parameters, see below
:param ipversion: ipv4 or ipv6
:param local_file: pointer to a file in the local filesystem, instead of trying to download the ris whois files
:param db_filename: pointer to the local database file.
"""
# set ip version
self.type = kwargs.get('ipversion', 'ipv4')
# import a local file instead of download
self.local_file = kwargs.get('local_file', None)
# database filename
self.db_filename = kwargs.get('db_filename', None)
s3_template = [ ('origin_as', 'text'), ('prefix', 'text'), ('viewed_by', 'integer')]
self.dbh = sql3load(s3_template, self.db_filename, "\t", "riswhois", comments_mark='%')
self._add_columns() # add meta columns
if not self.local_file:
# download Dumps
self.local_file2 = self._download_dump_files()
r = self.addEntries(local_file=self.local_file2[0])
r = self.addEntries(local_file=self.local_file2[1])
else:
r = self.addEntries(local_file=self.local_file)
#
return
def __init__(self, **kwargs):
"""
Default constructor
:param kwargs: name parameters, see below
:param ipversion: ipv4 or ipv6
:param local_file: pointer to a file in the local filesystem, instead of trying to download the ris whois files
:param db_filename: pointer to the local database file.
"""
# set ip version
self.type = kwargs.get('ipversion', 'ipv4')
# import a local file instead of download
self.local_file = kwargs.get('local_file', None)
# database filename
self.db_filename = kwargs.get('db_filename', None)
s3_template = [ ('origin_as', 'text'), ('prefix', 'text'), ('viewed_by', 'integer')]
self.dbh = sql3load(s3_template, self.db_filename, "\t", "riswhois")
self._add_columns() # add meta columns
if not self.local_file:
# download Dumps
self.local_file2 = self._download_dump_files()
r = self.addEntries(local_file=self.local_file2[0])
r = self.addEntries(local_file=self.local_file2[1])
else:
r = self.addEntries(local_file=self.local_file)
#
return
def __init__(self, **kwargs):
"""
Default constructor
:param kwargs: name parameters, see below
:param local_file: pointer to a file in the local filesystem, instead of trying to download the ris whois files
:param db_filename: pointer to the local database file.
:param url: url of a ripe validator exporting csv file
"""
# import a local file instead of download
self.local_file = kwargs.get('local_file', None)
# database filename
self.db_filename = kwargs.get('db_filename', None)
# set url or default to the well known one
self.validator_url = kwargs.get('url', "http://ripeval.labs.lacnic.net:8080/export.csv")
# retries
self._retries = 3
# s3_template = [ ('origin_as', 'text'), ('prefix', 'text'), ('viewed_by', 'integer')]
csv_template = [('origin_as', 'text'), ('prefix', 'text'), ('maxlen', 'integer'), ('ta', 'text')]
self.dbh = sql3load(csv_template, self.db_filename, ",", "roadata")
self._add_columns() # add meta columns
if not self.local_file:
# download Dumps
success = False
for n in range(0,self._retries):
try:
self.local_file = self._getRoaCSVExport()
r = self._addEntries(local_file=self.local_file)
success = True
break
except:
continue
# end for
if not success:
raise Exception("Unable to download ROA CSV export")
else:
r = self._addEntries(local_file=self.local_file)
#
return
def testMultipleCSVImport(self):
self.islands_tpl = [('name', 'text'), ('pop', 'integer'), ('discovered', 'integer') ]
self.islands_loader = sql3load(self.islands_tpl, self.db_fn, '\t', "islands")
r = self.islands_loader.importFile("test/test-import2.txt")
def setUp(self):
self.tp = [('name', 'text'), ('age', 'integer'), ('weigth', 'float')]
self.im = sql3load(self.tp, "tmp/test.db")
def setUp(self):
self.tp = [ ('name', 'text'), ('age', 'integer'), ('weigth', 'float') ]
self.im = sql3load(self.tp, "tmp/test.db")
def testImportTabSeparatedFile(self):
self.s3l2 = sql3load(self.s3_template, get_tmp_fn("islas.db"), "\t")
r = self.s3l2.importFile("test/test-import2.txt")
# print "imported rows %s" % (r)
self.assertTrue(r>3, "Number of lines read should be larger than 3 but is %s" % (r))
def setUp(self):
#sys.stderr.write("Creating sql3load class instance\n")
# self.s3_template = [{'name': 'text'}, {'age': 'integer'}, {'weigth': 'float'}]
self.s3_template = [ ('name', 'text'), ('age', 'integer'), ('weigth', 'float') ]
#se lf.s3l = sql3load(self.s3_template)
self.s3l = sql3load(self.s3_template, get_tmp_fn(".db") )
