def determine_new_entries(self):
"""
Uses a local voeventdb to check for previously broadcast events.
We assume that the currently configured stream_ivorn_prefix is the same
as any previously broadcast voevents in the database. Therefore we
can do an efficient lookup by calculating the relevant IVORN-prefix
for each event in the `event_id_data_map` of this class.
"""
s = session_registry()
new_ids = []
logger.debug("Checking database {} for duplicates from feed {}".format(
s.bind.engine.url.database, self.name))
logger.debug("Checking {} feed entries".format(
len(self.event_id_data_map)))
for feed_id in self.event_id_data_map:
ivo = self.feed_id_to_ivorn(feed_id)
if not dbconvenience.ivorn_present(s, ivo):
duplicate_prefixes = self.get_ivorn_prefixes_for_duplicate(
feed_id)
duplicate_present = False
for prefix in duplicate_prefixes:
if dbconvenience.ivorn_prefix_present(s, prefix):
duplicate_present = True
logger.warning(
"Possible duplicate prefix detected: '{}', "
"will not insert '{}'".format(prefix, ivo))
if not duplicate_present:
new_ids.append(feed_id)
return new_ids
def validate_ivorn(url_encoded_ivorn):
if url_encoded_ivorn and current_app.config.get("APACHE_NODECODE"):
ivorn = urllib.unquote(url_encoded_ivorn)
else:
ivorn = url_encoded_ivorn
if ivorn is None:
raise apierror.IvornNotSupplied(suggested_ivorn_url=url_for(apiv1.name + "." + ListIvorn.view_name))
if not convenience.ivorn_present(db_session, ivorn):
raise apierror.IvornNotFound(ivorn, suggested_ivorn_url=url_for(apiv1.name + "." + ListIvorn.view_name))
return ivorn
def load_from_tarfile(session, tarfile_path, check_for_duplicates,
pkts_per_commit=1000):
"""
Iterate through xml files in a tarball and attempt to load into database.
.. warning::
Very slow with duplicate checking enabled.
Returns:
tuple: (n_parsed, n_loaded) - Total number of packets parsed from
tarbar, and number successfully loaded.
"""
tf_stream = tarfile_xml_generator(tarfile_path)
logger.info("Loading: " + tarfile_path)
n_parsed = 0
n_loaded = 0
for tarinf in tf_stream:
try:
v = vp.loads(tarinf.xml, check_version=False)
if v.attrib['version'] != '2.0':
logger.debug(
'Packet: {} is not VO-schema version 2.0.'.format(
tarinf.name))
n_parsed += 1
except:
logger.exception('Error loading file {}, skipping'.format(
tarinf.name))
continue
try:
new_row = Voevent.from_etree(v)
if check_for_duplicates:
if ivorn_present(session, new_row.ivorn):
logger.debug(
"Ignoring duplicate ivorn: {} in file {}".format(
new_row.ivorn, tarinf.name))
continue
session.add(new_row)
n_loaded += 1
except:
logger.exception(
'Error converting file {} to database row, skipping'.
format(tarinf.name))
continue
if n_loaded % pkts_per_commit == 0:
session.commit()
session.commit()
logger.info("Successfully parsed {} packets, of which loaded {}.".format(n_parsed, n_loaded))
return n_parsed, n_loaded
def validate_ivorn(url_encoded_ivorn):
if url_encoded_ivorn and current_app.config.get('APACHE_NODECODE'):
ivorn = unquote(url_encoded_ivorn)
else:
ivorn = url_encoded_ivorn
if ivorn is None:
raise apierror.IvornNotSupplied(
suggested_ivorn_url=url_for(apiv1.name + '.' +
ListIvorn.view_name))
if not convenience.ivorn_present(db_session, ivorn):
raise apierror.IvornNotFound(
ivorn,
suggested_ivorn_url=url_for(apiv1.name + '.' +
ListIvorn.view_name))
return ivorn
def test_ivorn_present(self, fixture_db_session, simple_populated_db):
s = fixture_db_session
dbinf = simple_populated_db
assert ivorn_present(s, dbinf.inserted_ivorns[0]) == True
assert ivorn_present(s, dbinf.absent_ivorn) == False
def test_ivorn_present(self, fixture_db_session, simple_populated_db):
s = fixture_db_session
dbinf = simple_populated_db
assert ivorn_present(s, dbinf.inserted_ivorns[0]) == True
assert ivorn_present(s, dbinf.absent_ivorn) == False