示例#1
0
    def determine_new_entries(self):
        """
        Uses a local voeventdb to check for previously broadcast events.

        We assume that the currently configured stream_ivorn_prefix is the same
        as any previously broadcast voevents in the database. Therefore we
        can do an efficient lookup by calculating the relevant IVORN-prefix
        for each event in the `event_id_data_map` of this class.
        """
        s = session_registry()
        new_ids = []
        logger.debug("Checking database {} for duplicates from feed {}".format(
            s.bind.engine.url.database, self.name))
        logger.debug("Checking {} feed entries".format(
            len(self.event_id_data_map)))
        for feed_id in self.event_id_data_map:
            ivo = self.feed_id_to_ivorn(feed_id)
            if not dbconvenience.ivorn_present(s, ivo):
                duplicate_prefixes = self.get_ivorn_prefixes_for_duplicate(
                    feed_id)
                duplicate_present = False
                for prefix in duplicate_prefixes:
                    if dbconvenience.ivorn_prefix_present(s, prefix):
                        duplicate_present = True
                        logger.warning(
                            "Possible duplicate prefix detected: '{}', "
                            "will not insert '{}'".format(prefix, ivo))
                if not duplicate_present:
                    new_ids.append(feed_id)
        return new_ids
示例#2
0
def validate_ivorn(url_encoded_ivorn):
    if url_encoded_ivorn and current_app.config.get("APACHE_NODECODE"):
        ivorn = urllib.unquote(url_encoded_ivorn)
    else:
        ivorn = url_encoded_ivorn
    if ivorn is None:
        raise apierror.IvornNotSupplied(suggested_ivorn_url=url_for(apiv1.name + "." + ListIvorn.view_name))
    if not convenience.ivorn_present(db_session, ivorn):
        raise apierror.IvornNotFound(ivorn, suggested_ivorn_url=url_for(apiv1.name + "." + ListIvorn.view_name))
    return ivorn
示例#3
0
def load_from_tarfile(session, tarfile_path, check_for_duplicates,
                      pkts_per_commit=1000):
    """
    Iterate through xml files in a tarball and attempt to load into database.

    .. warning::
        Very slow with duplicate checking enabled.

    Returns:
        tuple: (n_parsed, n_loaded) - Total number of packets parsed from
            tarbar, and number successfully loaded.

    """
    tf_stream = tarfile_xml_generator(tarfile_path)
    logger.info("Loading: " + tarfile_path)
    n_parsed = 0
    n_loaded = 0
    for tarinf in tf_stream:
        try:
            v = vp.loads(tarinf.xml, check_version=False)
            if v.attrib['version'] != '2.0':
                logger.debug(
                    'Packet: {} is not VO-schema version 2.0.'.format(
                        tarinf.name))
            n_parsed += 1
        except:
            logger.exception('Error loading file {}, skipping'.format(
                tarinf.name))
            continue
        try:
            new_row = Voevent.from_etree(v)
            if check_for_duplicates:
                if ivorn_present(session, new_row.ivorn):
                    logger.debug(
                        "Ignoring duplicate ivorn: {} in file {}".format(
                            new_row.ivorn, tarinf.name))
                    continue
            session.add(new_row)
            n_loaded += 1
        except:
            logger.exception(
                'Error converting file {} to database row, skipping'.
                    format(tarinf.name))
            continue

        if n_loaded % pkts_per_commit == 0:
            session.commit()
    session.commit()
    logger.info("Successfully parsed {} packets, of which loaded {}.".format(n_parsed, n_loaded))
    return n_parsed, n_loaded
示例#4
0
def validate_ivorn(url_encoded_ivorn):
    if url_encoded_ivorn and current_app.config.get('APACHE_NODECODE'):
        ivorn = unquote(url_encoded_ivorn)
    else:
        ivorn = url_encoded_ivorn
    if ivorn is None:
        raise apierror.IvornNotSupplied(
            suggested_ivorn_url=url_for(apiv1.name + '.' +
                                        ListIvorn.view_name))
    if not convenience.ivorn_present(db_session, ivorn):
        raise apierror.IvornNotFound(
            ivorn,
            suggested_ivorn_url=url_for(apiv1.name + '.' +
                                        ListIvorn.view_name))
    return ivorn
 def test_ivorn_present(self, fixture_db_session, simple_populated_db):
     s = fixture_db_session
     dbinf = simple_populated_db
     assert ivorn_present(s, dbinf.inserted_ivorns[0]) == True
     assert ivorn_present(s, dbinf.absent_ivorn) == False
 def test_ivorn_present(self, fixture_db_session, simple_populated_db):
     s = fixture_db_session
     dbinf = simple_populated_db
     assert ivorn_present(s, dbinf.inserted_ivorns[0]) == True
     assert ivorn_present(s, dbinf.absent_ivorn) == False