示例#1
0
    def __init__(self, fixture_db_session):
        s = fixture_db_session
        packets = [swift_bat_grb_655721]
        packets.extend(fake.heartbeat_packets(role=vp.definitions.roles.test))

        extra_packets = fake.heartbeat_packets(
            start=fake.default_start_dt + timedelta(hours=24),
            role=vp.definitions.roles.utility)

        # Packets referenced by other packets:
        # self.cited = set()
        # Count times ivorn referenced by other packets in db :
        self.cite_counts = defaultdict(int)
        # Packets containing at least one cite entry
        self.followup_packets = []
        # Total number of citations (one packet may have multiple cite entries)
        self.n_citations = 0

        c0 = packets[0].attrib['ivorn']
        c1 = packets[1].attrib['ivorn']
        #One reference in ep0
        self.add_reference(extra_packets[0], c0)
        #Two references in ep1
        self.add_reference(extra_packets[1], c0)
        self.add_reference(extra_packets[1], c1)
        #
        # #Now cite ep[0], making it both cites / cited_by
        c2 = extra_packets[0].attrib['ivorn']
        self.add_reference(extra_packets[2], c2)

        # Add a citation to an external packet
        self.add_reference(extra_packets[3],
                           swift_xrt_grb_655721.attrib['ivorn'])

        packets.extend(extra_packets)
        self.packet_dict = {pkt.attrib['ivorn']: pkt for pkt in packets}

        self.insert_packets = packets[:-1]
        self.insert_packets_dumps = [vp.dumps(v) for v in self.insert_packets]
        self.streams = [
            v.attrib['ivorn'].split('#')[0][6:] for v in self.insert_packets
        ]
        self.stream_set = list(set(self.streams))
        self.roles = [v.attrib['role'] for v in self.insert_packets]
        self.role_set = list(set(self.roles))
        self.remaining_packet = packets[-1]
        # Insert all but the last packet, this gives us a useful counter-example
        s.add_all((Voevent.from_etree(p) for p in self.insert_packets))
        self.n_inserts = len(self.insert_packets)
        self.inserted_ivorns = [p.attrib['ivorn'] for p in self.insert_packets]
        self.absent_ivorn = self.remaining_packet.attrib['ivorn']
示例#2
0
    def __init__(self, fixture_db_session):
        s = fixture_db_session
        packets = [swift_bat_grb_655721]
        packets.extend(fake.heartbeat_packets(role=vp.definitions.roles.test))

        extra_packets = fake.heartbeat_packets(
            start=fake.default_start_dt + timedelta(hours=24),
            role=vp.definitions.roles.utility)

        # Packets referenced by other packets:
        # self.cited = set()
        # Count times ivorn referenced by other packets in db :
        self.cite_counts = defaultdict(int)
        # Packets containing at least one cite entry
        self.followup_packets = []
        # Total number of citations (one packet may have multiple cite entries)
        self.n_citations = 0

        c0 = packets[0].attrib['ivorn']
        c1 = packets[1].attrib['ivorn']
        #One reference in ep0
        self.add_reference(extra_packets[0],  c0)
        #Two references in ep1
        self.add_reference(extra_packets[1],c0)
        self.add_reference(extra_packets[1],c1)
        #
        # #Now cite ep[0], making it both cites / cited_by
        c2 = extra_packets[0].attrib['ivorn']
        self.add_reference(extra_packets[2],c2)

        # Add a citation to an external packet
        self.add_reference(extra_packets[3], swift_xrt_grb_655721.attrib['ivorn'])

        packets.extend(extra_packets)
        self.packet_dict = { pkt.attrib['ivorn'] : pkt for pkt in packets }

        self.insert_packets = packets[:-1]
        self.insert_packets_dumps = [vp.dumps(v) for v in self.insert_packets]
        self.streams = [v.attrib['ivorn'].split('#')[0][6:]
                        for v in self.insert_packets]
        self.stream_set = list(set(self.streams))
        self.roles = [v.attrib['role'] for v in self.insert_packets]
        self.role_set = list(set(self.roles))
        self.remaining_packet = packets[-1]
        # Insert all but the last packet, this gives us a useful counter-example
        s.add_all(
            (Voevent.from_etree(p) for p in self.insert_packets))
        self.n_inserts = len(self.insert_packets)
        self.inserted_ivorns = [p.attrib['ivorn'] for p in self.insert_packets]
        self.absent_ivorn = self.remaining_packet.attrib['ivorn']
示例#3
0
def test_simple_tarball_dump(named_temporary_file):
    voevent_etrees = fake.heartbeat_packets()
    voevent_rowgen = (models.Voevent.from_etree(v) for v in voevent_etrees)
    fname = named_temporary_file.name
    filestore.write_tarball(voevent_rowgen, fname)
    tarball = tarfile.open(fname)
    assert len(tarball.getmembers()) == len(voevent_etrees)
示例#4
0
def test_tarball_round_trip(named_temporary_file, fixture_db_session):
    voevent_etrees = fake.heartbeat_packets()
    # with open(assasn_non_ascii_packet_filepath, 'rb') as f:
    #     voevent_etrees.append(vp.load(f))
    s = fixture_db_session
    for etree in voevent_etrees:
        s.add(models.Voevent.from_etree(etree))
    s.flush()
    voevent_dbrows = s.query(models.Voevent.ivorn, models.Voevent.xml).all()
    assert len(voevent_dbrows) == len(voevent_etrees)
    voevent_rowgen = list(models.Voevent.from_etree(v) for v in voevent_etrees)
    assert voevent_dbrows[0].ivorn == voevent_rowgen[0].ivorn
    assert voevent_dbrows[0].xml == voevent_rowgen[0].xml

    assert type(voevent_dbrows[0].xml) == type(voevent_rowgen[0].xml)
    assert type(voevent_rowgen[0].xml) == six.binary_type

    # Therefore it's crucial to test with an actual round-tripped dataset,
    # the 'voevent_dbrows' from above:
    fname = named_temporary_file.name
    filestore.write_tarball(voevent_dbrows, fname)

    loaded_voevents = [
        vp.loads(s.xml) for s in filestore.tarfile_xml_generator(fname)
    ]

    def to_strings(voeventlist):
        return [vp.dumps(v) for v in voeventlist]

    def to_ivorn(voeventlist):
        return [v.attrib['ivorn'] for v in voeventlist]

    assert (to_ivorn(voevent_etrees) == to_ivorn(loaded_voevents))
    assert (to_strings(voevent_etrees) == to_strings(loaded_voevents))
示例#5
0
def test_simple_tarball_dump(named_temporary_file):
    voevent_etrees = fake.heartbeat_packets()
    voevent_rowgen = (models.Voevent.from_etree(v) for v in voevent_etrees)
    fname = named_temporary_file.name
    filestore.write_tarball(voevent_rowgen, fname)
    tarball = tarfile.open(fname)
    assert len(tarball.getmembers()) == len(voevent_etrees)
示例#6
0
def test_ingest_packet():
    #Must create corpusdb first:
    assert db_utils.check_database_exists(testdb_corpus_url)
    engine = sqlalchemy.create_engine(testdb_corpus_url)
    s = sqlalchemy.orm.Session(bind=engine)
    from voeventdb.server import __path__ as root_path
    root_path = root_path[0]
    script_path = os.path.join(root_path, 'bin', 'voeventdb_ingest_packet.py')

    print("Testing script at ", script_path)
    print("Using executable:", sys.executable)

    # Do stuff
    n_before = s.query(Voevent).count()
    proc = subprocess.Popen(
        [
            script_path,
            '-d={}'.format(testdb_corpus_url.database),
            '-l={}'.format('/tmp/vdbingest-test.log'),
        ],
        stdin=subprocess.PIPE,
    )
    proc.communicate(voeventparse.dumps(
        fake.heartbeat_packets(n_packets=1)[0]))
    proc.wait()

    assert proc.returncode == 0
    assert s.query(Voevent).count() == n_before + 1
示例#7
0
def test_unicode_voevent_tarball_dump(named_temporary_file):
    ## Now try some unicode characters
    voevent_etrees = fake.heartbeat_packets()
    vp.set_author(voevent_etrees[0], contactName=u"€€€€")
    voevent_rowgen = (models.Voevent.from_etree(v) for v in voevent_etrees)
    fname = named_temporary_file.name
    filestore.write_tarball(voevent_rowgen, fname)
    tarball = tarfile.open(fname)
    assert len(tarball.getmembers()) == len(voevent_etrees)
示例#8
0
def test_unicode_voevent_tarball_dump(named_temporary_file):
    ## Now try some unicode characters
    voevent_etrees = fake.heartbeat_packets()
    vp.set_author(voevent_etrees[0], contactName=u"€€€€")
    voevent_rowgen = (models.Voevent.from_etree(v) for v in voevent_etrees)
    fname = named_temporary_file.name
    filestore.write_tarball(voevent_rowgen, fname)
    tarball = tarfile.open(fname)
    assert len(tarball.getmembers()) == len(voevent_etrees)
示例#9
0
 def insert_voevents(self, fixture_db_session):
     """
     Insert two Vovents (mostly blank 'heartbeat' packet, GRB) as setup
     """
     s = fixture_db_session
     assert len(s.query(Voevent).all()) == 0  # sanity check
     s.add(Voevent.from_etree(fake.heartbeat_packets()[0]))
     s.add(Voevent.from_etree(swift_bat_grb_655721))
     s.flush()
     assert len(s.query(Voevent).all()) == 2  # 1 with, 1 without position
示例#10
0
 def insert_voevents(self, fixture_db_session):
     """
     Insert two Vovents (mostly blank 'heartbeat' packet, GRB) as setup
     """
     s = fixture_db_session
     assert len(s.query(Voevent).all()) == 0  # sanity check
     s.add(Voevent.from_etree(fake.heartbeat_packets()[0]))
     s.add(Voevent.from_etree(swift_bat_grb_655721))
     s.flush()
     assert len(s.query(Voevent).all()) == 2  # 1 with, 1 without position
示例#11
0
def test_tarball_round_trip(named_temporary_file, fixture_db_session):
    voevent_etrees = fake.heartbeat_packets()
    # with open(assasn_non_ascii_packet_filepath, 'rb') as f:
    #     voevent_etrees.append(vp.load(f))
    s = fixture_db_session
    for etree in voevent_etrees:
        s.add(models.Voevent.from_etree(etree))
    s.flush()
    voevent_dbrows = s.query(models.Voevent.ivorn, models.Voevent.xml).all()
    assert len(voevent_dbrows) == len(voevent_etrees)
    voevent_rowgen = list(models.Voevent.from_etree(v) for v in voevent_etrees)
    assert voevent_dbrows[0].ivorn == voevent_rowgen[0].ivorn
    assert voevent_dbrows[0].xml == voevent_rowgen[0].xml
    # Here's the crux
    # A newly instantiated model will store a string type same as Python 2;
    # bytestring stores as bytestring, unicode as unicode.
    # However, after a round-trip to the database, proper typing has been
    # asserted and the bytestring is returned as unicode!
    assert type(voevent_dbrows[0].xml) != type(voevent_rowgen[0].xml)
    assert type(voevent_rowgen[0].xml) == str
    assert type(voevent_dbrows[0].xml) == unicode

    # Therefore it's crucial to test with an actual round-tripped dataset,
    # the 'voevent_dbrows' from above:
    fname = named_temporary_file.name
    filestore.write_tarball(voevent_dbrows, fname)

    loaded_voevents = [vp.loads(s.xml) for s in
                       filestore.tarfile_xml_generator(fname)]

    def to_strings(voeventlist):
        return [vp.dumps(v) for v in voeventlist]

    def to_ivorn(voeventlist):
        return [v.attrib['ivorn'] for v in voeventlist]

    assert (to_ivorn(voevent_etrees) == to_ivorn(loaded_voevents))
    assert (to_strings(voevent_etrees) == to_strings(loaded_voevents))
示例#12
0
def test_ingest_packet():
    # Must create corpusdb first:
    assert db_utils.check_database_exists(testdb_corpus_url)
    engine = sqlalchemy.create_engine(testdb_corpus_url)
    s = sqlalchemy.orm.Session(bind=engine)
    from voeventdb.server import __path__ as root_path

    root_path = root_path[0]
    script_path = os.path.join(root_path, "bin", "voeventdb_ingest_packet.py")

    print("Testing script at ", script_path)
    print("Using executable:", sys.executable)

    # Do stuff
    n_before = s.query(Voevent).count()
    proc = subprocess.Popen(
        [script_path, "-d={}".format(testdb_corpus_url.database), "-l={}".format("/tmp/vdbingest-test.log")],
        stdin=subprocess.PIPE,
    )
    proc.communicate(voeventparse.dumps(fake.heartbeat_packets(n_packets=1)[0]))
    proc.wait()

    assert proc.returncode == 0
    assert s.query(Voevent).count() == n_before + 1
示例#13
0
 def assign_test_client_and_initdb(self, flask_test_client,
                                   fixture_db_session):
     self.c = flask_test_client  # Purely for brevity
     n_packets = 17
     packets = heartbeat_packets(n_packets=n_packets)
     for counter, pkt in enumerate(packets, start=1):
         packet_dec = 180.0 / n_packets * counter -90
         coords = vp.Position2D(
                 ra=15, dec=packet_dec, err=0.1,
                 units=vp.definitions.units.degrees,
                 system=vp.definitions.sky_coord_system.utc_icrs_geo)
         # print "Inserting coords", coords
         vp.add_where_when(
                 pkt,
                 coords=coords,
                 obs_time=iso8601.parse_date(pkt.Who.Date.text),
                 observatory_location=vp.definitions.observatory_location.geosurface
         )
     self.packets = packets
     self.ivorn_dec_map = {}
     for pkt in self.packets:
         posn = vp.get_event_position(pkt)
         self.ivorn_dec_map[pkt.attrib['ivorn']] = posn.dec
         fixture_db_session.add(Voevent.from_etree(pkt))
示例#14
0
 def assign_test_client_and_initdb(self, flask_test_client,
                                   fixture_db_session):
     self.c = flask_test_client  # Purely for brevity
     n_packets = 17
     packets = heartbeat_packets(n_packets=n_packets)
     for counter, pkt in enumerate(packets, start=1):
         packet_dec = 180.0 / n_packets * counter - 90
         coords = vp.Position2D(
             ra=15, dec=packet_dec, err=0.1,
             units=vp.definitions.units.degrees,
             system=vp.definitions.sky_coord_system.utc_icrs_geo)
         # print "Inserting coords", coords
         vp.add_where_when(
             pkt,
             coords=coords,
             obs_time=iso8601.parse_date(pkt.Who.Date.text),
             observatory_location=vp.definitions.observatory_location.geosurface
         )
     self.packets = packets
     self.ivorn_dec_map = {}
     for pkt in self.packets:
         posn = vp.get_event_position(pkt)
         self.ivorn_dec_map[pkt.attrib['ivorn']] = posn.dec
         fixture_db_session.add(Voevent.from_etree(pkt))
 def test_heartbeat(self):
     n_interval = 4 * 6
     packets = fake.heartbeat_packets(self.start, self.interval, n_interval)
     self.assertEqual(n_interval, len(packets))
 def test_heartbeat(self):
     n_interval = 4*6
     packets = fake.heartbeat_packets(self.start, self.interval,
                                      n_interval)
     self.assertEqual(n_interval, len(packets))