def __init__(self, fixture_db_session): s = fixture_db_session packets = [swift_bat_grb_655721] packets.extend(fake.heartbeat_packets(role=vp.definitions.roles.test)) extra_packets = fake.heartbeat_packets( start=fake.default_start_dt + timedelta(hours=24), role=vp.definitions.roles.utility) # Packets referenced by other packets: # self.cited = set() # Count times ivorn referenced by other packets in db : self.cite_counts = defaultdict(int) # Packets containing at least one cite entry self.followup_packets = [] # Total number of citations (one packet may have multiple cite entries) self.n_citations = 0 c0 = packets[0].attrib['ivorn'] c1 = packets[1].attrib['ivorn'] #One reference in ep0 self.add_reference(extra_packets[0], c0) #Two references in ep1 self.add_reference(extra_packets[1], c0) self.add_reference(extra_packets[1], c1) # # #Now cite ep[0], making it both cites / cited_by c2 = extra_packets[0].attrib['ivorn'] self.add_reference(extra_packets[2], c2) # Add a citation to an external packet self.add_reference(extra_packets[3], swift_xrt_grb_655721.attrib['ivorn']) packets.extend(extra_packets) self.packet_dict = {pkt.attrib['ivorn']: pkt for pkt in packets} self.insert_packets = packets[:-1] self.insert_packets_dumps = [vp.dumps(v) for v in self.insert_packets] self.streams = [ v.attrib['ivorn'].split('#')[0][6:] for v in self.insert_packets ] self.stream_set = list(set(self.streams)) self.roles = [v.attrib['role'] for v in self.insert_packets] self.role_set = list(set(self.roles)) self.remaining_packet = packets[-1] # Insert all but the last packet, this gives us a useful counter-example s.add_all((Voevent.from_etree(p) for p in self.insert_packets)) self.n_inserts = len(self.insert_packets) self.inserted_ivorns = [p.attrib['ivorn'] for p in self.insert_packets] self.absent_ivorn = self.remaining_packet.attrib['ivorn']
def __init__(self, fixture_db_session): s = fixture_db_session packets = [swift_bat_grb_655721] packets.extend(fake.heartbeat_packets(role=vp.definitions.roles.test)) extra_packets = fake.heartbeat_packets( start=fake.default_start_dt + timedelta(hours=24), role=vp.definitions.roles.utility) # Packets referenced by other packets: # self.cited = set() # Count times ivorn referenced by other packets in db : self.cite_counts = defaultdict(int) # Packets containing at least one cite entry self.followup_packets = [] # Total number of citations (one packet may have multiple cite entries) self.n_citations = 0 c0 = packets[0].attrib['ivorn'] c1 = packets[1].attrib['ivorn'] #One reference in ep0 self.add_reference(extra_packets[0], c0) #Two references in ep1 self.add_reference(extra_packets[1],c0) self.add_reference(extra_packets[1],c1) # # #Now cite ep[0], making it both cites / cited_by c2 = extra_packets[0].attrib['ivorn'] self.add_reference(extra_packets[2],c2) # Add a citation to an external packet self.add_reference(extra_packets[3], swift_xrt_grb_655721.attrib['ivorn']) packets.extend(extra_packets) self.packet_dict = { pkt.attrib['ivorn'] : pkt for pkt in packets } self.insert_packets = packets[:-1] self.insert_packets_dumps = [vp.dumps(v) for v in self.insert_packets] self.streams = [v.attrib['ivorn'].split('#')[0][6:] for v in self.insert_packets] self.stream_set = list(set(self.streams)) self.roles = [v.attrib['role'] for v in self.insert_packets] self.role_set = list(set(self.roles)) self.remaining_packet = packets[-1] # Insert all but the last packet, this gives us a useful counter-example s.add_all( (Voevent.from_etree(p) for p in self.insert_packets)) self.n_inserts = len(self.insert_packets) self.inserted_ivorns = [p.attrib['ivorn'] for p in self.insert_packets] self.absent_ivorn = self.remaining_packet.attrib['ivorn']
def test_simple_tarball_dump(named_temporary_file): voevent_etrees = fake.heartbeat_packets() voevent_rowgen = (models.Voevent.from_etree(v) for v in voevent_etrees) fname = named_temporary_file.name filestore.write_tarball(voevent_rowgen, fname) tarball = tarfile.open(fname) assert len(tarball.getmembers()) == len(voevent_etrees)
def test_tarball_round_trip(named_temporary_file, fixture_db_session): voevent_etrees = fake.heartbeat_packets() # with open(assasn_non_ascii_packet_filepath, 'rb') as f: # voevent_etrees.append(vp.load(f)) s = fixture_db_session for etree in voevent_etrees: s.add(models.Voevent.from_etree(etree)) s.flush() voevent_dbrows = s.query(models.Voevent.ivorn, models.Voevent.xml).all() assert len(voevent_dbrows) == len(voevent_etrees) voevent_rowgen = list(models.Voevent.from_etree(v) for v in voevent_etrees) assert voevent_dbrows[0].ivorn == voevent_rowgen[0].ivorn assert voevent_dbrows[0].xml == voevent_rowgen[0].xml assert type(voevent_dbrows[0].xml) == type(voevent_rowgen[0].xml) assert type(voevent_rowgen[0].xml) == six.binary_type # Therefore it's crucial to test with an actual round-tripped dataset, # the 'voevent_dbrows' from above: fname = named_temporary_file.name filestore.write_tarball(voevent_dbrows, fname) loaded_voevents = [ vp.loads(s.xml) for s in filestore.tarfile_xml_generator(fname) ] def to_strings(voeventlist): return [vp.dumps(v) for v in voeventlist] def to_ivorn(voeventlist): return [v.attrib['ivorn'] for v in voeventlist] assert (to_ivorn(voevent_etrees) == to_ivorn(loaded_voevents)) assert (to_strings(voevent_etrees) == to_strings(loaded_voevents))
def test_ingest_packet(): #Must create corpusdb first: assert db_utils.check_database_exists(testdb_corpus_url) engine = sqlalchemy.create_engine(testdb_corpus_url) s = sqlalchemy.orm.Session(bind=engine) from voeventdb.server import __path__ as root_path root_path = root_path[0] script_path = os.path.join(root_path, 'bin', 'voeventdb_ingest_packet.py') print("Testing script at ", script_path) print("Using executable:", sys.executable) # Do stuff n_before = s.query(Voevent).count() proc = subprocess.Popen( [ script_path, '-d={}'.format(testdb_corpus_url.database), '-l={}'.format('/tmp/vdbingest-test.log'), ], stdin=subprocess.PIPE, ) proc.communicate(voeventparse.dumps( fake.heartbeat_packets(n_packets=1)[0])) proc.wait() assert proc.returncode == 0 assert s.query(Voevent).count() == n_before + 1
def test_unicode_voevent_tarball_dump(named_temporary_file): ## Now try some unicode characters voevent_etrees = fake.heartbeat_packets() vp.set_author(voevent_etrees[0], contactName=u"€€€€") voevent_rowgen = (models.Voevent.from_etree(v) for v in voevent_etrees) fname = named_temporary_file.name filestore.write_tarball(voevent_rowgen, fname) tarball = tarfile.open(fname) assert len(tarball.getmembers()) == len(voevent_etrees)
def insert_voevents(self, fixture_db_session): """ Insert two Vovents (mostly blank 'heartbeat' packet, GRB) as setup """ s = fixture_db_session assert len(s.query(Voevent).all()) == 0 # sanity check s.add(Voevent.from_etree(fake.heartbeat_packets()[0])) s.add(Voevent.from_etree(swift_bat_grb_655721)) s.flush() assert len(s.query(Voevent).all()) == 2 # 1 with, 1 without position
def test_tarball_round_trip(named_temporary_file, fixture_db_session): voevent_etrees = fake.heartbeat_packets() # with open(assasn_non_ascii_packet_filepath, 'rb') as f: # voevent_etrees.append(vp.load(f)) s = fixture_db_session for etree in voevent_etrees: s.add(models.Voevent.from_etree(etree)) s.flush() voevent_dbrows = s.query(models.Voevent.ivorn, models.Voevent.xml).all() assert len(voevent_dbrows) == len(voevent_etrees) voevent_rowgen = list(models.Voevent.from_etree(v) for v in voevent_etrees) assert voevent_dbrows[0].ivorn == voevent_rowgen[0].ivorn assert voevent_dbrows[0].xml == voevent_rowgen[0].xml # Here's the crux # A newly instantiated model will store a string type same as Python 2; # bytestring stores as bytestring, unicode as unicode. # However, after a round-trip to the database, proper typing has been # asserted and the bytestring is returned as unicode! assert type(voevent_dbrows[0].xml) != type(voevent_rowgen[0].xml) assert type(voevent_rowgen[0].xml) == str assert type(voevent_dbrows[0].xml) == unicode # Therefore it's crucial to test with an actual round-tripped dataset, # the 'voevent_dbrows' from above: fname = named_temporary_file.name filestore.write_tarball(voevent_dbrows, fname) loaded_voevents = [vp.loads(s.xml) for s in filestore.tarfile_xml_generator(fname)] def to_strings(voeventlist): return [vp.dumps(v) for v in voeventlist] def to_ivorn(voeventlist): return [v.attrib['ivorn'] for v in voeventlist] assert (to_ivorn(voevent_etrees) == to_ivorn(loaded_voevents)) assert (to_strings(voevent_etrees) == to_strings(loaded_voevents))
def test_ingest_packet(): # Must create corpusdb first: assert db_utils.check_database_exists(testdb_corpus_url) engine = sqlalchemy.create_engine(testdb_corpus_url) s = sqlalchemy.orm.Session(bind=engine) from voeventdb.server import __path__ as root_path root_path = root_path[0] script_path = os.path.join(root_path, "bin", "voeventdb_ingest_packet.py") print("Testing script at ", script_path) print("Using executable:", sys.executable) # Do stuff n_before = s.query(Voevent).count() proc = subprocess.Popen( [script_path, "-d={}".format(testdb_corpus_url.database), "-l={}".format("/tmp/vdbingest-test.log")], stdin=subprocess.PIPE, ) proc.communicate(voeventparse.dumps(fake.heartbeat_packets(n_packets=1)[0])) proc.wait() assert proc.returncode == 0 assert s.query(Voevent).count() == n_before + 1
def assign_test_client_and_initdb(self, flask_test_client, fixture_db_session): self.c = flask_test_client # Purely for brevity n_packets = 17 packets = heartbeat_packets(n_packets=n_packets) for counter, pkt in enumerate(packets, start=1): packet_dec = 180.0 / n_packets * counter -90 coords = vp.Position2D( ra=15, dec=packet_dec, err=0.1, units=vp.definitions.units.degrees, system=vp.definitions.sky_coord_system.utc_icrs_geo) # print "Inserting coords", coords vp.add_where_when( pkt, coords=coords, obs_time=iso8601.parse_date(pkt.Who.Date.text), observatory_location=vp.definitions.observatory_location.geosurface ) self.packets = packets self.ivorn_dec_map = {} for pkt in self.packets: posn = vp.get_event_position(pkt) self.ivorn_dec_map[pkt.attrib['ivorn']] = posn.dec fixture_db_session.add(Voevent.from_etree(pkt))
def assign_test_client_and_initdb(self, flask_test_client, fixture_db_session): self.c = flask_test_client # Purely for brevity n_packets = 17 packets = heartbeat_packets(n_packets=n_packets) for counter, pkt in enumerate(packets, start=1): packet_dec = 180.0 / n_packets * counter - 90 coords = vp.Position2D( ra=15, dec=packet_dec, err=0.1, units=vp.definitions.units.degrees, system=vp.definitions.sky_coord_system.utc_icrs_geo) # print "Inserting coords", coords vp.add_where_when( pkt, coords=coords, obs_time=iso8601.parse_date(pkt.Who.Date.text), observatory_location=vp.definitions.observatory_location.geosurface ) self.packets = packets self.ivorn_dec_map = {} for pkt in self.packets: posn = vp.get_event_position(pkt) self.ivorn_dec_map[pkt.attrib['ivorn']] = posn.dec fixture_db_session.add(Voevent.from_etree(pkt))
def test_heartbeat(self): n_interval = 4 * 6 packets = fake.heartbeat_packets(self.start, self.interval, n_interval) self.assertEqual(n_interval, len(packets))
def test_heartbeat(self): n_interval = 4*6 packets = fake.heartbeat_packets(self.start, self.interval, n_interval) self.assertEqual(n_interval, len(packets))