def __init__(self, fixture_db_session):
s = fixture_db_session
packets = [swift_bat_grb_655721]
packets.extend(fake.heartbeat_packets(role=vp.definitions.roles.test))
extra_packets = fake.heartbeat_packets(
start=fake.default_start_dt + timedelta(hours=24),
role=vp.definitions.roles.utility)
# Packets referenced by other packets:
# self.cited = set()
# Count times ivorn referenced by other packets in db :
self.cite_counts = defaultdict(int)
# Packets containing at least one cite entry
self.followup_packets = []
# Total number of citations (one packet may have multiple cite entries)
self.n_citations = 0
c0 = packets[0].attrib['ivorn']
c1 = packets[1].attrib['ivorn']
#One reference in ep0
self.add_reference(extra_packets[0], c0)
#Two references in ep1
self.add_reference(extra_packets[1], c0)
self.add_reference(extra_packets[1], c1)
#
# #Now cite ep[0], making it both cites / cited_by
c2 = extra_packets[0].attrib['ivorn']
self.add_reference(extra_packets[2], c2)
# Add a citation to an external packet
self.add_reference(extra_packets[3],
swift_xrt_grb_655721.attrib['ivorn'])
packets.extend(extra_packets)
self.packet_dict = {pkt.attrib['ivorn']: pkt for pkt in packets}
self.insert_packets = packets[:-1]
self.insert_packets_dumps = [vp.dumps(v) for v in self.insert_packets]
self.streams = [
v.attrib['ivorn'].split('#')[0][6:] for v in self.insert_packets
]
self.stream_set = list(set(self.streams))
self.roles = [v.attrib['role'] for v in self.insert_packets]
self.role_set = list(set(self.roles))
self.remaining_packet = packets[-1]
# Insert all but the last packet, this gives us a useful counter-example
s.add_all((Voevent.from_etree(p) for p in self.insert_packets))
self.n_inserts = len(self.insert_packets)
self.inserted_ivorns = [p.attrib['ivorn'] for p in self.insert_packets]
self.absent_ivorn = self.remaining_packet.attrib['ivorn']
def __init__(self, fixture_db_session):
s = fixture_db_session
packets = [swift_bat_grb_655721]
packets.extend(fake.heartbeat_packets(role=vp.definitions.roles.test))
extra_packets = fake.heartbeat_packets(
start=fake.default_start_dt + timedelta(hours=24),
role=vp.definitions.roles.utility)
# Packets referenced by other packets:
# self.cited = set()
# Count times ivorn referenced by other packets in db :
self.cite_counts = defaultdict(int)
# Packets containing at least one cite entry
self.followup_packets = []
# Total number of citations (one packet may have multiple cite entries)
self.n_citations = 0
c0 = packets[0].attrib['ivorn']
c1 = packets[1].attrib['ivorn']
#One reference in ep0
self.add_reference(extra_packets[0], c0)
#Two references in ep1
self.add_reference(extra_packets[1],c0)
self.add_reference(extra_packets[1],c1)
#
# #Now cite ep[0], making it both cites / cited_by
c2 = extra_packets[0].attrib['ivorn']
self.add_reference(extra_packets[2],c2)
# Add a citation to an external packet
self.add_reference(extra_packets[3], swift_xrt_grb_655721.attrib['ivorn'])
packets.extend(extra_packets)
self.packet_dict = { pkt.attrib['ivorn'] : pkt for pkt in packets }
self.insert_packets = packets[:-1]
self.insert_packets_dumps = [vp.dumps(v) for v in self.insert_packets]
self.streams = [v.attrib['ivorn'].split('#')[0][6:]
for v in self.insert_packets]
self.stream_set = list(set(self.streams))
self.roles = [v.attrib['role'] for v in self.insert_packets]
self.role_set = list(set(self.roles))
self.remaining_packet = packets[-1]
# Insert all but the last packet, this gives us a useful counter-example
s.add_all(
(Voevent.from_etree(p) for p in self.insert_packets))
self.n_inserts = len(self.insert_packets)
self.inserted_ivorns = [p.attrib['ivorn'] for p in self.insert_packets]
self.absent_ivorn = self.remaining_packet.attrib['ivorn']
def test_simple_tarball_dump(named_temporary_file):
voevent_etrees = fake.heartbeat_packets()
voevent_rowgen = (models.Voevent.from_etree(v) for v in voevent_etrees)
fname = named_temporary_file.name
filestore.write_tarball(voevent_rowgen, fname)
tarball = tarfile.open(fname)
assert len(tarball.getmembers()) == len(voevent_etrees)
def test_tarball_round_trip(named_temporary_file, fixture_db_session):
voevent_etrees = fake.heartbeat_packets()
# with open(assasn_non_ascii_packet_filepath, 'rb') as f:
# voevent_etrees.append(vp.load(f))
s = fixture_db_session
for etree in voevent_etrees:
s.add(models.Voevent.from_etree(etree))
s.flush()
voevent_dbrows = s.query(models.Voevent.ivorn, models.Voevent.xml).all()
assert len(voevent_dbrows) == len(voevent_etrees)
voevent_rowgen = list(models.Voevent.from_etree(v) for v in voevent_etrees)
assert voevent_dbrows[0].ivorn == voevent_rowgen[0].ivorn
assert voevent_dbrows[0].xml == voevent_rowgen[0].xml
assert type(voevent_dbrows[0].xml) == type(voevent_rowgen[0].xml)
assert type(voevent_rowgen[0].xml) == six.binary_type
# Therefore it's crucial to test with an actual round-tripped dataset,
# the 'voevent_dbrows' from above:
fname = named_temporary_file.name
filestore.write_tarball(voevent_dbrows, fname)
loaded_voevents = [
vp.loads(s.xml) for s in filestore.tarfile_xml_generator(fname)
]
def to_strings(voeventlist):
return [vp.dumps(v) for v in voeventlist]
def to_ivorn(voeventlist):
return [v.attrib['ivorn'] for v in voeventlist]
assert (to_ivorn(voevent_etrees) == to_ivorn(loaded_voevents))
assert (to_strings(voevent_etrees) == to_strings(loaded_voevents))
def test_simple_tarball_dump(named_temporary_file):
voevent_etrees = fake.heartbeat_packets()
voevent_rowgen = (models.Voevent.from_etree(v) for v in voevent_etrees)
fname = named_temporary_file.name
filestore.write_tarball(voevent_rowgen, fname)
tarball = tarfile.open(fname)
assert len(tarball.getmembers()) == len(voevent_etrees)
def test_ingest_packet():
#Must create corpusdb first:
assert db_utils.check_database_exists(testdb_corpus_url)
engine = sqlalchemy.create_engine(testdb_corpus_url)
s = sqlalchemy.orm.Session(bind=engine)
from voeventdb.server import __path__ as root_path
root_path = root_path[0]
script_path = os.path.join(root_path, 'bin', 'voeventdb_ingest_packet.py')
print("Testing script at ", script_path)
print("Using executable:", sys.executable)
# Do stuff
n_before = s.query(Voevent).count()
proc = subprocess.Popen(
[
script_path,
'-d={}'.format(testdb_corpus_url.database),
'-l={}'.format('/tmp/vdbingest-test.log'),
],
stdin=subprocess.PIPE,
)
proc.communicate(voeventparse.dumps(
fake.heartbeat_packets(n_packets=1)[0]))
proc.wait()
assert proc.returncode == 0
assert s.query(Voevent).count() == n_before + 1
def test_unicode_voevent_tarball_dump(named_temporary_file):
## Now try some unicode characters
voevent_etrees = fake.heartbeat_packets()
vp.set_author(voevent_etrees[0], contactName=u"€€€€")
voevent_rowgen = (models.Voevent.from_etree(v) for v in voevent_etrees)
fname = named_temporary_file.name
filestore.write_tarball(voevent_rowgen, fname)
tarball = tarfile.open(fname)
assert len(tarball.getmembers()) == len(voevent_etrees)
def test_unicode_voevent_tarball_dump(named_temporary_file):
## Now try some unicode characters
voevent_etrees = fake.heartbeat_packets()
vp.set_author(voevent_etrees[0], contactName=u"€€€€")
voevent_rowgen = (models.Voevent.from_etree(v) for v in voevent_etrees)
fname = named_temporary_file.name
filestore.write_tarball(voevent_rowgen, fname)
tarball = tarfile.open(fname)
assert len(tarball.getmembers()) == len(voevent_etrees)
def insert_voevents(self, fixture_db_session):
"""
Insert two Vovents (mostly blank 'heartbeat' packet, GRB) as setup
"""
s = fixture_db_session
assert len(s.query(Voevent).all()) == 0 # sanity check
s.add(Voevent.from_etree(fake.heartbeat_packets()[0]))
s.add(Voevent.from_etree(swift_bat_grb_655721))
s.flush()
assert len(s.query(Voevent).all()) == 2 # 1 with, 1 without position
def insert_voevents(self, fixture_db_session):
"""
Insert two Vovents (mostly blank 'heartbeat' packet, GRB) as setup
"""
s = fixture_db_session
assert len(s.query(Voevent).all()) == 0 # sanity check
s.add(Voevent.from_etree(fake.heartbeat_packets()[0]))
s.add(Voevent.from_etree(swift_bat_grb_655721))
s.flush()
assert len(s.query(Voevent).all()) == 2 # 1 with, 1 without position
def test_tarball_round_trip(named_temporary_file, fixture_db_session):
voevent_etrees = fake.heartbeat_packets()
# with open(assasn_non_ascii_packet_filepath, 'rb') as f:
# voevent_etrees.append(vp.load(f))
s = fixture_db_session
for etree in voevent_etrees:
s.add(models.Voevent.from_etree(etree))
s.flush()
voevent_dbrows = s.query(models.Voevent.ivorn, models.Voevent.xml).all()
assert len(voevent_dbrows) == len(voevent_etrees)
voevent_rowgen = list(models.Voevent.from_etree(v) for v in voevent_etrees)
assert voevent_dbrows[0].ivorn == voevent_rowgen[0].ivorn
assert voevent_dbrows[0].xml == voevent_rowgen[0].xml
# Here's the crux
# A newly instantiated model will store a string type same as Python 2;
# bytestring stores as bytestring, unicode as unicode.
# However, after a round-trip to the database, proper typing has been
# asserted and the bytestring is returned as unicode!
assert type(voevent_dbrows[0].xml) != type(voevent_rowgen[0].xml)
assert type(voevent_rowgen[0].xml) == str
assert type(voevent_dbrows[0].xml) == unicode
# Therefore it's crucial to test with an actual round-tripped dataset,
# the 'voevent_dbrows' from above:
fname = named_temporary_file.name
filestore.write_tarball(voevent_dbrows, fname)
loaded_voevents = [vp.loads(s.xml) for s in
filestore.tarfile_xml_generator(fname)]
def to_strings(voeventlist):
return [vp.dumps(v) for v in voeventlist]
def to_ivorn(voeventlist):
return [v.attrib['ivorn'] for v in voeventlist]
assert (to_ivorn(voevent_etrees) == to_ivorn(loaded_voevents))
assert (to_strings(voevent_etrees) == to_strings(loaded_voevents))
def test_ingest_packet():
# Must create corpusdb first:
assert db_utils.check_database_exists(testdb_corpus_url)
engine = sqlalchemy.create_engine(testdb_corpus_url)
s = sqlalchemy.orm.Session(bind=engine)
from voeventdb.server import __path__ as root_path
root_path = root_path[0]
script_path = os.path.join(root_path, "bin", "voeventdb_ingest_packet.py")
print("Testing script at ", script_path)
print("Using executable:", sys.executable)
# Do stuff
n_before = s.query(Voevent).count()
proc = subprocess.Popen(
[script_path, "-d={}".format(testdb_corpus_url.database), "-l={}".format("/tmp/vdbingest-test.log")],
stdin=subprocess.PIPE,
)
proc.communicate(voeventparse.dumps(fake.heartbeat_packets(n_packets=1)[0]))
proc.wait()
assert proc.returncode == 0
assert s.query(Voevent).count() == n_before + 1
def assign_test_client_and_initdb(self, flask_test_client,
fixture_db_session):
self.c = flask_test_client # Purely for brevity
n_packets = 17
packets = heartbeat_packets(n_packets=n_packets)
for counter, pkt in enumerate(packets, start=1):
packet_dec = 180.0 / n_packets * counter -90
coords = vp.Position2D(
ra=15, dec=packet_dec, err=0.1,
units=vp.definitions.units.degrees,
system=vp.definitions.sky_coord_system.utc_icrs_geo)
# print "Inserting coords", coords
vp.add_where_when(
pkt,
coords=coords,
obs_time=iso8601.parse_date(pkt.Who.Date.text),
observatory_location=vp.definitions.observatory_location.geosurface
)
self.packets = packets
self.ivorn_dec_map = {}
for pkt in self.packets:
posn = vp.get_event_position(pkt)
self.ivorn_dec_map[pkt.attrib['ivorn']] = posn.dec
fixture_db_session.add(Voevent.from_etree(pkt))
def assign_test_client_and_initdb(self, flask_test_client,
fixture_db_session):
self.c = flask_test_client # Purely for brevity
n_packets = 17
packets = heartbeat_packets(n_packets=n_packets)
for counter, pkt in enumerate(packets, start=1):
packet_dec = 180.0 / n_packets * counter - 90
coords = vp.Position2D(
ra=15, dec=packet_dec, err=0.1,
units=vp.definitions.units.degrees,
system=vp.definitions.sky_coord_system.utc_icrs_geo)
# print "Inserting coords", coords
vp.add_where_when(
pkt,
coords=coords,
obs_time=iso8601.parse_date(pkt.Who.Date.text),
observatory_location=vp.definitions.observatory_location.geosurface
)
self.packets = packets
self.ivorn_dec_map = {}
for pkt in self.packets:
posn = vp.get_event_position(pkt)
self.ivorn_dec_map[pkt.attrib['ivorn']] = posn.dec
fixture_db_session.add(Voevent.from_etree(pkt))
def test_heartbeat(self):
n_interval = 4 * 6
packets = fake.heartbeat_packets(self.start, self.interval, n_interval)
self.assertEqual(n_interval, len(packets))
def test_heartbeat(self):
n_interval = 4*6
packets = fake.heartbeat_packets(self.start, self.interval,
n_interval)
self.assertEqual(n_interval, len(packets))