def create_processmap():
return siplib.ProcessMap(
strategyname="strategy1",
strategydescription="awesome strategy",
starttime="1980-03-23T10:20:15",
duration="P6Y3M10DT15H",
identifier=proc_id,
observation_identifier=obs_id,
parset_identifier=siplib.Identifier("test"),
relations=[
siplib.ProcessRelation(identifier=siplib.Identifier("test"), ),
siplib.ProcessRelation(identifier=siplib.Identifier("test"), )
])
def create_basicdoc():
return siplib.Sip(
project_code="code",
project_primaryinvestigator="pi",
project_contactauthor="coauthor",
# project_telescope="LOFAR",
project_description="awesome project",
project_coinvestigators=["sidekick1", "sidekick2"],
dataproduct=siplib.SimpleDataProduct(
siplib.DataProductMap(type="Unknown",
identifier=siplib.Identifier("test"),
size=1024,
filename="/home/paulus/test.h5",
fileformat="HDF5",
process_identifier=siplib.Identifier("test"),
checksum_md5="hash1",
checksum_adler32="hash2",
storageticket="ticket")))
def test_parset(self):
mysip = create_basicdoc()
print("===\nAdding parset:\n")
print(
mysip.add_parset(identifier=siplib.Identifier("test"),
contents="blabla"))
mysip.save_to_file(TMPFILE_PATH)
self.assertTrue(validator.validate(TMPFILE_PATH))
def test_dataproducts(self):
mysip = create_basicdoc()
print("===\nAdding related generic dataproduct:\n")
with open(FEEDBACK_PATH) as f:
text = f.readlines()
fb = feedback.Feedback(text)
pipe_label = siplib.Identifier('test')
dataproducts = fb.get_dataproducts(prefix="test.prefix",
process_identifier=pipe_label)
for dp in dataproducts:
print("...adding:", dp)
mysip.add_related_dataproduct(dp)
mysip.save_to_file(TMPFILE_PATH)
self.assertTrue(validator.validate(TMPFILE_PATH))
# query sips for each dataproduct id
input_sips = []
for input_dpid in input_dpids:
xml = query.get_dataproduct_sip(input_project, input_dpid)
dpsip = siplib.Sip.from_xml(xml)
input_sips.append(dpsip)
# obtain important metadata required for SIP creation form the input SIPs:
indpids = [x.get_dataproduct_identifier() for x in input_sips] # we have the ids as string already, but we obtain the actual identifier objects here
sapid = input_sips[0].get_dataproduct_subarraypointing_identifier() # determine the SAP identifier. Note: This may or may not stay the same depending on what you did to the data!
# create identifier, can be looked up later
pipelabel = 'myprocesslabel_'+str(uuid.uuid4())
dplabel = 'mydataproductlabel_'+str(uuid.uuid4())
siplib.Identifier('test', userlabel=pipelabel)
siplib.Identifier('test', userlabel=dplabel)
mysip = siplib.Sip(
project_code=input_project,
project_primaryinvestigator="pi",
project_contactauthor="coauthor",
#project_telescope="LOFAR",
project_description="awesome project",
project_coinvestigators=["sidekick1", "sidekick2"],
dataproduct=siplib.CorrelatedDataProduct(
siplib.DataProductMap(
type="Unknown",
identifier=siplib.Identifier.lookup('test', userlabel=dplabel),
size=1024,
filename=output_dataproduct_name,
def test_observation(self):
mysip = create_basicdoc()
# add optional observation item
print("===\nAdding observation:\n")
print(
mysip.add_observation(
siplib.Observation(
observingmode="Interferometer",
instrumentfilter="10-70 MHz",
clock_frequency='160',
clock_frequencyunit="MHz",
stationselection="Core",
antennaset="HBA Zero",
timesystem="UTC",
stations=[
siplib.Station.preconfigured("RS106", ["LBA"]),
siplib.Station.preconfigured("DE609", ["HBA"])
],
numberofstations=5,
numberofsubarraypointings=5,
numberoftbbevents=5,
numberofcorrelateddataproducts=5,
numberofbeamformeddataproducts=5,
numberofbitspersample=5,
process_map=create_processmap(),
observationdescription="description",
channelwidth_frequency=160,
channelwidth_frequencyunit="MHz",
channelspersubband=5,
subarraypointings=[
siplib.SubArrayPointing(
pointing=siplib.PointingAltAz(
az_angle=20,
az_angleunit="degrees",
alt_angle=30,
alt_angleunit="degrees",
equinox="SUN"),
beamnumber=5,
identifier=point_id,
measurementtype="All Sky",
targetname="Sun",
starttime="1980-03-23T10:20:15",
duration="P6Y3M10DT15H",
numberofprocessing=1,
numberofcorrelateddataproducts=2,
numberofbeamformeddataproducts=1,
relations=[
siplib.ProcessRelation(
identifier=siplib.Identifier("test"))
],
correlatorprocessing=siplib.CorrelatorProcessing(
integrationinterval=0.5,
integrationinterval_unit="ns",
channelwidth_frequency=160,
channelwidth_frequencyunit="MHz"),
coherentstokesprocessing=siplib.
CoherentStokesProcessing(
rawsamplingtime=20,
rawsamplingtime_unit="ns",
timesamplingdownfactor=2,
samplingtime=10,
samplingtime_unit="ns",
stokes=["XX"],
numberofstations=1,
stations=[
siplib.Station.preconfigured(
"CS002", ["HBA0", "HBA1"])
],
frequencydownsamplingfactor=2,
numberofcollapsedchannels=2,
channelwidth_frequency=160,
channelwidth_frequencyunit="MHz",
channelspersubband=122),
incoherentstokesprocessing=siplib.
IncoherentStokesProcessing(
rawsamplingtime=20,
rawsamplingtime_unit="ns",
timesamplingdownfactor=2,
samplingtime=10,
samplingtime_unit="ns",
stokes=["XX"],
numberofstations=1,
stations=[
siplib.Station.preconfigured(
"CS003", ["HBA0", "HBA1"])
],
frequencydownsamplingfactor=2,
numberofcollapsedchannels=2,
channelwidth_frequency=160,
channelwidth_frequencyunit="MHz",
channelspersubband=122),
flyseyeprocessing=siplib.FlysEyeProcessing(
rawsamplingtime=10,
rawsamplingtime_unit="ms",
timesamplingdownfactor=2,
samplingtime=2,
samplingtime_unit="ms",
stokes=["I"],
),
nonstandardprocessing=siplib.NonStandardProcessing(
channelwidth_frequency=160,
channelwidth_frequencyunit="MHz",
channelspersubband=122))
],
transientbufferboardevents=["event1", "event2"])))
mysip.save_to_file(TMPFILE_PATH)
self.assertTrue(validator.validate(TMPFILE_PATH))
def test_dataproducts(self):
mysip = create_basicdoc()
print("===\nAdding related generic dataproduct:\n")
# add optional dataproduct item
print(
mysip.add_related_dataproduct(
siplib.GenericDataProduct(create_dataproductmap())))
# add optional dataproduct item
print("===\nAdding related pulp summary dataproduct:\n")
print(
mysip.add_related_dataproduct(
siplib.PulpSummaryDataProduct(
create_dataproductmap(),
filecontent=["content_a", "content_b"],
datatype="CoherentStokes")))
# add optional dataproduct item
print("===\nAdding related pulp dataproduct:\n")
print(
mysip.add_related_dataproduct(
siplib.PulpDataProduct(
create_dataproductmap(),
filecontent=["content_a", "content_b"],
datatype="CoherentStokes",
arraybeam=siplib.SimpleArrayBeam(
siplib.ArrayBeamMap(
subarraypointing_identifier=point_id,
beamnumber=4,
dispersionmeasure=16,
numberofsubbands=3,
stationsubbands=[1, 2, 3],
samplingtime=3,
samplingtimeunit="ms",
centralfrequencies="",
centralfrequencies_unit="MHz",
channelwidth_frequency=160,
channelwidth_frequencyunit="MHz",
channelspersubband=5,
stokes=["I", "Q"])))))
# add optional dataproduct item
print("===\nAdding related beamformed dataproduct:\n")
print(
mysip.add_related_dataproduct(
siplib.BeamFormedDataProduct(
create_dataproductmap(),
beams=[
siplib.FlysEyeBeam(
arraybeam_map=siplib.ArrayBeamMap(
subarraypointing_identifier=point_id,
beamnumber=4,
dispersionmeasure=16,
numberofsubbands=3,
stationsubbands=[1, 2, 3],
samplingtime=3,
samplingtimeunit="ms",
centralfrequencies="",
centralfrequencies_unit="MHz",
channelwidth_frequency=160,
channelwidth_frequencyunit="MHz",
channelspersubband=5,
stokes=["I", "Q"]),
station=siplib.Station.preconfigured(
"CS001", ["HBA0", "HBA1"]))
])))
# add optional dataproduct item
print("===\nAdding related sky image dataproduct:\n")
print(
mysip.add_related_dataproduct(
siplib.SkyImageDataProduct(
create_dataproductmap(),
numberofaxes=2,
coordinates=[
siplib.SpectralCoordinate(quantity_type="Frequency",
quantity_value=20.0,
axis=siplib.LinearAxis(
number=5,
name="bla",
units="parsec",
length=5,
increment=5,
referencepixel=7.5,
referencevalue=7.4)),
siplib.SpectralCoordinate(quantity_type="Frequency",
quantity_value=20.0,
axis=siplib.TabularAxis(
number=5,
name="bla",
units="parsec",
length=5,
)),
siplib.DirectionCoordinate(
linearaxis_a=siplib.LinearAxis(number=5,
name="bla",
units="parsec",
length=5,
increment=5,
referencepixel=7.5,
referencevalue=7.4),
linearaxis_b=siplib.LinearAxis(number=5,
name="bla",
units="parsec",
length=5,
increment=5,
referencepixel=7.5,
referencevalue=7.4),
pc0_0=0.0,
pc0_1=0.1,
pc1_0=1.0,
pc1_1=1.1,
equinox="SUN",
radecsystem="ICRS",
projection="rear",
projectionparameters=[1.0, 1.0, 1.0],
longitudepole_angle=1.0,
longitudepole_angleunit="degrees",
latitudepole_angle=2.0,
latitudepole_angleunit="degrees",
),
siplib.PolarizationCoordinate(
tabularaxis=siplib.TabularAxis(
number=5,
name="bla",
units="parsec",
length=5,
),
polarizations=["I", "YY", "XX", "Q"]),
siplib.TimeCoordinate(
equinox="SUN",
axis=siplib.TabularAxis(
number=5,
name="timetabular",
units="parsec",
length=5,
),
)
],
locationframe="GEOCENTER",
timeframe="timeframe",
observationpointing=siplib.PointingRaDec(
ra_angle=1.0,
ra_angleunit="degrees",
dec_angle=42.0,
dec_angleunit="degrees",
equinox="SUN"),
restoringbeammajor_angle=1.0,
restoringbeammajor_angleunit="degrees",
restoringbeamminor_angle=2.0,
restoringbeamminor_angleunit="degrees",
rmsnoise=1.0)))
# add optional dataproduct item
print("===\nAdded related correlated dataproduct:\n")
print(
mysip.add_related_dataproduct(
siplib.CorrelatedDataProduct(
create_dataproductmap(),
subarraypointing_identifier=siplib.Identifier("test"),
subband="1",
starttime="1980-03-23T10:20:15",
duration="P6Y3M10DT15H",
integrationinterval=10,
integrationintervalunit="ms",
central_frequency=160,
central_frequencyunit="MHz",
channelwidth_frequency=200,
channelwidth_frequencyunit="MHz",
channelspersubband=122,
stationsubband=2,
)))
# add optional dataproduct item
print("===\nAdding related pixelmap dataproduct:\n")
print(
mysip.add_related_dataproduct(
siplib.PixelMapDataProduct(create_dataproductmap(),
numberofaxes=5,
coordinates=[
siplib.SpectralCoordinate(
quantity_type="Frequency",
quantity_value=20.0,
axis=siplib.LinearAxis(
number=5,
name="bla",
units="parsec",
length=5,
increment=5,
referencepixel=7.5,
referencevalue=7.4))
])))
# add optional dataproduct item
print(
"===\nAdding related pixelmap dataproduct using predefined constants:\n"
)
print(
mysip.add_related_dataproduct(
siplib.SkyImageDataProduct(
create_dataproductmap(),
numberofaxes=2,
coordinates=[
siplib.SpectralCoordinate(
quantity_type=constants.
SPECTRALQUANTITYTYPE_VELOCITYAPPRADIAL,
quantity_value=20.0,
axis=siplib.LinearAxis(number=5,
name="bla",
units="unit",
length=5,
increment=5,
referencepixel=7.5,
referencevalue=7.4)),
siplib.DirectionCoordinate(
linearaxis_a=siplib.LinearAxis(number=5,
name="bla",
units="unit",
length=5,
increment=5,
referencepixel=7.5,
referencevalue=7.4),
linearaxis_b=siplib.LinearAxis(number=5,
name="blb",
units="unit",
length=5,
increment=5,
referencepixel=7.5,
referencevalue=7.4),
pc0_0=0.0,
pc0_1=0.1,
pc1_0=1.0,
pc1_1=1.1,
equinox=constants.EQUINOXTYPE_JUPITER,
radecsystem=constants.RADECSYSTEM_FK4_NO_E,
projection="rear",
projectionparameters=[1.0, 1.0, 1.0],
longitudepole_angle=1.0,
longitudepole_angleunit=constants.
ANGLEUNIT_RADIANS,
latitudepole_angle=2.0,
latitudepole_angleunit=constants.ANGLEUNIT_ARCSEC,
),
siplib.PolarizationCoordinate(
tabularaxis=siplib.TabularAxis(
number=5,
name="bla",
units="someunit",
length=5,
),
polarizations=[
constants.POLARIZATIONTYPE_LR,
constants.POLARIZATIONTYPE_XRE
]),
],
locationframe=constants.LOCATIONFRAME_LOCAL_GROUP,
timeframe="timeframe",
observationpointing=siplib.PointingRaDec(
ra_angle=1.0,
ra_angleunit=constants.ANGLEUNIT_DEGREES,
dec_angle=42.0,
dec_angleunit=constants.ANGLEUNIT_DEGREES,
equinox=constants.EQUINOXTYPE_B1950),
restoringbeammajor_angle=1.0,
restoringbeammajor_angleunit=constants.ANGLEUNIT_DEGREES,
restoringbeamminor_angle=2.0,
restoringbeamminor_angleunit=constants.ANGLEUNIT_DEGREES,
rmsnoise=1.0)))
mysip.save_to_file(TMPFILE_PATH)
self.assertTrue(validator.validate(TMPFILE_PATH))
print(str(e))
print(
'Please install python3 package pyxb: sudo apt-get install python3-pyxb'
)
exit(3) # special lofar test exit code: skipped test
from lofar.lta.sip import siplib
from lofar.lta.sip import validator
from lofar.lta.sip import constants
import os
# d = os.path.dirname(os.path.realpath(__file__))
TMPFILE_PATH = "/tmp/test_siplib.xml"
RELATEDSIP = '/tmp/sipfrommom.xml' # todo: how to deploy in testdir?
dp_id = siplib.Identifier("test")
in_dpid1 = siplib.Identifier("test")
in_dpid2 = siplib.Identifier("test")
proc_id = siplib.Identifier("test")
pipe_id = siplib.Identifier("test")
obs_id = siplib.Identifier("test")
point_id = siplib.Identifier("test")
def create_basicdoc():
return siplib.Sip(
project_code="code",
project_primaryinvestigator="pi",
project_contactauthor="coauthor",
# project_telescope="LOFAR",
project_description="awesome project",