def ztestFullGenieParticulateSamples(self):
"""
test Genie Particulate samples
"""
request="spectrum=CURR/QC/BK, analysis=CURR"
# get full 2003-10-24 to 2003-10-26
listOfSamplesToTest = self.getListOfSampleIDs('2007-01-04',endDate='2007-01-15',spectralQualif='FULL',nbOfElem='5')
# error
# these two are working
#listOfSamplesToTest = [ "916900","917873" ]
#transform in numbers and retransform in str to remove the 0 at the beginning of the number"
#intifiedlist = map(int,listOfSamplesToTest)
#listOfSamplesToTest = map(str,intifiedlist)
TestSAMPMLCreator.c_log.info("list samples: %s"%listOfSamplesToTest)
cpt = 0
total_t0 = time.time()
for sampleID in listOfSamplesToTest:
TestSAMPMLCreator.c_log.info("\n********************************************************************************\n Start Test %d for SampleID %s.\n********************************************************************************\n"%(cpt,sampleID))
t0 = time.time()
# fetchnoble particulate
fetcher = DBDataFetcher.getDataFetcher(self.mainConn,self.archConn,sampleID)
fetcher.fetch(request,'PAR')
renderer = GenieParticulateRenderer(fetcher)
xmlStr = renderer.asXmlStr(request)
path = "/tmp/samples/sampml-full-%s.xml"%(sampleID)
ctbto.common.xml_utils.pretty_print_xml(StringIO.StringIO(xmlStr),path)
# check if no tags are left
self.assertIfNoTagsLeft(path)
self.assertAllCalibrationInfo(path)
t1 = time.time()
TestSAMPMLCreator.c_log.info("\n********************************************************************************\n End of Test %d for SampleID %s. Test executed in %s seconds.\n********************************************************************************\n"%(cpt,sampleID,(t1-t0)))
cpt +=1
total_t1 = time.time()
TestSAMPMLCreator.c_log.info("\n****************************************************************************\n****************************************************************************\n****** EXECUTED %d FULL SAMPLE RETRIEVALS in %s seconds ********\n****************************************************************************\n****************************************************************************\n"%(cpt,total_t1-total_t0))
def ztestGetOneParticulateSampleAndDoBitChecking(self):
"""
get a unique particulate sample and do a bit checking against a registered existing sample
"""
request="spectrum=CURR, analysis=CURR"
cpt = 0
total_t0 = time.time()
#listOfSamplesToTest = self.getListOfSampleIDs('2008-12-24',endDate='2008-12-25',spectralQualif='FULL',nbOfElem='1')
#sampleID = listOfSamplesToTest[0]
sampleID = 967273
# fetchnoble particulate
fetcher = DBDataFetcher.getDataFetcher(self.mainConn,self.archConn,sampleID)
fetcher.fetch(request,'PAR')
renderer = GenieParticulateRenderer(fetcher)
xmlStr = renderer.asXmlStr(request)
path = "/tmp/samples/sampml-full-%s.xml"%(sampleID)
ctbto.common.xml_utils.pretty_print_xml(StringIO.StringIO(xmlStr),path)
# check if no tags are left
self.assertIfNoTagsLeft(path)
self.assertAllCalibrationInfo(path)
self.assertFileContentEquals("%s/samples/sampml-full-%s.xml.master"%(self._get_tests_dir_path(),sampleID),path)
cpt +=1
total_t1 = time.time()
TestSAMPMLCreator.c_log.info("\n****************************************************************************\n****************************************************************************\n****** EXECUTED %d FULL SAMPLE RETRIEVALS in %s seconds ********\n****************************************************************************\n****************************************************************************\n"%(cpt,total_t1-total_t0))
def execute(self, a_args):
if a_args == None or a_args == {}:
raise Exception("No commands passed. See usage message.")
self._log.info("*************************************************************")
self._log.info("Configuration infos read from %s" % (self._conf.get_conf_file_path()))
self._log.info("*************************************************************\n")
cache_cleaned = False
local_spectra_cleaned = False
# check if we need to clean the cache
if a_args["clean_cache"]:
self._clean_cache()
cache_cleaned = True
if a_args["clean_local_spectra"]:
self._clean_cached_spectrum()
local_spectra_cleaned = True
# check if we can write in case the dir already exists
dir = a_args["dir"]
self._create_results_directories(dir)
# default request => do not retrieve PREL but all the rest
request = "spectrum=CURR/DETBK/GASBK/QC, analysis=CURR"
# check if we have some sids or we get it from some dates
self._log.info("*************************************************************")
if "sids" in a_args:
sids = a_args["sids"]
elif "from" in a_args or "end" in a_args or "stations" in a_args:
begin = a_args.get(
"from", ctbto.common.time_utils.getOracleDateFromISO8601(ctbto.common.time_utils.getYesterday())
)
end = a_args.get(
"end", ctbto.common.time_utils.getOracleDateFromISO8601(ctbto.common.time_utils.getToday())
)
stations = a_args.get("stations", None)
if stations != None:
stations = self._get_stations_ids(stations)
else:
stations = self._get_all_stations(a_args["station_types"])
sids = self._get_list_of_sampleIDs(stations, begin, end)
else:
# if the cache has been clean then exit quietly as one action has been performed
if cache_cleaned or local_spectra_cleaned:
return
else:
# no actions performed error
raise Exception("need either a sid or some dates or a station name")
self._log.info("Start the product generation")
self._log.info("*************************************************************\n")
to_ignore = self._conf.getlist("IgnoreSamples", "noblegazSamples")
always_recreate_files = a_args["always_recreate_files"]
for sid in sids:
if sid in to_ignore:
self._log.info("*************************************************************")
self._log.info("Ignore the retrieval of the sample id %s as it is incomplete." % (sid))
self._log.info("*************************************************************\n")
# skip this iteration
continue
self._log.info("*************************************************************")
self._log.info("Fetch data and build SAMPML data file for %s" % (sid))
# if the right flag is set and the file already exists do not recreate it
if always_recreate_files or not os.path.exists("%s/ARR/ARR-%s.html" % (dir, sid)):
# fetch noble gaz or particulate
fetcher = DBDataFetcher.getDataFetcher(self._ngMainConn, self._ngArchConn, sid)
# prevent from trying to generate particulate samples
if isinstance(fetcher, ctbto.db.datafetchers.ParticulateDataFetcher):
raise Exception("Generation for Particulate ARR files not supported")
# modify remoteHost
fetcher.setRemoteHost(self._conf.get("RemoteAccess", "nobleGazRemoteHost", "dls007"))
fetcher.fetch(request, "GAS")
renderer = BaseRenderer.getRenderer(fetcher)
xmlStr = renderer.asXmlStr(request)
station_code = fetcher.get("STATION_CODE")
path = "%s/samples/sampml-full-%s-%s.xml" % (dir, station_code, sid)
self._log.info("Save SAMPML data in %s" % (path))
# pretty print and save in file
ctbto.common.xml_utils.pretty_print_xml(StringIO.StringIO(xmlStr), path)
# create ARR if possible
self._create_arr(fetcher, dir, path, sid, station_code)
else:
self._log.info("products are already existing in %s for %s" % (dir, sid))
self._log.info("*************************************************************\n")
def testGenerateNobleGasARR(self):
"""
Generate a Noble Gaz ARR.
"""
request="spectrum=CURR/DETBK/GASBK/QC, analysis=CURR"
# get full
listOfSamplesToTest = self.getListOfSaunaSampleIDs('2007-11-25',endDate='2008-11-26',spectralQualif='FULL',nbOfElem='5')
# remove sampleID for which data isn't available
# 206975: No Calibration Available
toRemove = [141372,206975]
for id in toRemove:
if id in listOfSamplesToTest:
listOfSamplesToTest.remove(id)
listOfSamplesToTest = [310317]
TestSAMPMLCreator.c_log.info("list samples %s"%listOfSamplesToTest)
cpt = 1
total_t0 = time.time()
for sampleID in listOfSamplesToTest:
TestSAMPMLCreator.c_log.info("Start Test %d for SampleID %s.\n"%(cpt,sampleID))
t0 = time.time()
# fetch noble gaz or particulate
fetcher = DBDataFetcher.getDataFetcher(self.nbConn,self.archConn,sampleID)
#modify remoteHost
fetcher.setRemoteHost(self.conf.get('RemoteAccess','nobleGazRemoteHost','dls007'))
fetcher.fetch(request,'GAS')
renderer = SaunaRenderer(fetcher)
xmlStr = renderer.asXmlStr(request)
path = "/tmp/samples/sampml-full-%s.xml"%(sampleID)
ctbto.common.xml_utils.pretty_print_xml(StringIO.StringIO(xmlStr),path)
# check if no tags are left
self.assertIfNoTagsLeft(path)
self.assertAllCalibrationInfo(path)
t1 = time.time()
TestSAMPMLCreator.c_log.info("Fetch sample nb %d with SampleID %s.\nTest executed in %s seconds.\n\n**************************************************************** \n**************************************************************** \n"%(cpt,sampleID,(t1-t0)))
cpt +=1
r = XML2HTMLRenderer('%s/%s'%(self._get_tests_dir_path(),'templates'),'ArrHtml.html')
result = r.render(path)
utils.printInFile(result,"/tmp/ARR-%s.html"%(sampleID))
total_t1 = time.time()
TestSAMPMLCreator.c_log.info("\n****************************************************************************\n****************************************************************************\n****** EXECUTED %d FULL SAMPLE RETRIEVALS in %s seconds ********\n****************************************************************************\n****************************************************************************\n"%(cpt,total_t1-total_t0))
def ztestSpalaxFullNobleGazSamples(self):
"""
Get Full Noble Gaz samples.
"""
request="spectrum=ALL, analysis=CURR"
# get full
listOfSamplesToTest = self.getListOfSpalaxSampleIDs('2009-01-20',endDate='2009-02-01',spectralQualif='FULL',nbOfElem='2')
# remove sampleID for which data isn't available
#if "141372" in listOfSamplesToTest:
# listOfSamplesToTest.remove("141372")
#PREL 211385
listOfSamplesToTest = ['269892']
TestSAMPMLCreator.c_log.info("list samples :%s"%(listOfSamplesToTest))
cpt = 0
total_t0 = time.time()
for sampleID in listOfSamplesToTest:
TestSAMPMLCreator.c_log.info("Start Test %d for SampleID %s.\n"%(cpt,sampleID))
t0 = time.time()
# fetchnoble particulate
fetcher = DBDataFetcher.getDataFetcher(self.nbConn,self.archConn,sampleID)
#modify remoteHost
fetcher.setRemoteHost(self.conf.get('RemoteAccess','nobleGazRemoteHost','dls007'))
fetcher.fetch(request,'GAS')
renderer = SpalaxRenderer(fetcher)
xmlStr = renderer.asXmlStr(request)
#print "Non Formatted String [%s]\n"%(xmlStr)
#f = open("/tmp/xmlStr.xml","w")
#f.write(xmlStr)
#f.close()
path = "/tmp/samples/sampml-full-%s.xml"%(sampleID)
ctbto.common.xml_utils.pretty_print_xml(StringIO.StringIO(xmlStr),path)
# check if no tags are left
self.assertIfNoTagsLeft(path)
#self.assertAllCalibrationInfo(path)
t1 = time.time()
#TestSAMPMLCreator.c_log.info("End of Test %d for SampleID %s.\nTest executed in %s seconds.\n\n**************************************************************** \n**************************************************************** \n"%(cpt,sampleID,(t1-t0)))
TestSAMPMLCreator.c_log.info("\n********************************************************************************\n End of Test %d for SampleID %s. Test executed in %s seconds.\n********************************************************************************\n"%(cpt,sampleID,(t1-t0)))
cpt +=1
total_t1 = time.time()
TestSAMPMLCreator.c_log.info("\n****************************************************************************\n****************************************************************************\n****** EXECUTED %d FULL SAMPLE RETRIEVALS in %s seconds ********\n****************************************************************************\n****************************************************************************\n"%(cpt,total_t1-total_t0))