def make_data_package_files(self,
trigger_name,
ra=0.0,
dec=0.0,
trigger_time=None,
destination_dir='.'):
"""
Make data package for gtburst
:return:
"""
if trigger_time is None:
trigger_time = self._tstart
destination_dir = sanitize_filename(destination_dir)
with within_directory(destination_dir, create=True):
# Rename ft1 and ft2
new_ft1 = 'gll_ft1_tr_bn%s_v00.fit' % trigger_name.replace(
"bn", "")
new_ft2 = 'gll_ft2_tr_bn%s_v00.fit' % trigger_name.replace(
"bn", "")
shutil.copy(self._simulated_ft1, new_ft1)
shutil.copy(self._ft2, new_ft2)
_makeDatasetsOutOfLATdata(new_ft1,
new_ft2,
trigger_name,
self._tstart,
self._tstart + self._simulation_time,
ra,
dec,
trigger_time,
localRepository=".",
cspecstart=0,
cspecstop=10)
# Remove all other files
self._cleanup()
os.remove(self._simulated_ft1)
self._simulated_ft1 = None
def getFTP(self,what='Extended'):
#Re-implementing this
#This will complete automatically the form available at
#https://fermi.gsfc.nasa.gov/cgi-bin/ssc/LAT/LATDataQuery.cgi
#After submitting the form, an html page will inform about
#the identifier assigned to the query and the time which will be
#needed to process it. After retrieving the query number,
#this function will wait for the files to be completed on the server,
#then it will download them
url = "https://fermi.gsfc.nasa.gov/cgi-bin/ssc/LAT/LATDataQuery.cgi"
#Save parameters for the query in a dictionary
parameters = {}
parameters['coordfield'] = "%s,%s" %(self.ra,self.dec)
parameters['coordsystem'] = "J2000"
parameters['shapefield'] = "%s" %(self.roi)
parameters['timefield'] = "%s,%s" %(self.tstart,self.tstop)
parameters['timetype'] = "%s" %(self.timetype)
parameters['energyfield'] = "30,1000000"
parameters['photonOrExtendedOrNone'] = what
parameters['destination'] = 'query'
parameters['spacecraft'] = 'checked'
print("Query parameters:")
for k,v in parameters.iteritems():
print("%30s = %s" %(k,v))
#POST encoding
postData = urllib.urlencode(parameters)
temporaryFileName = "__temp_query_result.html"
try:
os.remove(temporaryFileName)
except:
pass
pass
urllib.urlcleanup()
try:
urllib.urlretrieve(url,
temporaryFileName,
lambda x,y,z:0, postData)
except socket.timeout:
raise GtBurstException(11,"Time out when connecting to the server. Check your internet connection, or that you can access https://fermi.gsfc.nasa.gov, then retry")
except:
raise GtBurstException(1,"Problems with the download. Check your connection or that you can access https://fermi.gsfc.nasa.gov, then retry.")
pass
#Now open the file, parse it and get the query ID
htmlFile = open(temporaryFileName)
lines = []
for line in htmlFile:
lines.append(line.encode('utf-8'))
pass
html = " ".join(lines).strip()
htmlFile.close()
print("\nAnswer from the LAT data server:\n")
text = html2text.html2text(html.encode('utf-8').strip()).split("\n")
if("".join(text).replace(" ","")==""):
raise GtBurstException(1,"Problems with the download. Empty answer from the LAT server. Normally this means that the server is ingesting new data, please retry in half an hour or so.")
text = filter(lambda x:x.find("[") < 0 and
x.find("]") < 0 and
x.find("#") < 0 and
x.find("* ") < 0 and
x.find("+") < 0 and
x.find("Skip navigation")<0,text)
text = filter(lambda x:len(x.replace(" ",""))>1,text)
print "\n".join(text)
print("\n\n")
os.remove(temporaryFileName)
if(" ".join(text).find("down due to maintenance")>=0):
raise GtBurstException(12,"LAT Data server looks down due to maintenance.")
parser = DivParser("sec-wrapper")
parser.feed(html)
if(parser.data==[]):
parser = DivParser("right-side")
parser.feed(html)
pass
try:
estimatedTimeLine = filter(lambda x:x.find("The estimated time for your query to complete is")==0,parser.data)[0]
estimatedTimeForTheQuery = re.findall("The estimated time for your query to complete is ([0-9]+) seconds",estimatedTimeLine)[0]
except:
raise GtBurstException(1,"Problems with the download. Empty or wrong answer from the LAT server (see console). Please retry later.")
pass
try:
httpAddress = filter(lambda x:x.find("http://fermi.gsfc.nasa.gov") >=0,parser.data)[0]
except IndexError:
# Try https
httpAddress = filter(lambda x:x.find("https://fermi.gsfc.nasa.gov") >=0,parser.data)[0]
#Now periodically check if the query is complete
startTime = time.time()
timeout = 1.5*max(5.0,float(estimatedTimeForTheQuery)) #Seconds
refreshTime = 2.0 #Seconds
#When the query will be completed, the page will contain this string:
#The state of your query is 2 (Query complete)
endString = "The state of your query is 2 (Query complete)"
#Url regular expression
regexpr = re.compile("wget (.*.fits)")
#Build the window for the progress
if(self.parent is None):
#No graphical output
root = None
else:
#make a transient window
root = Toplevel()
root.transient(self.parent)
root.grab_set()
l = Label(root,text='Waiting for the server to complete the query (estimated time: %s seconds)...' %(estimatedTimeForTheQuery))
l.grid(row=0,column=0)
m1 = Meter(root, 500,20,'grey','blue',0,None,None,'white',relief='ridge', bd=3)
m1.grid(row=1,column=0)
m1.set(0.0,'Waiting...')
pass
links = None
fakeName = "__temp__query__result.html"
while(time.time() <= startTime+timeout):
if(root is not None):
if(estimatedTimeForTheQuery==0):
m1.set(1)
else:
m1.set((time.time()-startTime)/float(max(estimatedTimeForTheQuery,1)))
sys.stdout.flush()
#Fetch the html with the results
try:
(filename, header) = urllib.urlretrieve(httpAddress,fakeName)
except socket.timeout:
urllib.urlcleanup()
if(root is not None):
root.destroy()
raise GtBurstException(11,"Time out when connecting to the server. Check your internet connection, or that you can access https://fermi.gsfc.nasa.gov, then retry")
except:
urllib.urlcleanup()
if(root is not None):
root.destroy()
raise GtBurstException(1,"Problems with the download. Check your connection or that you can access https://fermi.gsfc.nasa.gov, then retry.")
pass
f = open(fakeName)
html = " ".join(f.readlines())
status = re.findall("The state of your query is ([0-9]+)",html)[0]
#print("Status = %s" % status)
if(status=='2'):
#Get the download link
links = regexpr.findall(html)
break
f.close()
os.remove(fakeName)
urllib.urlcleanup()
time.sleep(refreshTime)
pass
if(root is not None):
root.destroy()
#Download the files
#if(links is not None):
# for link in links:
# print("Downloading %s..." %(link))
# urllib.urlretrieve(link,link.split("/")[-1])
# pass
#else:
# raise RuntimeError("Could not download LAT Standard data")
#pass
remotePath = "%s/%s/queries/" %(self.dataRepository,self.instrument)
if(links is not None):
filenames = map(lambda x:x.split('/')[-1],links)
try:
self.downloadDirectoryWithFTP(remotePath,filenames=filenames)
except Exception as e:
#Try with "wget", if the system has it
for ff in filenames:
try:
self.makeLocalDir()
dataHandling.runShellCommand("wget %s%s -P %s" %("https://fermi.gsfc.nasa.gov/FTP/fermi/data/lat/queries/",ff,self.localRepository),True)
except:
raise e
pass
pass
pass
else:
raise GtBurstException(1,"Could not download LAT Standard data")
pass
#Rename the files to something neater...
newFilenames = {}
for f in filenames:
#EV or SC?
suffix = f.split("_")[1]
if(suffix.find("EV")>=0):
suffix = 'ft1'
elif(suffix.find("SC")>=0):
suffix = 'ft2'
else:
raise GtBurstException(13,"Could not understand the type of a downloaded file (%s)" %(f))
newfilename = os.path.join(self.localRepository,"gll_%s_tr_bn%s_v00.fit" %(suffix,self.grbName))
localPath = os.path.join(self.localRepository,f)
os.rename(localPath,newfilename)
newFilenames[suffix] = newfilename
pass
###########################
if('ft1' in newFilenames.keys() and 'ft2' in newFilenames.keys()):
dataHandling._makeDatasetsOutOfLATdata(newFilenames['ft1'],newFilenames['ft2'],
self.grbName,self.tstart,self.tstop,
self.ra,self.dec,self.triggerTime,
self.localRepository,
cspecstart=-1000,
cspecstop=1000)
def run(**kwargs):
if(len(kwargs.keys())==0):
#Nothing specified, the user needs just help!
thisCommand.getHelp()
return
#Get parameters values
thisCommand.setParValuesFromDictionary(kwargs)
try:
ft2file = thisCommand.getParValue('ft2file')
infile = thisCommand.getParValue('xmlsimmodel')
simeventfile = thisCommand.getParValue('simeventfile')
srclist = thisCommand.getParValue('srclist')
tstart = thisCommand.getParValue('tstart')
tstop = thisCommand.getParValue('tstop')
triggertime = thisCommand.getParValue('triggertime')
irf = thisCommand.getParValue('irf')
seed = thisCommand.getParValue('seed')
figure = thisCommand.getParValue('figure')
outdir = thisCommand.getParValue('outdir')
clobber = _yesOrNoToBool(thisCommand.getParValue('clobber'))
verbose = _yesOrNoToBool(thisCommand.getParValue('verbose'))
except KeyError as err:
print("\n\nERROR: Parameter %s not found or incorrect! \n\n" %(err.args[0]))
print thisCommand.getHelp()
return
from GtBurst import dataHandling
irf_ = dataHandling._getParamFromXML(infile,'IRF')
ra = dataHandling._getParamFromXML(infile,'RA')
dec = dataHandling._getParamFromXML(infile,'DEC')
name = dataHandling._getParamFromXML(infile,'OBJECT')
if(irf_ is None and irf=='None'):
raise RuntimeError("Could not read IRF from XML file, and you did not specify it with the 'irf' command line parameter")
if(irf==''):
irf = irf_
pass
simtime = float(tstop)-float(tstart)
LATSimulation = dataHandling.Simulation(ft2file,irf,triggertime)
outfile, idsfile = LATSimulation.doSimulation(infile=infile,
srclist=srclist,
evroot ='__temp',
simtime=float(simtime),
tstart =float(tstart),
seed =int(seed))
os.rename(outfile,simeventfile)
############################
#This happens only if the input XML has been generated by the tool in GtBurst
if(irf_ is not None and ra is not None and dec is not None and name is not None):
try:
#If the simeventfile follows the convention gll_ft1_tr_[triggerName]_v[version].fit
#then produce a ft2 file which follows that too
rootname = simeventfile.split("_")[3]
version = simeventfile.split("_")[4]
newft2 = 'gll_ft2_tr_%s_%s.fit' %(rootname,version)
except:
#Just add a 'sim' before the extension
rootname = "%ssim" % name
atoms = os.path.basename(ft2file).split('.')
newft2 = "%ssim.%s" %(atoms[0],atoms[1])
pass
shutil.copyfile(ft2file,newft2)
ft1,rsp,ft2,cspec = dataHandling._makeDatasetsOutOfLATdata(simeventfile,newft2,rootname,
tstart,tstop,ra,dec,triggertime,'.')
if(figure is not None):
tempfile = '__temp__skymap_sim.fit'
gtdocountsmap.gtdocountsmap(eventfile=ft1,rspfile=rsp,ft2file=ft2,ra=ra,dec=dec,rad=40,
irf='transient',zmax=180,tstart=tstart,tstop=tstop,emin=10,
emax=1e9,skybinsize=0.2,skymap=tempfile,figure=figure)
os.remove(tempfile)
pass
pass
#Now move them in the output directory
if(outdir!='.'):
destdir = os.path.abspath(os.path.expanduser(outdir))
try:
os.makedirs(destdir)
except os.error:
print("Note: directory %s already exists" %(outdir))
for orig,dest in zip([ft1,ft2,rsp,cspec],map(lambda x:os.path.join(destdir,x),[ft1,ft2,rsp,cspec])):
if(os.path.exists(dest)):
if(clobber==True):
os.remove(dest)
else:
raise RuntimeError("File %s already exists and clobber=no" %(dest))
pass
pass
shutil.move(orig,dest)
pass
pass
return 'simeventfile', simeventfile
if __name__ == "__main__":
if len(sys.argv) < 7:
print("Usage: %s [ft1] [ft2] [triggertime] [triggername] [ra] [dec]" %
sys.argv[0])
sys.exit(0)
ft1 = sys.argv[1]
ft2 = sys.argv[2]
triggertime = float(sys.argv[3])
triggername = sys.argv[4]
ra = sys.argv[5]
dec = sys.argv[6]
# Rename ft1 and ft2
new_ft1 = 'gll_ft1_tr_bn%s_v00.fit' % triggername
new_ft2 = 'gll_ft2_tr_bn%s_v00.fit' % triggername
shutil.copy(ft1, new_ft1)
shutil.copy(ft2, new_ft2)
# Get start and stop from ft1
tstart = pyfits.getval(new_ft1, 'TSTART', 'EVENTS')
tstop = pyfits.getval(new_ft1, 'TSTOP', 'EVENTS')
_makeDatasetsOutOfLATdata(new_ft1, new_ft2, triggername, tstart, tstop, ra,
dec, triggertime, '.', triggertime,
triggertime + 1000)
def run(**kwargs):
if (len(kwargs.keys()) == 0):
#Nothing specified, the user needs just help!
thisCommand.getHelp()
return
#Get parameters values
thisCommand.setParValuesFromDictionary(kwargs)
try:
ft2file = thisCommand.getParValue('ft2file')
infile = thisCommand.getParValue('xmlsimmodel')
simeventfile = thisCommand.getParValue('simeventfile')
srclist = thisCommand.getParValue('srclist')
tstart = thisCommand.getParValue('tstart')
tstop = thisCommand.getParValue('tstop')
triggertime = thisCommand.getParValue('triggertime')
irf = thisCommand.getParValue('irf')
seed = thisCommand.getParValue('seed')
figure = thisCommand.getParValue('figure')
outdir = thisCommand.getParValue('outdir')
clobber = _yesOrNoToBool(thisCommand.getParValue('clobber'))
verbose = _yesOrNoToBool(thisCommand.getParValue('verbose'))
except KeyError as err:
print("\n\nERROR: Parameter %s not found or incorrect! \n\n" %
(err.args[0]))
print thisCommand.getHelp()
return
from GtBurst import dataHandling
irf_ = dataHandling._getParamFromXML(infile, 'IRF')
ra = dataHandling._getParamFromXML(infile, 'RA')
dec = dataHandling._getParamFromXML(infile, 'DEC')
name = dataHandling._getParamFromXML(infile, 'OBJECT')
if (irf_ is None and irf == 'None'):
raise RuntimeError(
"Could not read IRF from XML file, and you did not specify it with the 'irf' command line parameter"
)
if (irf == ''):
irf = irf_
pass
simtime = float(tstop) - float(tstart)
LATSimulation = dataHandling.Simulation(ft2file, irf, triggertime)
outfile, idsfile = LATSimulation.doSimulation(infile=infile,
srclist=srclist,
evroot='__temp',
simtime=float(simtime),
tstart=float(tstart),
seed=int(seed))
os.rename(outfile, simeventfile)
############################
#This happens only if the input XML has been generated by the tool in GtBurst
if (irf_ is not None and ra is not None and dec is not None
and name is not None):
try:
#If the simeventfile follows the convention gll_ft1_tr_[triggerName]_v[version].fit
#then produce a ft2 file which follows that too
rootname = simeventfile.split("_")[3]
version = simeventfile.split("_")[4]
newft2 = 'gll_ft2_tr_%s_%s.fit' % (rootname, version)
except:
#Just add a 'sim' before the extension
rootname = "%ssim" % name
atoms = os.path.basename(ft2file).split('.')
newft2 = "%ssim.%s" % (atoms[0], atoms[1])
pass
shutil.copyfile(ft2file, newft2)
ft1, rsp, ft2, cspec = dataHandling._makeDatasetsOutOfLATdata(
simeventfile, newft2, rootname, tstart, tstop, ra, dec,
triggertime, '.')
if (figure is not None):
tempfile = '__temp__skymap_sim.fit'
gtdocountsmap.gtdocountsmap(eventfile=ft1,
rspfile=rsp,
ft2file=ft2,
ra=ra,
dec=dec,
rad=40,
irf='transient',
zmax=180,
tstart=tstart,
tstop=tstop,
emin=10,
emax=1e9,
skybinsize=0.2,
skymap=tempfile,
figure=figure)
os.remove(tempfile)
pass
pass
#Now move them in the output directory
if (outdir != '.'):
destdir = os.path.abspath(os.path.expanduser(outdir))
try:
os.makedirs(destdir)
except os.error:
print("Note: directory %s already exists" % (outdir))
for orig, dest in zip([ft1, ft2, rsp, cspec],
map(lambda x: os.path.join(destdir, x),
[ft1, ft2, rsp, cspec])):
if (os.path.exists(dest)):
if (clobber == True):
os.remove(dest)
else:
raise RuntimeError(
"File %s already exists and clobber=no" % (dest))
pass
pass
shutil.move(orig, dest)
pass
pass
return 'simeventfile', simeventfile
ft1 = sys.argv[1]
ft2 = sys.argv[2]
triggertime = float(sys.argv[3])
triggername = sys.argv[4]
ra = sys.argv[5]
dec = sys.argv[6]
# Rename ft1 and ft2
new_ft1 = 'gll_ft1_tr_bn%s_v00.fit' % triggername
new_ft2 = 'gll_ft2_tr_bn%s_v00.fit' % triggername
shutil.copy(ft1, new_ft1)
shutil.copy(ft2, new_ft2)
# Get start and stop from ft1
tstart = pyfits.getval(new_ft1,'TSTART','EVENTS')
tstop = pyfits.getval(new_ft1,'TSTOP','EVENTS')
_makeDatasetsOutOfLATdata(new_ft1,
new_ft2,
triggername,
tstart,
tstop,
ra,
dec,
triggertime,
'.',
triggertime,
triggertime+1000)