Пример #1
0
def CrawlThruLyraData():
    swiftdict = ParseSwiftCat.parseswiftcat(swift_cat_path)
    rawpaths=[]
    ptel_dict={}
    error_paths=[]
    
    globstr = '/Volumes/BR2/Bloom/PAIRITEL-DATA/sem20???/Dir20??-???-??/'
    rawpaths = glob.glob(globstr)
    for path in rawpaths:
        try:
            ptel_dict = RawToDatabase(path,objtype='GRB',pteldict=ptel_dict,swiftcatdict=swiftdict)
        except:
            error_paths.append(path)
    
    globstr = '/Volumes/BR2/Bloom/PAIRITEL-DATA/sem20???/Dir20??-???-??/Raw/'
    rawpaths = glob.glob(globstr)
    for path in rawpaths:
        try:
            ptel_dict = RawToDatabase(path,objtype='GRB',pteldict=ptel_dict,swiftcatdict=swiftdict)
        except:
            error_paths.append(path)
    
    globstr = '/Volumes/BR2/Bloom/PAIRITEL-DATA/sem20???/Dir20??-???-??/raw/'
    rawpaths = glob.glob(globstr)
    for path in rawpaths:
        try:
            ptel_dict = RawToDatabase(path,objtype='GRB',pteldict=ptel_dict,swiftcatdict=swiftdict)
        except:
            error_paths.append(path)
    
    
    print 'error paths:', error_paths
    return ptel_dict
Пример #2
0
def CrawlThruLyraData():
    swiftdict = ParseSwiftCat.parseswiftcat(swift_cat_path)
    rawpaths = []
    ptel_dict = {}
    error_paths = []

    globstr = "/Volumes/BR2/Bloom/PAIRITEL-DATA/sem20???/Dir20??-???-??/"
    rawpaths = glob.glob(globstr)
    for path in rawpaths:
        try:
            ptel_dict = RawToDatabase(path, objtype="GRB", pteldict=ptel_dict, swiftcatdict=swiftdict)
        except:
            error_paths.append(path)

    globstr = "/Volumes/BR2/Bloom/PAIRITEL-DATA/sem20???/Dir20??-???-??/Raw/"
    rawpaths = glob.glob(globstr)
    for path in rawpaths:
        try:
            ptel_dict = RawToDatabase(path, objtype="GRB", pteldict=ptel_dict, swiftcatdict=swiftdict)
        except:
            error_paths.append(path)

    globstr = "/Volumes/BR2/Bloom/PAIRITEL-DATA/sem20???/Dir20??-???-??/raw/"
    rawpaths = glob.glob(globstr)
    for path in rawpaths:
        try:
            ptel_dict = RawToDatabase(path, objtype="GRB", pteldict=ptel_dict, swiftcatdict=swiftdict)
        except:
            error_paths.append(path)

    print "error paths:", error_paths
    return ptel_dict
Пример #3
0
def RawToDatabase(raw_path,objtype='GRB',pteldict={},swiftcatdict={}):
    '''Given a path to raw pairitel data and and object ID (could be '*'),
    load some basic information from the pairitel data and, if a swift trigger,
    attempt to load extra information about the trigger and put it in a database
    (for now, just a csv file)
    
    How to get all the raw data from lyra?  Maybe just search for all p0-0 files
    with GRB string; this still may be thousands of files..
    
    '''
    if not os.path.exists(swift_cat_path): print "WARNING: %s does not exist." % (swift_cat_path)
    # Feed it a raw data folder, grab a list of all the raw p0-0.fits files
    if swiftcatdict=={}:
        swiftcatdict = ParseSwiftCat.parseswiftcat(swift_cat_path)
    
#    pteldict = {}
    
    if not os.path.isdir(raw_path): sys.exit('Not a Directory. Exiting.')
    globstr = raw_path + 'r20*' + objtype + '*p0-0.fits'
    raw_list = glob.glob(globstr)
    print raw_list
    for filepath in raw_list:
        semester = ''
        burst_time_str = ''
        grb = ''
        time_delta_hours_str = ''
        comments = ''
        
        
#        filepath = raw_path + '/' + filename
        # Open the fits file
        hdulist = pyfits.open(filepath)
        # Get the primary header.  For raw ptel data, should only be 1 header.
        prihdr = hdulist[0].header
        # Get the observation ID, e.g. swift-123456, integral-3501
        target_id = prihdr['TRGTNAME']
        splittarget_id = target_id.split('-')
        # If it was able to be split into two, the format is mission-triggerid
        if len(splittarget_id) == 2:
            mission = splittarget_id[0]
            triggerid = splittarget_id[1]
        else:
            mission = 'Unknown'
            triggerid = 'Unknown'
        # GRB.10000.1
        object_id = prihdr['OBJECT']
        
        
        # Get the time of the first (p0-0) observation for a particular ID
        # '2006-09-29 08:45:31.824422'
        ptel_time = prihdr['STRT_CPU']
        # Split off the microseconds at the end since I don't know how to deal with them atm
        ptel_time_split = ptel_time.split('.')
        # The followin is the format in the ptel header with the microseconds stripped off
        fmt = '%Y-%m-%d %H:%M:%S'
        ptel_time_tuple=time.strptime(ptel_time_split[0],fmt)
        # Convert to seconds since the epoch (sse)
        ptel_time_sse = time.mktime(ptel_time_tuple)
        
        try:
            ra = prihdr['RA']
            dec = prihdr['DEC']
        except:
            ra = 0.00
            dec = 0.00
        
        
        if target_id not in pteldict:
            targdict = {target_id:{'mission':mission,'triggerid':triggerid,'obs':{object_id:{'scope_ra':ra,'scope_dec':dec,'filename':filepath,'first_obs_time_sse':ptel_time_sse,'first_obs_time':ptel_time_split[0]}}}}
            pteldict.update(targdict)
        else:
            # Check to see if there is already an object id with that name
            # if there is, make a new object id name
            if object_id not in pteldict[target_id]['obs']:
                pteldict[target_id]['obs'].update({object_id:{'scope_ra':ra,'scope_dec':dec,'filename':filepath,'first_obs_time_sse':ptel_time_sse,'first_obs_time':ptel_time_split[0]}})
            else:
                new_obj_id = str(object_id) + '_' + str(ptel_time_sse)
                pteldict[target_id]['obs'].update({new_obj_id:{'scope_ra':ra,'scope_dec':dec,'filename':filepath,'first_obs_time_sse':ptel_time_sse,'first_obs_time':ptel_time_split[0]}})
                
        # if 'ptel_time_sse' not in pteldict[target_id]:
        #     pteldict[target_id].update({'ptel_time_sse':pteldict[target_id]['obs'][object_id]['first_obs_time_sse']})
        #     pteldict[target_id].update({'ptel_time':pteldict[target_id]['obs'][object_id]['first_obs_time']})
        # else: # if new object observation time is less than the old recorded first time, then subtract
        #     if pteldict[target_id]['ptel_time_sse'] > pteldict[target_id]['obs'][object_id]['first_obs_time_sse']:
        #         pteldict[target_id]['ptel_time_sse'] = pteldict[target_id]['obs'][object_id]['first_obs_time_sse']
        #         pteldict[target_id]['ptel_time'] = pteldict[target_id]['obs'][object_id]['first_obs_time']
        
        # If mission == swift, grab info from published Swift Catalog
        if mission == 'swift' and swiftcatdict != {}:
            found_id = False
            # THE FOLLOWING IS A VERY INEFFICIENT LOOP.  But it should work in the interim.
            for grb_str,catdict in swiftcatdict.iteritems():
                # if the triggerids match, then grab the info
                if triggerid == catdict['triggerid_str']:
                    found_id = True
                    grb = grb_str
                    # YYMMDD
                    grb_ymd = grb_str[0:6]
                    # HH:MM:SS.??
                    burst_time = catdict['burst_time_str']
                    burst_time_split = burst_time.split('.')
                    # YYMMDDHH:MM:SS
                    burst_time_toparse = grb_ymd + burst_time_split[0]
                    bt_fmt = '%y%m%d%H:%M:%S'
                    burst_time_tuple = time.strptime(burst_time_toparse,bt_fmt)
                    # Convert to seconds since last epoch
                    burst_time_sse = time.mktime(burst_time_tuple)
                    burst_time_str = time.strftime(fmt,burst_time_tuple)
                    
                    # Get difference from PTEL time from Burst Time in seconds
                    time_delta = ptel_time_sse - burst_time_sse
                    time_delta_hours = time_delta/3600.0
                    time_delta_hours_str = str(time_delta_hours)
                    
                    # pteldict[target_id].update({'time_delta':time_delta_hours,'grb_time_sse':burst_time_sse,'grb_time':burst_time_str})
                    pteldict[target_id].update({'grb_time_sse':burst_time_sse,'grb_time':burst_time_str,'grb_name':grb})
            if found_id == False:
                print 'COULD NOT FIND ID %s in SWIFT CATALOG' % (triggerid)
                    
        else:
            print 'Cannot yet grab extra info for %s.' % (target_id)
        
        
        string_output = '%s,%s,%s,%s,%s,%s,%s,%s' % (semester,grb,object_id,target_id,ptel_time_split[0],burst_time_str,time_delta_hours_str,comments)
        print string_output
        hdulist.close()
        
        
        # loop through observations in pteldict to find the earliest one for time purposes
        for target,targdict in pteldict.iteritems():
            min_ptel_time = 2.0E10  # Reset value
            min_ptel_time_string = ''
            time_delta = 0.0
            for observation,obsdict in targdict['obs'].iteritems():
                if obsdict['first_obs_time_sse'] < min_ptel_time:
                    min_ptel_time = obsdict['first_obs_time_sse']
                    min_ptel_time_string = obsdict['first_obs_time']
                    # if we have the GRB time, calculate the time_delta in hours
                    if 'grb_time_sse' in targdict:
                        time_delta = (min_ptel_time - targdict['grb_time_sse'])/3600.0
                        targdict.update({'time_delta':time_delta})
                    targdict.update({'ptel_time_sse':min_ptel_time,'ptel_time':min_ptel_time_string})
        #TODO: If two object_ids have the same target_id, combine them.
    return pteldict
Пример #4
0
def RawToDatabase(raw_path, objtype="GRB", pteldict={}, swiftcatdict={}):
    """Given a path to raw pairitel data and and object ID (could be '*'),
    load some basic information from the pairitel data and, if a swift trigger,
    attempt to load extra information about the trigger and put it in a database
    (for now, just a csv file)
    
    How to get all the raw data from lyra?  Maybe just search for all p0-0 files
    with GRB string; this still may be thousands of files..
    
    """
    if not os.path.exists(swift_cat_path):
        print "WARNING: %s does not exist." % (swift_cat_path)
    # Feed it a raw data folder, grab a list of all the raw p0-0.fits files
    if swiftcatdict == {}:
        swiftcatdict = ParseSwiftCat.parseswiftcat(swift_cat_path)

    #    pteldict = {}

    if not os.path.isdir(raw_path):
        sys.exit("Not a Directory. Exiting.")
    globstr = raw_path + "r20*" + objtype + "*p0-0.fits"
    raw_list = glob.glob(globstr)
    print raw_list
    for filepath in raw_list:
        semester = ""
        burst_time_str = ""
        grb = ""
        time_delta_hours_str = ""
        comments = ""

        #        filepath = raw_path + '/' + filename
        # Open the fits file
        hdulist = pyfits.open(filepath)
        # Get the primary header.  For raw ptel data, should only be 1 header.
        prihdr = hdulist[0].header
        # Get the observation ID, e.g. swift-123456, integral-3501
        target_id = prihdr["TRGTNAME"]
        splittarget_id = target_id.split("-")
        # If it was able to be split into two, the format is mission-triggerid
        if len(splittarget_id) == 2:
            mission = splittarget_id[0]
            triggerid = splittarget_id[1]
        else:
            mission = "Unknown"
            triggerid = "Unknown"
        # GRB.10000.1
        object_id = prihdr["OBJECT"]

        # Get the time of the first (p0-0) observation for a particular ID
        # '2006-09-29 08:45:31.824422'
        ptel_time = prihdr["STRT_CPU"]
        # Split off the microseconds at the end since I don't know how to deal with them atm
        ptel_time_split = ptel_time.split(".")
        # The followin is the format in the ptel header with the microseconds stripped off
        fmt = "%Y-%m-%d %H:%M:%S"
        ptel_time_tuple = time.strptime(ptel_time_split[0], fmt)
        # Convert to seconds since the epoch (sse)
        ptel_time_sse = time.mktime(ptel_time_tuple)

        try:
            ra = prihdr["RA"]
            dec = prihdr["DEC"]
        except:
            ra = 0.00
            dec = 0.00

        if target_id not in pteldict:
            targdict = {
                target_id: {
                    "mission": mission,
                    "triggerid": triggerid,
                    "obs": {
                        object_id: {
                            "scope_ra": ra,
                            "scope_dec": dec,
                            "filename": filepath,
                            "first_obs_time_sse": ptel_time_sse,
                            "first_obs_time": ptel_time_split[0],
                        }
                    },
                }
            }
            pteldict.update(targdict)
        else:
            # Check to see if there is already an object id with that name
            # if there is, make a new object id name
            if object_id not in pteldict[target_id]["obs"]:
                pteldict[target_id]["obs"].update(
                    {
                        object_id: {
                            "scope_ra": ra,
                            "scope_dec": dec,
                            "filename": filepath,
                            "first_obs_time_sse": ptel_time_sse,
                            "first_obs_time": ptel_time_split[0],
                        }
                    }
                )
            else:
                new_obj_id = str(object_id) + "_" + str(ptel_time_sse)
                pteldict[target_id]["obs"].update(
                    {
                        new_obj_id: {
                            "scope_ra": ra,
                            "scope_dec": dec,
                            "filename": filepath,
                            "first_obs_time_sse": ptel_time_sse,
                            "first_obs_time": ptel_time_split[0],
                        }
                    }
                )

        # if 'ptel_time_sse' not in pteldict[target_id]:
        #     pteldict[target_id].update({'ptel_time_sse':pteldict[target_id]['obs'][object_id]['first_obs_time_sse']})
        #     pteldict[target_id].update({'ptel_time':pteldict[target_id]['obs'][object_id]['first_obs_time']})
        # else: # if new object observation time is less than the old recorded first time, then subtract
        #     if pteldict[target_id]['ptel_time_sse'] > pteldict[target_id]['obs'][object_id]['first_obs_time_sse']:
        #         pteldict[target_id]['ptel_time_sse'] = pteldict[target_id]['obs'][object_id]['first_obs_time_sse']
        #         pteldict[target_id]['ptel_time'] = pteldict[target_id]['obs'][object_id]['first_obs_time']

        # If mission == swift, grab info from published Swift Catalog
        if mission == "swift" and swiftcatdict != {}:
            found_id = False
            # THE FOLLOWING IS A VERY INEFFICIENT LOOP.  But it should work in the interim.
            for grb_str, catdict in swiftcatdict.iteritems():
                # if the triggerids match, then grab the info
                if triggerid == catdict["triggerid_str"]:
                    found_id = True
                    grb = grb_str
                    # YYMMDD
                    grb_ymd = grb_str[0:6]
                    # HH:MM:SS.??
                    burst_time = catdict["burst_time_str"]
                    burst_time_split = burst_time.split(".")
                    # YYMMDDHH:MM:SS
                    burst_time_toparse = grb_ymd + burst_time_split[0]
                    bt_fmt = "%y%m%d%H:%M:%S"
                    burst_time_tuple = time.strptime(burst_time_toparse, bt_fmt)
                    # Convert to seconds since last epoch
                    burst_time_sse = time.mktime(burst_time_tuple)
                    burst_time_str = time.strftime(fmt, burst_time_tuple)

                    # Get difference from PTEL time from Burst Time in seconds
                    time_delta = ptel_time_sse - burst_time_sse
                    time_delta_hours = time_delta / 3600.0
                    time_delta_hours_str = str(time_delta_hours)

                    # pteldict[target_id].update({'time_delta':time_delta_hours,'grb_time_sse':burst_time_sse,'grb_time':burst_time_str})
                    pteldict[target_id].update(
                        {"grb_time_sse": burst_time_sse, "grb_time": burst_time_str, "grb_name": grb}
                    )
            if found_id == False:
                print "COULD NOT FIND ID %s in SWIFT CATALOG" % (triggerid)

        else:
            print "Cannot yet grab extra info for %s." % (target_id)

        string_output = "%s,%s,%s,%s,%s,%s,%s,%s" % (
            semester,
            grb,
            object_id,
            target_id,
            ptel_time_split[0],
            burst_time_str,
            time_delta_hours_str,
            comments,
        )
        print string_output
        hdulist.close()

        # loop through observations in pteldict to find the earliest one for time purposes
        for target, targdict in pteldict.iteritems():
            min_ptel_time = 2.0e10  # Reset value
            min_ptel_time_string = ""
            time_delta = 0.0
            for observation, obsdict in targdict["obs"].iteritems():
                if obsdict["first_obs_time_sse"] < min_ptel_time:
                    min_ptel_time = obsdict["first_obs_time_sse"]
                    min_ptel_time_string = obsdict["first_obs_time"]
                    # if we have the GRB time, calculate the time_delta in hours
                    if "grb_time_sse" in targdict:
                        time_delta = (min_ptel_time - targdict["grb_time_sse"]) / 3600.0
                        targdict.update({"time_delta": time_delta})
                    targdict.update({"ptel_time_sse": min_ptel_time, "ptel_time": min_ptel_time_string})
        # TODO: If two object_ids have the same target_id, combine them.
    return pteldict