Beispiel #1
0
 def get(self, request, *args, **kwargs):
   jobid = self.kwargs['jobid']
   job = AsyncResult(jobid)
   completed = job.ready()
   db_entry = UserTask_photometry.objects.get(jobid=jobid)
   if job.failed():
     #We will let the client-side jquery/ajax handle the error that will come 
     #from trying to parse the results of a failed job.
     #We only care to delete our record of the job.
     db_entry.delete()
   lastline = db_entry.logfile_line_number
   with open(os.path.join(MEDIA_ROOT,jobid,'logfile'),'r') as f:
     lines = f.readlines()
   if not completed or lastline < len(lines):
       loglines = ''.join(lines[lastline:]).strip()
       loglines = loglines.replace('\n','<br />')
       db_entry.logfile_line_number = len(lines)
       db_entry.save()
       context = {'completed':False,'log':loglines}
   else:
     results = job.get()
     mag = round(results['APP'][2],2)
     mag_err = round(results['APP'][3],2)
     hashtable = {
           'MAG_APP': lambda d: round(d['APP'][2],2) if d['APP'] else None,
           'MAG_APP_ERR': lambda d: round(d['APP'][3],2) if d['APP'] else None,
           'MAG_PSF': lambda d: round(d['PSF'][2],2) if d['PSF'] else None,
           'MAG_PSF_ERR': lambda d: round(d['PSF'][3],2) if d['PSF'] else None,
           'CALIB_SCHEME': lambda d: d['CALIB_SCHEME'],
           'CALIB_FILE': lambda d: d['CALIB_FILE'],
     }
     fields = {}
     fields['user'] = request.user
     fields['BAND'] = db_entry.band
     astrosource = AstroSource.objects.get(sourceID=db_entry.sourceID)
     imageheader = ImageHeader.objects.filter(TARGETID=db_entry.targetID).filter(OB=db_entry.OB).filter(FILTER=db_entry.band)[0]
     fields['imageheader'] = imageheader
     fields['astrosource'] = astrosource
     fields.update( {'MAG_APP':hashtable['MAG_APP'](results) if 'APP' in results else None} )
     fields.update( {'MAG_APP_ERR':hashtable['MAG_APP_ERR'](results) if 'APP' in results else None} )
     fields.update( {'MAG_PSF':hashtable['MAG_PSF'](results) if 'PSF' in results else None} )
     fields.update( {'MAG_PSF_ERR':hashtable['MAG_PSF_ERR'](results) if 'PSF' in results else None} )
     fields.update( {'CALIB_SCHEME':hashtable['CALIB_SCHEME'](results)}  )
     fields.update( {'CALIB_FILE':hashtable['CALIB_FILE'](results)} )
     p = Photometry(**fields)
     p.save()
     db_entry.delete() #Delete database entry, this should eventually be tied to the redis backend!
     context = {'completed':True,'jobid':jobid,
                'PSF':hashtable['MAG_PSF'](results) if 'PSF' in results else None,
                'PSF_ERR':hashtable['MAG_PSF_ERR'](results) if 'PSF' in results else None,
                'APP':hashtable['MAG_APP'](results) if 'APP' in results else None,
                'APP_ERR':hashtable['MAG_APP_ERR'](results) if 'APP' in results else None,
                'band':db_entry.band,
                'OB':db_entry.OB,
                'targetID':db_entry.targetID}
   return self.render_to_response(context)
Beispiel #2
0
  def _make_Photometry(self):   
    for source in self.sources:
      matched = self.resultfile.getNearbyObjs(source.RA,source.DEC,limit=1)
      s = matched.keys()[0]
      if matched[s]['DISTANCE'] > self.args['match_tolerance']:
        continue
      fields = {}
      fields['user'] = self.user
      fields['BAND'] = self.resultfile.header['BAND']
      fields['imageheader'] = self.imageheader
      fields['astrosource'] = source

      [fields.update( {k:None} ) for k in ['MAG_PSF','MAG_PSF_ERR','MAG_APP','MAG_APP_ERR']] #Default to null photometry
      fields['CALIB_SCHEME'] = self.resultfile.header['CATALOG_CALIB']
      #Set up magnitudes from resultfiles      
      if fields['BAND'] in constants.infrared:
        #If JHK, take MAG_APP=MAG_CALIB, point to downloaded 2MASS catalog
        try:
          fields['MAG_APP'] = matched[s]['MAG_CALIB']+constants.convert_to_AB[fields['BAND']]
          fields['MAG_APP_ERR'] = matched[s]['MAG_CALIB_ERR']
        except TypeError:
          pass #Default None is already set
        fields['CALIB_FILE'] = os.path.join(os.path.dirname(self.imageheader.PATH),'GROND_%s_OB_cat_2MASS.tsv' % fields['BAND'])
      else:
        #if griz, take either CALIB is sdss/user catalog, else take zeropoints
        if fields['CALIB_SCHEME'].lower() in ['sdss','user']:
          fields['MAG_PSF'] = matched[s]['MAG_CALIB']
          fields['MAG_PSF_ERR'] = matched[s]['MAG_CALIB_ERR']
          try:
            fields['MAG_APP'] = matched[s]['MAG_APP']+matched[s]['OFFSET_ALL']
            fields['MAG_APP_ERR'] = ( matched[s]['MAG_APP_ERR']**2 + matched[s]['OFFSET_ALL_ERR']**2 ) ** (1/2.)
          except TypeError: #In case of value=INDEF from iraf
            pass #default None is already set
          if fields['CALIB_SCHEME'].lower() == 'sdss':
            fields['CALIB_FILE'] = os.path.join(os.path.dirname(self.imageheader.PATH),'GROND_%s_OB_cat_SDSS.tsv' % fields['BAND'])
          else:
            iniFile = os.path.join(DATADIR,self.imageheader.TARGETID,self.imageheader.OB,'%sana.ini' % fields['BAND'])
            cp = ConfigParser.ConfigParser()
            cp.read(iniFile)
            fields['CALIB_FILE'] = os.path.join(DATADIR,self.imageheader.TARGETID,self.imageheader.OB,'calcat_%s' % os.path.basename(cp.get('task','calcat')))  
        else:
          [fields.update( {k:matched[s][k]} ) for k in ['MAG_PSF','MAG_PSF_ERR','MAG_APP','MAG_APP_ERR']]
          fields['CALIB_SCHEME'] = 'ZP' #Overwrite 'USNO' calib scheme, as we never will report photometry against USNO
          fields['CALIB_FILE'] = None

      result = Photometry.objects.filter(astrosource__sourceID=source.sourceID).filter(imageheader__PATH=self.imageheader.PATH) 
      if result:
        fields['pk'] = result[0].pk
      self.photometry = Photometry(**fields)
      self.photometry.save()
Beispiel #3
0
class GrondData:
  def __repr__(self):
    return self.path

  def __init__(self,path,user,**kwargs):
    self.args = kwargs
    self.path = path
    self.user = user
    self.resultfile = None
    for f in os.listdir(path):
      if re.search(kwargs['fits_regex'],f):
        self.image = os.path.join(path,f)
      if re.search(kwargs['results_regex'],f): 
        self.resultfile = resultfile.ResultFile(os.path.join(path,f))
    if not self.resultfile:
      sys.exit("FATAL: No resultfiles found in [%s], yet the reduced image seems to exist." % self.path)
    hdulist = pyfits.open(self.image)
    self.header = hdulist[0].header

  def populateDB(self):
    self._make_ImageHeader()
    print "  --> ImageHeader done."
    self._make_ImageProperties()
    print "  --> ImageProperties done."
    if self.args['skip_photometry']:
      return
    self._make_AstroSource()
    print "  --> AstroSource done."
    self._make_Photometry()
    print "  --> Photometry done."

  def _make_ImageHeader(self):
    fields = {}
    def _match_db_fields(fields):
      #Match the FITS header keys with those defined in the database table
      modelfields = ('PATH','NAXIS1', 'NAXIS2', 'EXPTIME', 'MJD_OBS', 
                     'DATE_OBS', 'CRVAL1', 'CRVAL2', 'NGR', 'NINT', 'NIZ', 'NMD', 
                     'NTD', 'NTP', 'OBSEQNUM', 'OBSRUNID', 'TARGETID', 'FILTER', 'RON',
                     'GAIN', 'MJD_MID', 'OBSERR', 'NCOMBINE', 'NIMGS', 'TDP_MID', 
                     'INTERPSM', 'AIRMASS', 'IMGEXP','OBTYPEID')
      matched = dict( [(k,v) for k,v in fields.iteritems() if k in modelfields]  )
      return matched
    fields = dict([(k.replace('-','_'),self.header[k]) for k in self.header if k]) 
    fields = _match_db_fields(fields) #Need to remove the extraneous keys...Django blindly tries to copy all keys to models 
    fields['PATH'] = self.image
    fields['OBTYPEID'] = fields['OBTYPEID'].replace('min','m')
    fields['OB_CORR'] = self.header.get('OB_CORR','')
    wcs = astWCS.WCS(self.image)
    fields['RA'], fields['DEC'] = wcs.pix2wcs(self.header['NAXIS1']/2.0,self.header['NAXIS2']/2.0)
    fields['BOTTOMLEFT_RA'], fields['BOTTOMLEFT_DEC'] = wcs.pix2wcs(0,0)
    fields['TOPRIGHT_RA'], fields['TOPRIGHT_DEC'] = wcs.pix2wcs(self.header['NAXIS1'],self.header['NAXIS2'])
    try:
      result = ImageHeader.objects.get(
                                      TARGETID=fields['TARGETID'],
                                      OB_CORR=fields['OB_CORR'],
                                      OBSEQNUM=fields['OBSEQNUM'],
                                      OBSRUNID=fields['OBSRUNID'],
                                      FILTER=fields['FILTER'],
                                      )
      print "NOTE: This field already exists in the database. I will UPDATE the values for it instead of INSERT a new row"
      fields['pk'] = result.pk
    except ImageHeader.DoesNotExist:
      pass
    self.imageheader = ImageHeader(**fields)
    self.imageheader.save()

  def _make_ImageProperties(self):
    translation = {'LIMITING_MAG_3S_ZP':'LIMITING_MAG_3SIG_ZP',
                 'LIMITING_MAG_3S_CALIB':'LIMITING_MAG_3SIG_CALIB',
                 'SEEING':'MEAN_FWHM_ARCSEC',
                 'MEAN_AIRMASS':'MEAN_AIRMASS',
                 'ASTROMETRY_ACCURACY_RA':'ASTRO_SOLN_ACCURACY_RA',
                 'ASTROMETRY_ACCURACY_DEC':'ASTRO_SOLN_ACCURACY_DEC',
                 'KRON_CORRECTION':'KRON_CORRECTION',
                 'APP_SIZE':'APERTURE_SIZE_ARCSEC',
                 'CALIB_CHI2':'CALIB_CHI2',
                 'CALIB_RMS':'CALIB_RMS'}
    fields = {}      
    for k in translation:
      fields[k] = self.resultfile.header.get(translation[k],-99)
    fields['LIMITING_MAG_3S_ZP'] += constants.convert_to_AB[self.imageheader.FILTER]
    fields['LIMITING_MAG_3S_CALIB'] += constants.convert_to_AB[self.imageheader.FILTER]
    fields['imageheader'] = self.imageheader
    try:
      result = ImageHeader.objects.get(PATH=fields['imageheader'].PATH)
      fields['pk'] = result.pk
    except ImageHeader.DoesNotExist:
      pass
    self.imageproperties = ImageProperties(**fields)
    self.imageproperties.save()

  def _make_AstroSource(self):
    all_sources = self.resultfile.objects[:]
    print "     (detected %s sources in this resultfile)" % len(all_sources)
    #Compare the sources detected in this resultfile the database. If sourceID already exists, we shouldn't re-write it!
    self.old_sources = []
    sources_to_remove = []
    for source in all_sources:
      r = sorted(AstroSource.objects.positionFilter(source['RA'],source['DEC'],self.args['match_tolerance']),key=lambda k: k.distance)
      if r:
        #Even if this source already exists, this may be a new observation of it
        #Therefore, we need to check also the ImageHeader(s) of this source, and
        #Add this one in if it doesnt exist.
        this_source = r[0] 
        if self.imageheader not in this_source.imageheader.all():
          this_source.imageheader.add(self.imageheader)
          this_source.save()
        self.old_sources.append(this_source)
        sources_to_remove.append(source)
    all_sources = [i for i in all_sources if i not in sources_to_remove]
    print "     (after removal of sources already in the database, %s new sources remain)" % len(all_sources)

    #Finally, make the Django models and save to DB
    self.new_sources = []
    for source in all_sources:
      fields = {}
      fields['user'] = self.user
      fields['RA'] = source['RA']
      fields['DEC'] = source['DEC']
      sexRa,sexDec = deg2sex.main(source['RA'],source['DEC'])
      fields['sourceID'] = 'GROND_J%s%s' % (sexRa,sexDec)
      d = AstroSource(**fields)
      d.save()
      d.imageheader.add(self.imageheader)
      self.new_sources.append(d)
    [i.save() for i in self.new_sources]
    self.sources = []
    self.sources.extend(self.old_sources)
    self.sources.extend(self.new_sources)

  def _make_Photometry(self):   
    for source in self.sources:
      matched = self.resultfile.getNearbyObjs(source.RA,source.DEC,limit=1)
      s = matched.keys()[0]
      if matched[s]['DISTANCE'] > self.args['match_tolerance']:
        continue
      fields = {}
      fields['user'] = self.user
      fields['BAND'] = self.resultfile.header['BAND']
      fields['imageheader'] = self.imageheader
      fields['astrosource'] = source

      [fields.update( {k:None} ) for k in ['MAG_PSF','MAG_PSF_ERR','MAG_APP','MAG_APP_ERR']] #Default to null photometry
      fields['CALIB_SCHEME'] = self.resultfile.header['CATALOG_CALIB']
      #Set up magnitudes from resultfiles      
      if fields['BAND'] in constants.infrared:
        #If JHK, take MAG_APP=MAG_CALIB, point to downloaded 2MASS catalog
        try:
          fields['MAG_APP'] = matched[s]['MAG_CALIB']+constants.convert_to_AB[fields['BAND']]
          fields['MAG_APP_ERR'] = matched[s]['MAG_CALIB_ERR']
        except TypeError:
          pass #Default None is already set
        fields['CALIB_FILE'] = os.path.join(os.path.dirname(self.imageheader.PATH),'GROND_%s_OB_cat_2MASS.tsv' % fields['BAND'])
      else:
        #if griz, take either CALIB is sdss/user catalog, else take zeropoints
        if fields['CALIB_SCHEME'].lower() in ['sdss','user']:
          fields['MAG_PSF'] = matched[s]['MAG_CALIB']
          fields['MAG_PSF_ERR'] = matched[s]['MAG_CALIB_ERR']
          try:
            fields['MAG_APP'] = matched[s]['MAG_APP']+matched[s]['OFFSET_ALL']
            fields['MAG_APP_ERR'] = ( matched[s]['MAG_APP_ERR']**2 + matched[s]['OFFSET_ALL_ERR']**2 ) ** (1/2.)
          except TypeError: #In case of value=INDEF from iraf
            pass #default None is already set
          if fields['CALIB_SCHEME'].lower() == 'sdss':
            fields['CALIB_FILE'] = os.path.join(os.path.dirname(self.imageheader.PATH),'GROND_%s_OB_cat_SDSS.tsv' % fields['BAND'])
          else:
            iniFile = os.path.join(DATADIR,self.imageheader.TARGETID,self.imageheader.OB,'%sana.ini' % fields['BAND'])
            cp = ConfigParser.ConfigParser()
            cp.read(iniFile)
            fields['CALIB_FILE'] = os.path.join(DATADIR,self.imageheader.TARGETID,self.imageheader.OB,'calcat_%s' % os.path.basename(cp.get('task','calcat')))  
        else:
          [fields.update( {k:matched[s][k]} ) for k in ['MAG_PSF','MAG_PSF_ERR','MAG_APP','MAG_APP_ERR']]
          fields['CALIB_SCHEME'] = 'ZP' #Overwrite 'USNO' calib scheme, as we never will report photometry against USNO
          fields['CALIB_FILE'] = None

      result = Photometry.objects.filter(astrosource__sourceID=source.sourceID).filter(imageheader__PATH=self.imageheader.PATH) 
      if result:
        fields['pk'] = result[0].pk
      self.photometry = Photometry(**fields)
      self.photometry.save()