def Ok(self,widget):
'''When the Add Cast Dialog's OK button is hit'''
Casts = KirbyBase()
treeCastList = self.pyCast.get_widget("podcastlist")
treeSelection = treeCastList.get_selection()
(model, iter) = treeSelection.get_selected()
selected = model.get_value(iter,0)
print selected
Casts.delete(CASTDB,['name'],['^'+selected+'$'])
UpdateCastDisplay(self.pyCast)
self.wRemoveCast.destroy()
def __init__(self,db_dir):
# open objects for our dbs and locks for them
self.Rss = KirbyBase()
self.RssLock = Lock()
self.Casts = KirbyBase()
self.CastsLock = Lock()
# simple config stuff
self.db_dir = db_dir
self.cdb = db_dir+'casts.db'
self.rdb =db_dir+'rss.db'
# error checkin n stuff
# CHECK IF NEEDED ......
self.rsserror = []
self.casterror = []
socket.setdefaulttimeout(30)
# RSS updating vars.
self.RSSUpdates = Queue.Queue()
# do i need this? rewrite with lock if so
self.RSSUpdateLock = False
# checking for connection to internet
# such an easy way out :)
if checkurl('www.google.com'):
self.connection = True
else:
self.connection = False
# checks for existance of databases and creates if not existing
if not checkdb(self.cdb):
self.Casts.create(self.cdb, ['name:str',
'url:str',
'update:bool'])
if not checkdb(self.rdb):
self.Rss.create(self.rdb, ['name:str',
'date:datetime.datetime',
'title:str',
'description:str',
'link:str',
'enclosure:str',
'enclosuretype:str',
'downloaded:bool',
'filename:str',
'opml:str'])
# create some vars with some usefull info on the casts..
self.AllCasts = delistify(self.Casts.select(self.cdb, ['name'],[''],['name']))
self.AllCastDetails = self.Casts.select(self.cdb, ['recno'],['*'])
def UpdateCastDisplay(pyCast):
'''Update the cast treelist'''
## EXTRA CODE??
Casts = KirbyBase()
treeCastList=pyCast.get_widget("podcastlist")
modlCastList=gtk.TreeStore(gobject.TYPE_STRING)
treeCastList.set_model(modlCastList)
treeCastList.set_headers_visible(gtk.FALSE)
x=0
for name in delistify(Casts.select(CASTDB, ['name'],[''],['name'])):
position=modlCastList.insert_after(None,None)
modlCastList.set_value(position,0,name)
x=x+1
def __init__(self,db_dir):
self.Rss = KirbyBase()
self.Casts = KirbyBase()
# simple config stuff
self.db_dir = db_dir
self.cdb = db_dir+'casts.db'
self.rdb =db_dir+'rss.db'
# error checkin n stuff
self.rsserror = []
self.casterror = []
socket.setdefaulttimeout(30)
# RSS updating vars.
self.RSSUpdateList = []
self.RSSUpdateLock = False
# checking for connection to internet
# such an easy way out :)
if checkurl('www.google.com'):
self.connection = True
else:
self.connection = False
# Names of all casts.
if not checkdb(self.cdb):
self.Casts.create(self.cdb, ['name:str',
'url:str',
'update:bool'])
if not checkdb(self.rdb):
self.Rss.create(self.rdb, ['name:str',
'date:datetime.datetime',
'title:str',
'description:str',
'link:str',
'enclosure:str',
'enclosuretype:str',
'downloaded:bool',
'filename:str',
'opml:str'])
self.AllCasts = delistify(self.Casts.select(self.cdb, ['name'],[''],['name']))
self.AllCastDetails = self.Casts.select(self.cdb, ['recno'],['*'])
def Ok(self,widget):
'''When the Add Cast Dialog's OK button is hit'''
Casts = KirbyBase()
CloseWindow=True
EditCastName = self.EditCast.get_widget("tEditCastName")
EditCastUrl = self.EditCast.get_widget("tEditCastUrl")
EditCastUpdate= self.EditCast.get_widget("cEditCastUpdate")
for name in delistify(Casts.select(CASTDB, ['name'],[''],['name'])):
if EditCastName.get_text()== name:
EditCastName.set_text("Name Already Used")
CloseWindow=False
if CloseWindow is True:
UpdateCastDisplay(self.pyCast)
self.wEditCast.destroy()
def ListCasts(pyCast):
'''Lists all the casts into their container'''
treeCastList=pyCast.get_widget("podcastlist")
modlCastList=gtk.TreeStore(gobject.TYPE_STRING)
Casts = KirbyBase()
treeCastList.set_model(modlCastList)
treeCastList.set_headers_visible(gtk.FALSE)
# Set the columns
column=gtk.TreeViewColumn("Podcast",gtk.CellRendererText(), text=0)
column.set_resizable(gtk.TRUE)
treeCastList.append_column(column)
x=0
# Loop through and add the casts
for name in delistify(Casts.select(CASTDB, ['name'],[''],['name'])):
position=modlCastList.insert_after(None,None)
modlCastList.set_value(position, 0, name)
x=x+1
selection=treeCastList.get_selection()
selection.select_path(0)
def Ok(self,widget):
'''When the Add Cast Dialog's OK button is hit'''
Casts = KirbyBase()
CloseWindow=True
AddCastName = self.AddCast.get_widget("tAddCastName")
AddCastUrl = self.AddCast.get_widget("tAddCastUrl")
AddCastUpdate= self.AddCast.get_widget("cAddCastUpdate")
for name in delistify(Casts.select(CASTDB, ['name'],[''],['name'])):
if AddCastName.get_text()== name:
AddCastName.set_text("Name Already Used")
CloseWindow=False
if CloseWindow is True:
if casts.DownloadTestRSS(AddCastUrl.get_text()):
Casts.insert(CASTDB, [AddCastName.get_text(),
AddCastUrl.get_text(),
AddCastUpdate.get_mode(),
datetime.datetime.now()])
casts.DownloadRSS(self.pyCast,AddCastUrl.get_text())
UpdateCastDisplay(self.pyCast)
self.wAddCast.destroy()
#
# sets treatme to True/False for the images you want
# Syntax: if column1=column2 then set column 4 to column 3
# Example: if setname = 1 then set treatme to false.
#
execfile("../config.py")
from kirbybase import KirbyBase, KBError
from variousfct import *
print "This script is to change the treatme flags... adapt the source !"
proquest(askquestions)
backupfile(imgdb, dbbudir, "treatmeupdate")
db = KirbyBase()
# We do not have to select images first: directly update with a criteria !
n = db.update(imgdb, ['imgname'], ['3_ECAM.2012-04-16T09:26:06.000'], [True],
['gogogo'])
#n = db.update(imgdb, ['setname'], ['3'], [True], ['treatme'])
#n = db.update(imgdb, ['recno'], ['*'], [False], ['treatme']) # there is a bug, this does not seem to work :-(
#n = db.update(imgdb, ['treatme'], [False], [True], ['treatme'])
#n = db.update(imgdb, ['seeingpixels'], ['>-100'], [True], ['treatme'])
#n = db.update(imgdb, ['imgname'], [refimgname], [True], ['treatme'])
def Load():
Conf = KirbyBase()
if not app.checkdb(CONFDB):
Conf.create(CONFDB, ['name:str', 'value:str'])
if os.name == 'posix':
LIBRARY=os.getenv('HOME')+'/podcasts/'
elif os.name == 'nt':
LIBRARY='My Documents/My Podcasts/'
Conf.insert(CONFDB, ['LIBRARY',LIBRARY])
Conf.insert(CONFDB, ['KEEPDAYS','7'])
Conf.insert(CONFDB, ['KEEPNUM','5'])
Conf.insert(CONFDB, ['DATE','%D%m%Y'])
Conf.insert(CONFDB, ['LIBFORMAT','date/name'])
print "Done."
print "- " * 40
print "Checking number of fields per line ..."
nbroffields = len(lines[0].split("|")) - 2
for line in lines[1:]:
n = len(line.split("|"))
if n != nbroffields:
print "\n\nWrong number of fields in line :"
print line
print "Done."
print "- " * 40
print "Checking field types ..."
db = KirbyBase()
output = db.validate(database)
if len(output) == 0:
print "All entries are correct, my dear master."
else:
print "\n\nBummer !\n\n"
for line in output:
print line
print "Done."
sys.exit()
#
# Runs sextractor on the input images. Next script will measure seeing + ellipticity.
#
execfile("../config.py")
from kirbybase import KirbyBase, KBError
from variousfct import *
from datetime import datetime, timedelta
#from readandreplace_fct import *
import shutil
import os
db = KirbyBase()
if thisisatest:
print "This is a test run."
images = db.select(imgdb, ['gogogo', 'treatme', 'testlist'],
[True, True, True],
returnType='dict')
elif update:
print "This is an update."
images = db.select(imgdb, ['gogogo', 'treatme', 'updating'],
[True, True, True],
returnType='dict')
askquestions = False
else:
images = db.select(imgdb, ['gogogo', 'treatme'], [True, True],
returnType='dict')
nbrofimages = len(images)
print "Number of images to treat :", nbrofimages
proquest(askquestions)
#
# A little script to inspect the status quo of the testlist flag
#
execfile("../config.py")
from kirbybase import KirbyBase, KBError
from variousfct import *
fields = ['setname', 'imgname', 'testcomment']
db = KirbyBase()
report = db.select(imgdb, ['testlist'], ['True'],
fields,
sortFields=['setname', 'imgname'],
returnType='report')
images = db.select(imgdb, ['testlist'], ['True'], returnType='dict')
print report
print "There are %i images selected." % len(images)
execfile("../config.py")
from kirbybase import KirbyBase, KBError
from variousfct import *
import star
# Select images to treat
db = KirbyBase()
#images = db.select(imgdb, ['gogogo','treatme'], [True, True], returnType='dict')
images = db.select(imgdb, ['recno'], ["*"], returnType='dict')
nbrofimages = len(images)
print "Number of images to treat :", nbrofimages
proquest(askquestions)
# As we will tweak the database, do a backup first
backupfile(imgdb, dbbudir, "tweak")
for image in images:
if image["telescopename"] == "EulerCAM":
print image["imgname"]
db.update(imgdb, ['recno'], [image['recno']],
{'scalingfactor': 0.89767829371})
db.pack(imgdb) # to erase the blank lines
print "Done"
execfile("../config.py")
from kirbybase import KirbyBase, KBError
from variousfct import *
if not update:
raise mterror("update keyword not set to True !")
print "I will update the database with new images in set %s, telescope %s from %s" %(setname, telescopename, rawdir)
print ""
db = KirbyBase()
images = db.select(imgdb, ['gogogo', 'treatme', 'updating'], [True, True, True], returnType='dict', sortFields=['setname', 'mjd'])
nbrofimages = len(images)
print "Number of images to treat :", nbrofimages
print "If you are sure that everything is all right, I can change the updating flag of the new images to False."
proquest(True)
print "So be it. Shall your word be my credo, I will fully devote myself to this task and shall not stop until the skies fall upon me."
print "As I want your FULL ATTENTION here, I'm asking a second time..."
proquest(True)
updateimages = db.select(imgdb, ['gogogo','treatme','updating'], [True, True, True], returnType='dict')
for image in updateimages:
db.update(imgdb, ['recno'], [image['recno']], [False], ['updating'])
db.pack(imgdb) # always a good idea !
print "...aaaaand done !"
#
# write a report about this identification of coordinates
# This report is written for ALL imagesets, independent of current "treatme" flag.
# But results are present one imageset after the other.
execfile("../config.py")
from kirbybase import KirbyBase, KBError
from variousfct import *
from star import *
db = KirbyBase()
fields = [
'imgname', 'seeing', 'goodstars', 'nbralistars', 'maxalistars', 'flagali',
'angle', 'alicomment'
]
usedsetnames = set(
map(lambda x: x[0], db.select(imgdb, ['recno'], ['*'], ['setname'])))
reporttxt = ""
for setname in sorted(list(usedsetnames)):
reporttxt += "\n\n\n###################### %10s ###################\n\n" % setname
noali = db.select(imgdb, ['flagali', 'setname'], ['!= 1', setname],
fields,
sortFields=['seeing'],
returnType='report')
nbr = len(
pngdir = os.path.join(workdir, "imgpngs") if update: print "I will complete the existing sky folder. Or create it if you deleted it to save space" if not os.path.isdir(pngdir): os.mkdir(pngdir) else: if os.path.isdir(pngdir): print "I will delete existing stuff." proquest(askquestions) shutil.rmtree(pngdir) os.mkdir(pngdir) # We select the images to treat : db = KirbyBase() if thisisatest : print "This is a test run." images = db.select(imgdb, ['gogogo', 'treatme', 'testlist'], [True, True, True], returnType='dict', sortFields=['setname','mjd']) elif update: print "This is an update." images = db.select(imgdb, ['gogogo', 'treatme', 'updating'], [True, True, True], returnType='dict', sortFields=['setname','mjd']) askquestions=False else : #images = db.select(imgdb, ['gogogo', 'treatme'], [True, True], returnType='dict', sortFields=['mjd']) images = db.select(imgdb, ['gogogo', 'treatme'], [True, True], returnType='dict', sortFields=['setname','mjd']) #images = db.select(imgdb, ['gogogo', 'treatme'], [True, True], returnType='dict', sortFields=['date']) #images = db.select(imgdb, ['gogogo', 'treatme'], [True, True], returnType='dict', sortFields=['seeing']) #images = db.select(imgdb, ['gogogo', 'treatme'], [True, True], returnType='dict', sortFields=['mjd']) #images = db.select(imgdb, ['gogogo', 'ell'], [True, ">0.65"], returnType='dict', sortFields=['mjd'])
execfile("../config.py")
from kirbybase import KirbyBase, KBError
from variousfct import *
fields = [
'imgname', 'skylevel', 'prealistddev', 'seeing', 'goodstars',
'moonpercent', 'moondist', 'moonalt', 'sunalt'
]
db = KirbyBase()
reporttxt = ""
usedsetnames = set(
map(lambda x: x[0], db.select(imgdb, ['recno'], ['*'], ['setname'])))
for setname in usedsetnames:
reporttxt += "\n\n ########### %10s ########\n\n" % setname
setreport = db.select(imgdb, ['gogogo', 'setname'], ['True', setname],
fields,
sortFields=['skylevel'],
returnType='report')
reporttxt += setreport
reporttxtfile = open(os.path.join(workdir, "report_skylevel.txt"), "w")
reporttxtfile.write(reporttxt)
reporttxtfile.close()
#
# put gogogo to False for images that could not be aligned
# this is not done in the script, so that you can rerun the alignement with another set of parameters
# if you are not happy with it, without breaking your previoulsy set gogogo flags.
#
#
#
execfile("../config.py")
from kirbybase import KirbyBase, KBError
from variousfct import *
backupfile(imgdb, dbbudir, "aliflagupdate")
db = KirbyBase()
nbdemotions = db.update(imgdb, ['flagali', 'treatme'], ['!=1', 'True'],
[False, "Could not be aligned."], ['gogogo', 'whynot'])
print "I've kicked", nbdemotions, "images."
db.pack(imgdb)
#!/usr/bin/env python from conf import glade,gladefile,config,castdb from kirbybase import KirbyBase, KBError import datetime import app Casts = KirbyBase() #Casts.create(castdb, ['name:str','url:str','update:bool','utime:datetime.datetime']) #x=0 #while x<len(OCast.names): # #if OCast.update[x] == '0': # udatev=False # else: # udatev=True # Casts.insert(castdb, [OCast.names[x],OCast.urls[x],udatev,datetime.datetime.now()]) # x=x+1 joe= Casts.select(castdb, ['name'],[''],['name']) x=0 while x<len(joe): joe[x]=joe[x][0] x=x+1 print joe
# NOW LOOK AT THIS !
#
# Using pyfits, we will replace the zeroes in the sigma files prior to the psf construction
# superfast, superclean...
#
execfile("../config.py")
from kirbybase import KirbyBase, KBError
from variousfct import *
import pyfits
origdir = os.getcwd()
# select images to treat
db = KirbyBase()
images = db.select(imgdb, ['gogogo', 'treatme', objkeyflag], [True, True, True], returnType='dict')
print "Number of images to treat :", len(images)
proquest(askquestions)
def isNaN(x):
return x!=x
def replaceNaN(filename, value):
sigstars = pyfits.open(filename, mode='update')
scidata = sigstars[0].data
if True in isNaN(scidata):
print "Yep, some work for me : ", len(scidata[isNaN(scidata)]), "pixels."
scidata[isNaN(scidata)] = value
sigstars.flush()
# the actual normalization coefficients that were used for your deconvolution.
writeheader = True
outputfilename = "imgbyimg_%s.rdb" % selectiondeckey
# - - - - - - - - - - - - - - - - - - - - - - -
from kirbybase import KirbyBase, KBError
from variousfct import *
from combibynight_fct import *
from rdbexport import *
import star
import numpy as np
db = KirbyBase()
images = db.select(imgdb,
["gogogo", "treatme", "decfilenum_" + selectiondeckey],
[True, True, '\d\d*'],
returnType='dict',
useRegExp=True)
print "I will export the deconvolution %s" % selectiondeckey
print "This corresponds to ", len(images), "images."
# We extend these general fields by the flux measurements for the selected deconvolution.
ptsrc = star.readmancat(os.path.join(configdir,
selectiondeckey + "_ptsrc.cat"))
print "Sources : " + ", ".join([src.name for src in ptsrc])
proquest(askquestions)
from kirbybase import KirbyBase
db = KirbyBase()
print db.select('/home/micheles/packages/KirbyBase-1.8/plane.tbl',
['country', 'speed'], ['USA', '>400'])
datestr = now.strftime("%Y-%m-%d")
filename = "%s_%s" % (datestr, configstr)
configdir = "."
readmefilepath = os.path.join(configdir, filename + "_readme.txt")
pklfilepath = os.path.join(configdir, filename + "_db.pkl")
print "My basename : %s" % (filename)
########### The readme #############
readme = ["This is the automatic readme file for\n%s\n" % pklfilepath]
# We do only one select :
db = KirbyBase()
images = db.select(databasepath, ['recno'], ['*'],
sortFields=['setname', 'mjd'],
returnType='dict')
mjdsortedimages = sorted(images, key=lambda k: k['mjd'])
readme.append("Target : %s" % configstr)
readme.append("Total : %i images" % (len(images)))
readme.append("Time span : %s -> %s" %
(mjdsortedimages[0]["datet"], mjdsortedimages[-1]["datet"]))
telescopes = sorted(list(set([image["telescopename"] for image in images])))
setnames = sorted(list(set([image["setname"] for image in images])))
readme.append("Telescopes : %s" % ",".join(telescopes))
if os.path.isdir(pngdir):
print "Deleting existing stuff."
proquest(askquestions)
shutil.rmtree(pngdir)
reflinkdestpath = os.path.join(workdir, deckey + "_ref.png")
if os.path.exists(reflinkdestpath):
os.remove(reflinkdestpath)
os.mkdir(pngdir)
# Read input positions of point sources, to draw a legend.
ptsrcs = star.readmancat(ptsrccat)
print "Number of point sources :", len(ptsrcs)
db = KirbyBase()
images = db.select(imgdb, [deckeyfilenum], ['\d\d*'],
returnType='dict',
useRegExp=True,
sortFields=[deckeyfilenum])
# This time we do not include the duplicated reference !
print "Number of images (we disregard the duplicated reference) : %i" % (
len(images))
# Now, just for these pngs, lets put the unique ref image into the first position :
# We select the reference image
refimage = [image for image in images if image['imgname'] == refimgname
][0] # we take the first and only element.
# The "others" :
images = [image for image in images if image['imgname'] != refimgname]
# here we do the actual geomap and gregister (pyraf)
# we apply this inprinciple to all images gogogo, treatme, and flagali
#
execfile("../config.py")
from kirbybase import KirbyBase, KBError
import math
from pyraf import iraf
from variousfct import *
from datetime import datetime, timedelta
# As we will tweak the database, let's do a backup
backupfile(imgdb, dbbudir, "alignimages")
db = KirbyBase()
if thisisatest:
print "This is a test."
images = db.select(imgdb, ['flagali', 'gogogo', 'treatme', 'testlist'],
['==1', True, True, True], ['recno', 'imgname'],
sortFields=['imgname'],
returnType='dict')
elif update:
print "This is an update."
images = db.select(imgdb, ['flagali', 'gogogo', 'treatme', 'updating'],
['==1', True, True, True], ['recno', 'imgname'],
sortFields=['imgname'],
returnType='dict')
askquestions = False
else:
images = db.select(imgdb, ['flagali', 'gogogo', 'treatme'],
#
# write a report about the renormalization
# not sure if useful... let's see
#
execfile("../config.py")
from kirbybase import KirbyBase, KBError
from variousfct import *
fields = ['imgname', renormname, renormerrfieldname, renormcommentfieldname]
db = KirbyBase()
reporttxt = ""
usedsetnames = set(
map(lambda x: x[0], db.select(imgdb, ['recno'], ['*'], ['setname'])))
for setname in sorted(list(usedsetnames)):
reporttxt += "\n\n\n###################### %10s ###################\n\n" % setname
setreport = db.select(imgdb, ['gogogo', 'treatme', 'setname'],
[True, True, setname],
fields,
sortFields=[renormname],
returnType='report')
reporttxt += setreport
reportname = "report_renorm_" + renormname + ".txt"
reporttxtfile = open(os.path.join(workdir, reportname), "w")
execfile("../config.py")
from kirbybase import KirbyBase, KBError
from variousfct import *
import star
import numpy as np
import matplotlib.pyplot as plt
db = KirbyBase()
images = db.select(imgdb, ['gogogo', 'treatme'], [True, True], returnType='dict', sortFields = ["mhjd"])
if update:
askquestions = False
nbrofimages = len(images)
print "I respect treatme, and selected only %i images" % (nbrofimages)
# Read the manual star catalog :
photomstarscatpath = os.path.join(configdir, "photomstars.cat")
photomstars = star.readmancat(photomstarscatpath)
print "Photom stars :"
print "\n".join(["%s\t%.2f\t%.2f" % (s.name, s.x, s.y) for s in photomstars])
nbrofimages = len(images)
print "I respect treatme and will treat", nbrofimages, "images."
proquest(askquestions)
example usage:
>>> a=rand(6,4); b=rebin(a,(3,2))
"""
shape = a.shape
lenShape = len(shape)
factor = np.asarray(shape) / np.asarray(newshape)
evList = ['a.reshape('] + \
['newshape[%d],factor[%d],'%(i,i) for i in xrange(lenShape)] + \
[')'] + ['.sum(%d)'%(i+1) for i in xrange(lenShape)] + \
['/factor[%d]'%i for i in xrange(lenShape)]
return eval(''.join(evList))
db = KirbyBase()
# We select only the images that are deconvolved (and thus have a deckeyfilenum)
images = db.select(imgdb, [deckeyfilenum], ['\d\d*'],
returnType='dict',
useRegExp=True) # the sorting is not important
# We duplicate the ref image, this will be easier for the output reading.
refimage = [image for image in images if image['imgname'] == refimgname
][0] # we take the first and only element.
images.insert(0, refimage.copy()) # This copy is important !!!
images[0][deckeyfilenum] = mcsname(1) # The duplicated ref image gets number 1
nbimg = len(images)
print "Number of images (including duplicated reference) : %i" % (nbimg)
# read params of point sources
print "WARNING OLD VERSION LOOK AT PIPE_VLT FOR UPDATED VERSION"
import sys
sys.exit()
execfile("../config.py")
from kirbybase import KirbyBase, KBError
from variousfct import *
import pyfits
import numpy as np
import cosmics # used to read and write the fits files
askquestions = False
db = KirbyBase()
images = db.select(imgdb, ['flagali','gogogo','treatme'], ['==1',True, True], ['recno','imgname', 'stddev', 'emptymean'], sortFields=['imgname'], returnType='dict')
print "I will treat %i images." % len(images)
proquest(askquestions)
for n, image in enumerate(images):
tempfile = open(os.path.join(configdir, 'skysubcorr_done.txt'), 'a')
lines = open(os.path.join(configdir, 'skysubcorr_done.txt'), 'r').readlines()
lines = [line.split('\n')[0] for line in lines]
print n+1, "/", len(images), ":", image['imgname']
if image['imgname'] in lines:
print 'Already corrected !'
execfile("../config.py")
from kirbybase import KirbyBase, KBError
from variousfct import *
from datetime import datetime
from variousfct import *
if update:
# override config settings...
execfile(os.path.join(configdir, 'deconv_config_update.py'))
askquestions = False
# nothing more. Let's run on the whole set of images now.
print "Starting deconvolution %s" % (deckey)
db = KirbyBase()
images = db.select(imgdb, [deckeyfilenum], ['\d\d*'],
returnType='dict',
useRegExp=True,
sortFields=[deckeyfilenum])
nbimg = len(images) + 1
print "%i images (ref image is duplicated)" % nbimg
proquest(askquestions)
print "Of course I do not update the database here,"
print "so you can go on with something else in the meantime."
starttime = datetime.now()
db = KirbyBase()
images = db.select(imgdb, [deckeyfilenum], ['\d\d*'],
class podcasts:
def __init__(self,db_dir):
self.Rss = KirbyBase()
self.Casts = KirbyBase()
# simple config stuff
self.db_dir = db_dir
self.cdb = db_dir+'casts.db'
self.rdb =db_dir+'rss.db'
# error checkin n stuff
self.rsserror = []
self.casterror = []
socket.setdefaulttimeout(30)
# RSS updating vars.
self.RSSUpdateList = []
self.RSSUpdateLock = False
# checking for connection to internet
# such an easy way out :)
if checkurl('www.google.com'):
self.connection = True
else:
self.connection = False
# Names of all casts.
if not checkdb(self.cdb):
self.Casts.create(self.cdb, ['name:str',
'url:str',
'update:bool'])
if not checkdb(self.rdb):
self.Rss.create(self.rdb, ['name:str',
'date:datetime.datetime',
'title:str',
'description:str',
'link:str',
'enclosure:str',
'enclosuretype:str',
'downloaded:bool',
'filename:str',
'opml:str'])
self.AllCasts = delistify(self.Casts.select(self.cdb, ['name'],[''],['name']))
self.AllCastDetails = self.Casts.select(self.cdb, ['recno'],['*'])
def AppendRSS(self,name):
def UpdateRSS(self,name=None):
''' Download/Rss of name 'name' '''
print "hello world"
# Function declared, new, clean, fixed and complete.. I wish :)
# By god joe, i think this function is Threadsafe!
if self.RSSUpdateLock:
if name is None:
# ok, we gotta make sure we dont double requests.
objects = self.Casts.select(self.cdb,['name','url'],['',''],['name','url'])
for object in objects:
if object not in self.RSSUpdateList:
print "hi jo"
self.RSSUpdateList = self.RSSUpdateList.append(object)
else:
if name not in self.RSSUpdateList:
appendage=self.Casts.select(self.cdb,['name'],[name],['name','url'])
self.RSSUpdateList = self.RSSUpdateList.append(appendage)
else:
print 1
# and on we go!
if name is None:
self.RSSUpdateList=self.RSSUpdateList+self.Casts.select(self.cdb,['name','url'],['',''],['name','url'])
self.RSSUpdateLock = True
else:
# they specified a name...
appendage=self.Casts.select(self.cdb,['name'],[name],['name','url'])
self.RSSUpdateList = self.RSSUpdateList.append(appendage)
print 2
for cast in self.RSSUpdateList:
print cast
print name
print 3
self.RSSUpdateList.pop()
if cast[0] == name and name != None or name == None:
print 4
(handle, tmp) = tempfile.mkstemp()
urlgood = checkurl(cast[1])
print urlgood
print "ok",cast
if urlgood:
print 5
print cast
try:
urllib.urlretrieve(cast[1],tmp)
except socket.timeout, ex:
self.rsserror.append({ 'name' : cast[0], 'url': cast[1]})
except socket.gaierror, ex:
self.rsserror.append({ 'name' : cast[0], 'url': cast[1]})
rssfile=feedparser.parse(tmp)
for entry in rssfile.entries:
try:
enclosureurl=entry.enclosures[0].url
enclosuretype=entry.enclosures[0].type
except AttributeError:
enclosureurl=''
enclosuretype=''
try:
NewEntry = []
NewEntry.append(cast[0])
NewEntry.append(datetime.datetime.now())
NewEntry.append(str(entry.title))
except AttributeError:
NewEntry.append('')
try:
NewEntry.append(str(entry.summary))
except AttributeError:
NewEntry.append('')
try:
NewEntry.append(str(entry.link))
except AttributeError:
NewEntry.append('')
NewEntry=NewEntry+[str(enclosureurl),str(enclosuretype),False,'na','na']
NewEntryCut = [NewEntry[0]]+NewEntry[2:6]
print NewEntry
ExistingEntries = self.Rss.select(self.rdb,['name','title','description','link','enclosure',],
['','','','',''],['name','title','description','link','enclosure',])
if NewEntryCut not in ExistingEntries:
self.Rss.insert(self.rdb,NewEntry)
print "Added: "+NewEntry[2]
else:
print "well the url "+cast[0]+" went bad, offer to remove later..."
self.RSSUpdateLock=False
import app from conf import CASTDB from kirbybase import KirbyBase, KBError import datetime Casts = KirbyBase() Cast2=Casts.select(CASTDB, ['name','url'],['',''],['name','url']) print Cast2
#
# Similar to the 1a_checkalistars, we draw the alignment stars, this time on the combi image (to make a "nice" map)
#
execfile("../config.py")
from kirbybase import KirbyBase, KBError
from variousfct import *
import star
#import shutil
import f2n
#from datetime import datetime, timedelta
# Read reference image info from database
db = KirbyBase()
refimage = db.select(imgdb, ['imgname'], [refimgname], returnType='dict')
refimage = refimage[0]
refsexcat = os.path.join(alidir, refimage['imgname'] + ".cat")
refautostars = star.readsexcat(refsexcat)
refautostars = star.sortstarlistbyflux(refautostars)
refscalingfactor = refimage['scalingfactor']
# read and identify the manual reference catalog
refmanstars = star.readmancat(
alistarscat) # So these are the "manual" star coordinates
id = star.listidentify(
refmanstars, refautostars, tolerance=identtolerance
) # We find the corresponding precise sextractor coordinates
preciserefmanstars = star.sortstarlistbyflux(id["match"])
maxalistars = len(refmanstars)
"""
execfile("../config.py")
from kirbybase import KirbyBase, KBError
from variousfct import *
import star
import numpy as np
import os, sys
import f2n
import datetime
from headerstuff import juliandate
import matplotlib.pyplot as plt
db = KirbyBase()
images = db.select(imgdb, ['gogogo', 'treatme'], [True, True],
returnType='dict')
# remove saturated
skipimages = [
image[0]
for image in readimagelist(os.path.join(configdir, 'saturatedlist.txt'))
] # image[1] would be the comment
#print "Something special here: I do not plot images that you put in saturatedlist.txt"
images = [image for image in images if not image['imgname'] in skipimages]
nbrofimages = len(images)
print "I respect treatme, saturatedlist and other keywords defined in the script, and selected only %i images" % (
nbrofimages)
#
# Histogramm of the measured seeings, for each set.
#
execfile("../config.py")
from kirbybase import KirbyBase, KBError
from variousfct import *
import matplotlib.pyplot as plt
import numpy as np
#imgdb = "/Users/mtewes/Desktop/vieuxdb.dat"
db = KirbyBase()
# We read this only once.
images = db.select(imgdb, ['gogogo'], [True], returnType='dict')
times = [image['mhjd'] for image in images]
ellipticities = [image['ell'] for image in images]
plt.figure(figsize=(6, 3))
plt.scatter(times, ellipticities)
plt.title('')
plt.show()
elif radalt > math.pi / 2.0:
return -2.0
else:
return 1.0 / (math.sin(radalt) +
0.025 * math.exp(-11.0 * math.sin(radalt)))
# A plot to test :
#n = 100
#alts = [(float(i)/float(n)) *math.pi/2.0 for i in range(n+1)]
#ams = [airmass(alt) for alt in alts]
#degs = [alt*180.0/math.pi for alt in alts]
#plt.plot(degs, ams)
#plt.show()
db = KirbyBase()
if update:
images = db.select(imgdb, ['gogogo', 'treatme', 'updating'],
[True, True, True],
returnType='dict')
askquestions = False
else:
images = db.select(imgdb, ['gogogo', 'treatme'], [True, True],
returnType='dict')
nbrofimages = len(images)
print "Number of images to treat :", nbrofimages
proquest(askquestions)
print "Doublecheck that this is true : ", xephemlens
proquest(askquestions)
import matplotlib.dates print "You want to analyze the deconvolution %s" %deckey print "Deconvolved object : %s" % decobjname if plotnormfieldname == None: print "I will use the normalization coeffs used for the deconvolution." else: print "Using %s for the normalization." % (plotnormfieldname) deckeynormused = plotnormfieldname ptsources = star.readmancat(ptsrccat) print "Number of point sources : %i" % len(ptsources) print "Names of sources : %s" % ", ".join([s.name for s in ptsources]) db = KirbyBase() images = db.select(imgdb, [deckeyfilenum], ['\d\d*'], returnType='dict', useRegExp=True, sortFields=['mjd']) print "%i images" % len(images) fieldnames = db.getFieldNames(imgdb) plt.figure(figsize=(15,15)) mhjds = np.array([image["mhjd"] for image in images]) seeings = np.array([image["seeing"] for image in images]) for s in ptsources: fluxfieldname = "out_%s_%s_flux" % (deckey, s.name) mags = -2.5*np.log10(np.array([image[fluxfieldname]*image[deckeynormused] for image in images]))
"""
We add raw sextractor photom into the database (i.e., not normalized).
Can be run at any time once :
- the alignment is done
- the normalisation catalogs are written (2_runsex.py)
"""
execfile("../config.py")
from kirbybase import KirbyBase, KBError
from variousfct import *
import star
# Selecting the images
db = KirbyBase()
if update:
print "This is an update."
images = db.select(imgdb, ['gogogo', 'treatme', 'updating'],
[True, True, True],
returnType='dict')
askquestions = False
else:
images = db.select(imgdb, ['gogogo', 'treatme'], [True, True],
returnType='dict')
nbrofimages = len(images)
# Read the manual star catalog :
photomstarscatpath = os.path.join(configdir, "photomstars.cat")
photomstars = star.readmancat(photomstarscatpath)
print "I will look for these stars :"
print "\n".join(["%s\t%.2f\t%.2f" % (s.name, s.x, s.y) for s in photomstars])
execfile("../config.py")
from kirbybase import KirbyBase, KBError
db = KirbyBase()
import matplotlib.pyplot as plt
import numpy as np
images = db.select(imgdb, ['recno'], ['*'], returnType='dict')
#images = db.select(imgdb, ['gogogo','treatme'], [True, True], returnType='dict')
jds = np.array([float(image['jd']) for image in images])
hjds = np.array([float(image['hjd']) for image in images])
moondist = np.array([float(image['moondist']) for image in images])
plt.plot(jds, hjds - jds, "b.")
plt.show()
decskipimages = [image[0] for image in readimagelist(decskiplist)
] # image[1] would be the comment
print "It contains %i images." % len(decskipimages)
#proquest(askquestions)
# We have a look at the psfkicklists and the flags for each psf in the database:
print "Ok, now looking at the PSFs ..."
psfimages = {
} # we will build a dictionary, one entry for each psfname, containing
# available images.
db = KirbyBase()
for particularpsfname in decpsfnames:
print "- " * 30
# the database
particularpsfkey = "psf_" + particularpsfname
particularpsfkeyflag = "flag_" + particularpsfkey
particulartreatedimages = [
image[0] for image in db.select(imgdb, [particularpsfkeyflag], [True],
['imgname'])
] # we get a list of imgnames
# the [0] is just to get rid of the list-of-list structure
# the skiplist
particularskiplist = os.path.join(configdir,
particularpsfkey + "_skiplist.txt")
particularskiplistimages = [
class podcasts:
def __init__(self,db_dir):
# open objects for our dbs and locks for them
self.Rss = KirbyBase()
self.RssLock = Lock()
self.Casts = KirbyBase()
self.CastsLock = Lock()
# simple config stuff
self.db_dir = db_dir
self.cdb = db_dir+'casts.db'
self.rdb =db_dir+'rss.db'
# error checkin n stuff
# CHECK IF NEEDED ......
self.rsserror = []
self.casterror = []
socket.setdefaulttimeout(30)
# RSS updating vars.
self.RSSUpdates = Queue.Queue()
# do i need this? rewrite with lock if so
self.RSSUpdateLock = False
# checking for connection to internet
# such an easy way out :)
if checkurl('www.google.com'):
self.connection = True
else:
self.connection = False
# checks for existance of databases and creates if not existing
if not checkdb(self.cdb):
self.Casts.create(self.cdb, ['name:str',
'url:str',
'update:bool'])
if not checkdb(self.rdb):
self.Rss.create(self.rdb, ['name:str',
'date:datetime.datetime',
'title:str',
'description:str',
'link:str',
'enclosure:str',
'enclosuretype:str',
'downloaded:bool',
'filename:str',
'opml:str'])
# create some vars with some usefull info on the casts..
self.AllCasts = delistify(self.Casts.select(self.cdb, ['name'],[''],['name']))
self.AllCastDetails = self.Casts.select(self.cdb, ['recno'],['*'])
def AppendRSS(self,name):
''' Add an RSS item to the queue to be updated '''
appendable = self.Casts.select(self.cdb,['name','url'],['^'+name+'$',''],['name','url'])
self.RSSUpdates.put(appendable)
print appendable
def UpdateRSS(self):
''' Download/Rss of name 'name' '''
# Function declared, new, clean, fixed and complete.. I wish :)
# By god joe, i think this function is Threadsafe!
# if process not already running...
if not self.RSSUpdateLock:
# mark as running
self.RSSUpdateLock = True
# and go forever.....
while True:
# get our first item, get temp item, check item, then download
cast = self.RSSUpdates.get(block=True,timeout=None)
(handle, tmp) = mkstemp()
urlgood = checkurl(cast[1])
if urlgood:
# url's good, im downloading
try:
urlretrieve(cast[1],tmp)
except socket.timeout, ex:
self.rsserror.append({ 'name' : cast[0], 'url': cast[1]})
except socket.gaierror, ex:
self.rsserror.append({ 'name' : cast[0], 'url': cast[1]})
# load it into feedparser :D
rssfile=feedparser.parse(tmp)
# lotsa try's here >.< just in case objects arent in cast.
for entry in rssfile.entries:
try:
enclosureurl=entry.enclosures[0].url
enclosuretype=entry.enclosures[0].type
except AttributeError:
enclosureurl=''
enclosuretype=''
try:
NewEntry = []
NewEntry.append(cast[0])
NewEntry.append(datetime.datetime.now())
NewEntry.append(str(entry.title))
except AttributeError:
NewEntry.append('')
try:
NewEntry.append(str(entry.summary))
except AttributeError:
NewEntry.append('')
try:
NewEntry.append(str(entry.link))
except AttributeError:
NewEntry.append('')
# if we made it through there, we create the new entry
NewEntry=NewEntry+[str(enclosureurl),str(enclosuretype),False,'na','na']
NewEntryCut = [NewEntry[0]]+NewEntry[2:6]
ExistingEntries = self.Rss.select(self.rdb,['name','title','description','link','enclosure',],
['','','','',''],['name','title','description','link','enclosure',])
# check its not already existing
if NewEntryCut not in ExistingEntries:
# get lock and add :D
self.RssLock.acquire(blocking=True)
self.Rss.insert(self.rdb,NewEntry)
self.RssLock.release()
# yay!!!!
print "Added: "+NewEntry[2]
else:
# bad url!!!
print "well the url "+cast[0]+" went bad, offer to remove later..."
execfile("../config.py")
from kirbybase import KirbyBase, KBError
from variousfct import *
import glob
print "I will simply remove all"
print "imgname_skysub.fits"
print "imgname_sky.fits"
print "imgname.fits"
print "images from the alidir."
print 'This will save some disk space, now that images are aligned...'
print 'You can rebuild these images at any time if needed.'
# We select the images to treat :
db = KirbyBase()
images = db.select(imgdb, ['recno'], ["*"],
returnType='dict',
sortFields=['setname', 'mjd'])
nbrofimages = len(images)
for i, image in enumerate(images):
print i + 1, "/", nbrofimages, " : ", image['imgname']
rmfiles = [
"%s%s" % (image["imgname"], ext)
for ext in [".fits", "_skysub.fits", "_sky.fits"]
]
for rmfile in rmfiles:
rmpath = os.path.join(alidir, rmfile)
def DownloadRSS(pycast,name):
Rss = KirbyBase()
Casts = KirbyBase()
if not app.checkdb(RSSDB):
Rss.create(RSSDB, ['name:str',
'date:datetime.datetime',
'title:str',
'description:str',
'link:str',
'enclosure:str',
'enclosuretype:str',
'downloaded:bool',
'filename:str',
'opml:str'])
CastList=Casts.select(CASTDB,['name','url'],['',''],['name','url'])
for cast in CastList:
if cast[0] == name and name != '' or name == '':
(handle, tmp) = tempfile.mkstemp()
urlgood = DownloadTestRSS(cast[1])
print "ok",cast
if urlgood:
print cast
urllib.urlretrieve(cast[1],tmp)
rssfile=feedparser.parse(tmp)
for entry in rssfile.entries:
debug=False
if cast[0]=="Evil Genius Chronicles":
print entry
debug=True
try:
enclosureurl=entry.enclosures[0].url
enclosuretype=entry.enclosures[0].type
except AttributeError:
enclosureurl=''
enclosuretype=''
try:
NewEntry = []
NewEntry.append(cast[0])
NewEntry.append(datetime.datetime.now())
NewEntry.append(str(entry.title))
except AttributeError:
NewEntry.append('')
try:
NewEntry.append(str(entry.summary))
except AttributeError:
NewEntry.append('')
try:
NewEntry.append(str(entry.link))
except AttributeError:
NewEntry.append('')
NewEntry=NewEntry+[str(enclosureurl),str(enclosuretype),False,'na','na']
NewEntryCut = [NewEntry[0]]+NewEntry[2:6]
ExistingEntries = Rss.select(RSSDB,['name','title','description','link','enclosure',],
['','','','',''],['name','title','description','link','enclosure',])
if NewEntryCut not in ExistingEntries:
Rss.insert(RSSDB,NewEntry)
print "Added: "+NewEntry[2]
else:
print "well the url "+cast[0]+" went bad, offer to remove later..."
"""Simple test of kirbybase.
"""
import os
from kirbybase import KirbyBase, KBError
import datetime
# Uncomment one or the other of next two lines to switch between multiuser
# and embedded. If multiuser, make sure you have a kbserver running.
db = KirbyBase()
#db = KirbyBase('client', '127.0.0.1', 44444)
# If table exists, delete it.
if os.path.exists('plane.tbl'):
db.drop('plane.tbl')
# Create a table.
db.create('plane.tbl', ['name:str','country:str','role:str','speed:int',
'range:int','began_service:datetime.date', 'still_flying:bool'])
# Insert a bunch of records.
print db.insert('plane.tbl', ['P-51','USA','Fighter',403,1201,
datetime.date(1943,6,24),True])
print db.insert('plane.tbl', ['P-47','USA','Fighter',365,888,
datetime.date(1943,3,12),False])
print db.insert('plane.tbl', ['B-17','USA','Bomber',315,1400,
datetime.date(1937,5,1),True])
print db.insert('plane.tbl', ['Typhoon', 'Great Britain','Fighter-Bomber',
389,690,datetime.date(1944,11,20),False])
print db.insert('plane.tbl', ['Sptitfire','Great Britain','Fighter',345,