def recipe_paths(self, line):
context = None
if len(line)>0:
context = line
try:
self.ROOTDIR = self.ipython.ev("ROOTDIR")
except NameError:
ROOTDIR = self.ROOTDIR = os.getcwd()
self.ipython.push({"ROOTDIR":self.ROOTDIR})
print "ROOTDIR = %s" % self.ROOTDIR
print "CURRENTDIR = %s" % os.getcwd()
tdirs_table = get_lookup_table("HydroBio/directories", context=context)
if not tdirs_table:
print "NO SUCH CONTEXT:", context
return line
tdirs = tdirs_table[0]
#print "tdirs",tdirs
#print ks.dict2pretty("the_dirs",tdirs)
the_dirs = {}
for key in tdirs:
the_dirs[key] = tdirs[key] % os.environ
self.ipython.push(the_dirs)
print ks.dict2pretty("adding variables to namespace", the_dirs)
return True
def report_qametrics_2adcc(self, qd):
self.events_manager.append_event(qd)
print ks.dict2pretty("rIM209:",qd)
# if publishers, publish
for publisher in self.publishers:
publisher.publish_document(qd)
return
def sample_shape(self, line):
args = line.split()
options = parser.parse_args(args = args)
phrase = options.phrase
print "Looking for '%s' in fieldname" % (phrase)
mdb = MDBStorage(collection_name = "shapes")
reg = ".*?%s.*?" % phrase
cursor = mdb.collection.find({"geojson.feature.properties.fieldname": {"$regex": reg} } )
numfound = cursor.count()
if numfound == 0:
print tc.colored("NO SHAPES MATCH", "red")
return None
print "numfound = %d selecting 0'th" % numfound
if options.verbose:
outd = []
for shape in cursor:
outd.append(shape["_id"])
print ks.dict2pretty("shapes found", outd)
hbshape = HBShapeData({"setref":cursor[0]})
addn = {"hbshape": hbshape, "shape_id": hbshape.meta("_id"), "loaded_from_mongo":True}
addnmsg = {"hbshape":type(hbshape), "shape_id": addn["shape_id"],
"loaded_from_mongo":True}
print ks.dict2pretty("adding to namespace", addnmsg)
self.ipython.push(addn)
return None
def configure(self, options):
collection_name = None
config_location = None
config_key = None
if "collection_name" in options:
collection_name = options["collection_name"]
if "db_info" in options:
config_location = options["db_info"]["location"]
config_key = options["db_info"]["key"]
config_location = options["db_info"]["location"]
mtret = compose_multi_table(config_location, config_key, just_one=True)
mdbinfo = mtret["mdb_info"]
print ksutil.dict2pretty("mdb_info", mdbinfo)
self.host = mdbinfo["host"]
self.dbname = mdbinfo["db"]
self.user = mdbinfo["user"]
self.password = base64.b64decode(mdbinfo["password"])
self.get_connection()
if collection_name:
self.collection_name = collection_name
self.collection = self.db[collection_name]
def __init__(self, reduceport):
# get my client for the reduce commands
print "starting xmlrpc client to port %d..." % reduceport,
self.reducecmds = xmlrpclib.ServerProxy("http://localhost:%d/" \
% reduceport, allow_none=True)
print "started"
try:
self.reducecmds.prs_ready()
except socketError:
print "prs50: no reduce instances running"
self.reducedict = {}
# these members save command history so that tools have access, e.g.
# a display tool
self.stackKeeper = StackKeeper(local=True)
self.displayCmdHistory = TSCmdQueue()
self.events_manager = EventsManager(persist="adcc_events.jsa")
# publishers
self.publishers = []
pubret = compose_multi_table("*/adcc_settings", "publish_qametrics")
print ks.dict2pretty("em37:publish_qametrics", pubret)
if pubret and "publish_qametrics" in pubret:
pubconfig = pubret["publish_qametrics"]
for publer in pubconfig:
module = publer["publisher"][0]
classname = publer["publisher"][1]
dbinfo = publer["db_info"]
collection_name = publer["collection"]
exec("import %s" % module)
publisher = eval("%s.%s()" % (module,classname))
publisher.configure({ "collection_name": collection_name,
"db_info":dbinfo
})
self.publishers.append(publisher)
def recipe_paths(self, line):
context = None
if len(line) > 0:
context = line
try:
self.ROOTDIR = self.ipython.ev("ROOTDIR")
except NameError:
ROOTDIR = self.ROOTDIR = os.getcwd()
self.ipython.push({"ROOTDIR": self.ROOTDIR})
print "ROOTDIR = %s" % self.ROOTDIR
print "CURRENTDIR = %s" % os.getcwd()
tdirs_table = get_lookup_table("HydroBio/directories", context=context)
if not tdirs_table:
print "NO SUCH CONTEXT:", context
return line
tdirs = tdirs_table[0]
#print "tdirs",tdirs
#print ks.dict2pretty("the_dirs",tdirs)
the_dirs = {}
for key in tdirs:
the_dirs[key] = tdirs[key] % os.environ
self.ipython.push(the_dirs)
print ks.dict2pretty("adding variables to namespace", the_dirs)
return True
def __init__(self, source = None):
self._current_manifests = []
if source:
self.source = source
if "elements" in self.source:
elements = self.source["elements"]
else:
elements = {}
if "shelf_name" not in elements:
elements["shelf_name"] = source["ingest_shelf"]
if "frequency" in source:
self.frequency = source["frequency"]
if "user" not in source:
source["user"] = getpass.getuser()
if "home" not in source:
if "HOME" in os.environ:
source["home"] = os.environ["HOME"]
print "d_p8:", ks.dict2pretty("source", source)
in_pt = source["ingest_package_type"]
in_shelf = source["ingest_shelf"]
in_package = package_dict[in_pt]()
self.prefix = in_package.get_store_prefix( elements = elements )
print "Daemon Process __init__: source prefix/key: %s (d_p68)" % self.prefix
self.ingest_package = in_package
if "publish_package_type" in source:
pub_pt = source["publish_package_type"]
pub_package = package_dict[pub_pt]()
self.publish_package = pub_package
def tend_process_queue():
global procs_by_shelf, queue_by_shelf
done = False
while not done:
try:
sleep(POLL_RESOLUTION)
for shelf_name in queue_by_shelf:
result_q = queue_by_shelf[shelf_name]["result_queue"]
try:
mess = result_q.get(False)
except Empty:
mess = None
if mess and ("result" in mess) and mess["result"]:
print ks.dict2pretty("d_p92 main proc Queue.get: %s" % shelf_name, mess)
except KeyboardInterrupt:
print "keyboard interupt, finishing..."
done = True
def _accept_initarg(self, initarg):
if DEBUG:
print ks.dict2pretty("sr187: initarg", initarg)
if isinstance(initarg, basestring):
self.filename = initarg
self._loaded_by = "filename"
elif isinstance(initarg, dict):
if "filename" in initarg:
self.filename = initarg["filename"]
if "setref" in initarg:
self._setref = initarg["setref"]
#self.filename = "hb_shape:%s" % self._setref["_id"]
self._loaded_by = "setref"
else:
if hasattr(initarg, "filename"):
self.filename = initarg.filename
self._loaded_by = self.filename
def _accept_initarg(self, initarg):
if DEBUG:
print ks.dict2pretty( "sr187: initarg", initarg)
if isinstance(initarg, basestring):
self.filename = initarg
self._loaded_by = "filename"
elif isinstance(initarg, dict):
if "filename" in initarg:
self.filename = initarg["filename"]
if "setref" in initarg:
self._setref = initarg["setref"]
#self.filename = "hb_shape:%s" % self._setref["_id"]
self._loaded_by = "setref"
else:
if hasattr(initarg, "filename"):
self.filename = initarg.filename
self._loaded_by = self.filename
def __init__(self, initarg = None, force_load = None):
"""
"""
super(SetrefData, self) .__init__( initarg)
# child defined
self._setref = {}
self._initialize_()
# @@NOTE:@@TODO: remove this, extraneous reference
self.initarg = initarg
#print "sr35:",initarg
self._accept_initarg(initarg)
if DEBUG:
print ks.dict2pretty( "sr37: __init__", self._setref)
if force_load:
self.load_header()
self.load()
else:
self.load_header()
def __init__(self, initarg=None, force_load=None):
"""
"""
super(SetrefData, self).__init__(initarg)
# child defined
self._setref = {}
self._initialize_()
# @@NOTE:@@TODO: remove this, extraneous reference
self.initarg = initarg
#print "sr35:",initarg
self._accept_initarg(initarg)
if DEBUG:
print ks.dict2pretty("sr37: __init__", self._setref)
if force_load:
self.load_header()
self.load()
else:
self.load_header()
def sample_shape(self, line):
args = line.split()
options = parser.parse_args(args=args)
phrase = options.phrase
print "Looking for '%s' in fieldname" % (phrase)
mdb = MDBStorage(collection_name="shapes")
reg = ".*?%s.*?" % phrase
cursor = mdb.collection.find(
{"geojson.feature.properties.fieldname": {
"$regex": reg
}})
numfound = cursor.count()
if numfound == 0:
print tc.colored("NO SHAPES MATCH", "red")
return None
print "numfound = %d selecting 0'th" % numfound
if options.verbose:
outd = []
for shape in cursor:
outd.append(shape["_id"])
print ks.dict2pretty("shapes found", outd)
hbshape = HBShapeData({"setref": cursor[0]})
addn = {
"hbshape": hbshape,
"shape_id": hbshape.meta("_id"),
"loaded_from_mongo": True
}
addnmsg = {
"hbshape": type(hbshape),
"shape_id": addn["shape_id"],
"loaded_from_mongo": True
}
print ks.dict2pretty("adding to namespace", addnmsg)
self.ipython.push(addn)
return None
def get_store_dirname(self, setref=None, elements=None):
if setref:
self.elements_from_setref(setref)
if elements:
print ks.dict2pretty("fsp134:elements", self.elements)
self.elements.update(elements)
print ks.dict2pretty("fsp134:elements", self.elements)
if "shelf_name" in self.elements:
# presumably by override
storepath = self.format_storage_location(
self.elements["shelf_name"])
else:
# find the shelf based on types
settype = self.elements["type"]
if settype in self.type_shelf_names:
shelfname = self.type_shelf_names[settype]
storepath = self.format_storage_location(shelfname)
else:
storepath = self.format_storage_location(shelfname)
self.store_dirname = storepath
return storepath
def get_recipe_dirs(self):
"""This function returns a list of directories to walk for a given
configuration space.
@param spacename: name of the config space to collect directories for
@type spacename: string
@returns: list of directories
@rtype: list"""
if (self.recipedirs != None):
return self.recipedirs
retdirs = []
# get the ADCONFIG package dirs
adconfdirs = []
i = 1
pathlist = sys.path
# support for ADCONFIGPATH and RECIPEPATH
# NOTE: due to the way the list is extended, ADCONFIGPATH appearing second
# means it has precedence over RECIPEPATH, that is, ADCONFIGPATH
# is searched prior to RECIPEPATH
if "RECIPEPATH" in os.environ:
rpath = os.environ["RECIPEPATH"].split(":")
# we want this path in front...
rpath.extend(pathlist)
pathlist = rpath
if "ADCONFIGPATH" in os.environ:
rpath = os.environ["ADCONFIGPATH"].split(":")
# we want this path in front...
rpath.extend(pathlist)
pathlist = rpath
if False:
from astrodata.adutils.ksutil import dict2pretty
from math import ceil, log
print dict2pretty("pathlist (CS270):", pathlist,complete = True, namewidth = 3 ) #ceil(log(len(pathlist))) )
pkmask = {}
for path in pathlist:
# print "@@@@@@@@:",".svn" in path,":::", path
if os.path.isdir(path):
#print "CS279: ISADIR ", path
subdirs = os.listdir(path)
for subpath in subdirs:
if not os.path.isdir(os.path.join(path,subpath)):
continue
#print "CS285: subpath", subpath
if re.match(PACKAGEMARKER, subpath):
#print "CS288: .... is a PACKAGE"
# this logic allows packages early in the path
# to hide packages late in the path
pknam = os.path.basename(subpath)
if pknam in pkmask:
continue
else:
pkmask[pknam] = subpath
subsubpaths = os.listdir(os.path.join(path,subpath))
for subsubpath in subsubpaths:
if RECIPEMARKER in subsubpath:
fullpath = os.path.join(path, subpath, subsubpath)
#print "RECIPEMARKER full", fullpath
if os.path.isdir(fullpath):
# then this is one of the config space directories
adconfdirs.append(fullpath)
elif RECIPEMARKER in subpath:
fullpath = os.path.join(path, subpath)
if os.path.isdir(fullpath):
adconfdirs.append(fullpath)
else:
pass # print ""
self.recipedirs = adconfdirs
# print "CS183:",repr(adconfdirs)
return adconfdirs
def pretty_string(self):
retstr = ksutil.dict2pretty(self.filename, self.json)
return retstr
remove_local = False
if args.archive:
remove_local = True
if args.store:
remove_local = False
if args.remove_local != None:
remove_local = args.remove_local
for key in package_class_struct:
package_key = key
package_class = package_class_struct[key]
break; # only one supported atm, always first, controlled by path order
if args.info:
print ks.dict2pretty("contributing files", package_classes["_contributors"])
for i in range(len(package_class_list)):
package_def = package_class_list[i]
print ks.dict2pretty("packager #%d" % i, package_def)
print "choosing %s package class %s" % (tc.colored(key, attrs=["bold"]), tc.colored(package_class, attrs=["dark"]))
pkg = package_class()
print ks.dict2pretty("shelf_addresses", pkg.shelf_addresses)
print ks.dict2pretty("type_shelf_names", pkg.type_shelf_names)
print ks.dict2pretty("type_store_precedence", pkg.type_store_precedence)
print ks.dict2pretty("daemon_settings: ingest_sources", dp.ingest_sources)
if args.fetch :
args.manifest = True
if args.manifest:
elements = {
"shelf_name" : args.shelf,
def pretty_setref(self, start_indent = 0):
retstr = ""
retstr += ksutil.dict2pretty("_setref", self._setref, indent = start_indent)
return retstr
def sample_tiff(self, line):
import glob
args = line.split()
options = parser.parse_args(args = args)
date_r = options.date_range
phrase = options.phrase
verbose = options.verbose
index = int(options.index)
settype = options.settype
make_list = options.make_list
directory = options.dir
canddict = {}
# options.verbose used below
print "Using date_range = %s and phrase = %s" % (date_r, phrase)
globpart = ("*%(datestr)s*%(phrase)s*.tif" %
{ "datestr":date_r, # can't be range atm
"phrase":phrase
}
)
if not options.cwd:
dirs = self.get_config_paths()
datadir = dirs["processed_data"]% os.environ
else:
datadir = os.path.abspath(os.curdir)
print "Data Directory: %s" % datadir
if options.subdir:
datadir = os.path.join(datadir, options.subdir)
globpath = os.path.join(datadir, globpart)
fils = glob.glob(globpath)
if len(fils) == 0:
print tc.colored("globbed: %s" %globpath, "blue")
print tc.colored("NO FILES MATCH", "red")
return None
hbgeolist = []
for fil in fils:
canddict[fil] = { "path": fil,
"basename": os.path.basename(fil)
}
if make_list:
hbgeolist.append(HBGeoTIFF(fil))
if settype or verbose:
for fil in fils:
dat = HBGeoTIFF(fil)
canddict[fil]["types"] = dat.get_types()
dat = None
if verbose:
print ks.dict2pretty("found", [
{ "name" :canddict[fil]["basename"],
"types":canddict[fil]["types"]
} for fil in fils])
print "found %d files, choosing image index = %d" % (len(fils), index)
fil = fils[index]
hbgeo = HBGeoTIFF(fil)
ipython = self.ipython
addn = {"hbgeo": hbgeo, "tiffname": fil}
addn["hbgeos"] = hbgeolist
addnmsg = {"hbgeo":type(hbgeo), "tiffname": os.path.basename(fil)}
print ks.dict2pretty("adding to namespace", addnmsg)
ipython.push(addn)
return None
def pretty_setref(self, start_indent=0):
retstr = ""
retstr += ksutil.dict2pretty("_setref",
self._setref,
indent=start_indent)
return retstr
def do_POST(self):
global webserverdone
parms = parsepath(self.path)
vlen = int(self.headers["Content-Length"])
head = self.rfile.read(vlen)
pdict = head
if parms["path"].startswith("/runreduce"):
import time
import json
# Get events manager
evman = None
if "rim" in ADCCHandler.informers:
rim = ADCCHandler.informers["rim"]
evman = rim.events_manager
self.send_response(200)
self.send_header('Content-type', 'text/plain')
self.send_header("Access-Control-Allow-Origin", "http://localhost")
self.end_headers()
reduce_params = json.loads(pdict)
from astrodata.adutils import ksutil as ks
print ks.dict2pretty("runreduce_params", reduce_params)
if reduce_params.has_key("filepath"):
fp = reduce_params["filepath"]
else:
fp = None
if reduce_params.has_key("options"):
opt = reduce_params["options"]
else:
opt = None
if reduce_params.has_key("parameters"):
prm = reduce_params["parameters"]
else:
prm = None
cmdlist = ["kit", "--invoked"]
if opt is not None:
for key in opt:
cmdlist.extend(["--"+str(key),str(opt[key])])
if prm is not None:
prm_str = ""
for key in prm:
prm_str += str(key)+"="+str(prm[key])+","
if prm_str!="":
prm_str = prm_str.rstrip(",")
cmdlist.extend(["-p",prm_str])
if fp is not None:
cmdlist.append(str(fp))
# Check that file can be opened
try:
ad = AstroData(fp)
except:
self.wfile.write("Can't use AstroData to open %s"% fp)
return
# Report reduction status
self.wfile.write("Reducing %s\n" % fp)
self.wfile.write("Command: %s\n" % " ".join(cmdlist))
evman.append_event(ad,"status",{"current":"reducing",
"logfile":None},
msgtype="reduce_status")
# Send reduce log to hidden directory
logdir = ".autologs"
if not os.path.exists(logdir):
os.mkdir(logdir)
reducelog = os.path.join(
logdir, "reduce-addcinvokedlog-%d%s" % (
os.getpid(), str(time.time())))
f = open(reducelog, "w")
loglink = "reducelog-latest"
if os.path.exists(loglink):
os.remove(loglink)
os.symlink(reducelog, loglink)
print "prs1048:", ks.dict2pretty("cmdlist", cmdlist)
# Call reduce
pid = subprocess.call( cmdlist,
stdout = f,
stderr = f)
f.close()
# Report finished status
if fp is not None:
if pid==0:
evman.append_event(ad,"status",
{"current":"reduction finished",
"logfile":reducelog},
msgtype="reduce_status")
else:
evman.append_event(ad,"status",
{"current":"reduction ERROR",
"logfile":reducelog},
msgtype="reduce_status")
# Get text from log
f = open(reducelog, "r")
txt = f.read()
f.close()
self.wfile.write(txt)
self.wfile.flush()
return
if parms["path"].startswith("/calmgr"):
from FitsStorageWebSummary.Selection import getselection
from FitsStorageWebSummary.CalMGR import calmgr
things = parms["path"].split("/")[2:-1]
print "ppwDOPOST:"+ repr(things)
self.send_response(200)
self.send_header('Content-type', 'text/xml')
self.end_headers()
# Parse the rest of the URL.
selection=getselection(things)
# If we want other arguments like order by
# we should parse them here
# print "PPW1125:"+repr(pdict)
req = PRec(pdict=pdict, method="POST")
retval = calmgr(req, selection)
print "ppw1128::::"*3, req._buff
self.wfile.write(req._buff)
return
global rootnode
try:
ctype, pdict = cgi.parse_header(self.headers.getheader('content-type'))
if ctype == 'multipart/form-data':
query=cgi.parse_multipart(self.rfile, pdict)
self.send_response(301)
self.end_headers()
upfilecontent = query.get('upfile')
print "filecontent", upfilecontent[0]
self.wfile.write("<HTML>POST OK.<BR><BR>");
self.wfile.write(upfilecontent[0]);
except :
pass
def pretty_string(self):
retstr = ksutil.dict2pretty(self.filename, self.json)
return retstr
def showContext(self, rc):
log.status(ksutil.dict2pretty("local parameters", rc.localparms))
log.status(ksutil.dict2pretty("global parameters", rc))
yield rc
def showContext(self, rc):
log.status(ksutil.dict2pretty("local parameters", rc.localparms))
log.status(ksutil.dict2pretty("global parameters", rc))
yield rc
def sample_tiff(self, line):
import glob
args = line.split()
options = parser.parse_args(args=args)
date_r = options.date_range
phrase = options.phrase
verbose = options.verbose
index = int(options.index)
settype = options.settype
make_list = options.make_list
directory = options.dir
canddict = {}
# options.verbose used below
print "Using date_range = %s and phrase = %s" % (date_r, phrase)
globpart = (
"*%(datestr)s*%(phrase)s*.tif" % {
"datestr": date_r, # can't be range atm
"phrase": phrase
})
if not options.cwd:
dirs = self.get_config_paths()
datadir = dirs["processed_data"] % os.environ
else:
datadir = os.path.abspath(os.curdir)
print "Data Directory: %s" % datadir
if options.subdir:
datadir = os.path.join(datadir, options.subdir)
globpath = os.path.join(datadir, globpart)
fils = glob.glob(globpath)
if len(fils) == 0:
print tc.colored("globbed: %s" % globpath, "blue")
print tc.colored("NO FILES MATCH", "red")
return None
hbgeolist = []
for fil in fils:
canddict[fil] = {"path": fil, "basename": os.path.basename(fil)}
if make_list:
hbgeolist.append(HBGeoTIFF(fil))
if settype or verbose:
for fil in fils:
dat = HBGeoTIFF(fil)
canddict[fil]["types"] = dat.get_types()
dat = None
if verbose:
print ks.dict2pretty("found", [{
"name": canddict[fil]["basename"],
"types": canddict[fil]["types"]
} for fil in fils])
print "found %d files, choosing image index = %d" % (len(fils), index)
fil = fils[index]
hbgeo = HBGeoTIFF(fil)
ipython = self.ipython
addn = {"hbgeo": hbgeo, "tiffname": fil}
addn["hbgeos"] = hbgeolist
addnmsg = {"hbgeo": type(hbgeo), "tiffname": os.path.basename(fil)}
print ks.dict2pretty("adding to namespace", addnmsg)
ipython.push(addn)
return None