def process(self, fname):
cmop.debug("Seabird file found: %s" % (fname, ))
cast = seabird.SeabirdCast(fname)
cast.GetConverted()
cast.GetBottle()
# uncomment for speedier testing
#f = file('sample.cnv')
#cast.datcnvoutput = f.read()
cast.Parse()
cast.ParseBottle()
record = self.makeCast(cast)
cmop.debug("Parsed Seabird File %s" % (fname, ))
# yield {"sqlcommand":"begin"}
try:
yield record
for o in cast.IterateBottle(self.mapping['depth']):
yield self.makeBottle(record, o)
for o in cast.Iterate():
yield self.makeObservation(record, o)
# yield {"sqlcommand":"commit"}
except:
pass
# yield {"sqlcommand":"rollback"}
cmop.info("Processed %s observations for file %s" %
(len(cast.rows), fname))
def isRunning(current,rootdir,sta,level):
import glob
if rootdir != 'archive':
rootdir = 'test'
testname = "/tmp/flat-cache-db2cdf-level%s-%s-%s-%s" % (level,current,sta,rootdir)
pids = glob.glob("%s.*" % testname)
instanceCount = 0
for pid in pids:
pid = pid.split(".")[1]
# is running
if os.path.exists("/proc/%s" % (pid)):
instanceCount = instanceCount + 1
# not running
else:
cmop.info("instance [%s] is death" % (pid))
os.remove("%s.%s" % (testname,pid))
if instanceCount==0:
pidfile = open("%s.%d" % (testname,os.getpid()), "w")
pidfile.close()
return (instanceCount,testname)
def Scan(self, filename):
f = file(filename)
hdr = self.ReadHeader(f)
pos, item = self.state.get(filename, (0, 0))
cmop.debug("Scanning %s as RecordStream" % (filename, ), 8)
if pos == 0:
pos = f.tell()
opos, oitem = pos, item
f.seek(pos)
for x in self.ReadBlock(hdr, f):
x['row'] = item
item += 1
# add a column for filename
x['file'] = filename
self.Enqueue(x)
pos = f.tell()
if item != oitem:
cmop.info("Scanned %s records (%s bytes) from %s" %
(item - oitem, pos - opos, filename))
self.state[filename] = (pos, item)
def Subscribe(self, table):
# all new records will be marked for broadcast to current host
if not self.Subscribed(table):
msg = "SubscriptionBroker: %s subscribing to table %s on %s"
cmop.info(msg % (self.tohost, table, self.fromhost))
self.Configure(table)
default = '''ALTER TABLE %s ALTER COLUMN "%s" SET DEFAULT True'''
self.fromdb.execCommand(default % (table, self.dirtycolumn))
def UnSubscribe(self, table):
# new records will not be marked for broadcast,
# but the subscription column will not be dropped
# no effect if subscription column does not exist
if self.SubscriptionColumnExists(table):
msg = "SubscriptionBroker: %s UNsubscribing from table %s on %s"
cmop.info(msg % (self.tohost, table, self.fromhost))
mod = '''ALTER TABLE %s ALTER COLUMN "%s" DROP DEFAULT'''
self.fromdb.execCommand(mod % (table, self.dirtycolumn))
def Clean(self, table):
# Remove the subscription column
# Don't do this as part of normal operation;
# dropped columns are not permanently removed!
msg = "SubscriptionBroker: CLEAN CALLED: %s dropping column from table %s on %s"
cmop.info(msg % (self.tohost, table, self.fromhost))
mod = '''ALTER TABLE %s DROP COLUMN "%s"'''
modind = '''DROP INDEX %s_%s'''
self.fromdb.begin()
try:
self.fromdb.execCommand(mod % (table, self.dirtycolumn))
self.fromdb.execCommand(
modind % (table.replace(".", "_"), self.dirtycolumn))
self.fromdb.commit()
except:
self.fromdb.rollback()
def ValuesClause(self, xfertable, extravals=[]):
'''
Read all tuples from the given table and prepare a VALUES clause from the results
Returns a tuple of the values clause as a string and the number of tuples involved.
'''
select = '''SELECT * FROM %s''' % (xfertable, )
cmop.debug("Reading tuples to transfer.", 8)
rs = self.fromdb.execQuery(select)
cnt = len(rs)
if not rs:
cmop.info("No tuples to transfer.")
return "()", 0
def preprow(r):
vals = ["%s" % (db.quote(a), ) for a in r]
vals += extravals
return "(%s)" % (", ".join(vals), )
values = ", ".join([preprow(r) for r in rs])
return values, cnt
def Configure(self, table):
# Add a column on source table representing this subscription
# also add an index
if not self.SubscriptionColumnExists(table):
if not self.tohost.Primarykey(table):
msg = "Table %s does not have a primary key defined; cannot transfer."
raise TypeError(msg % (table, ))
msg = "SubscriptionBroker: Configuring %s subscribing to table %s on %s"
cmop.info(msg % (self.tohost, table, self.fromhost))
self.fromdb.begin()
mod = '''ALTER TABLE %s ADD COLUMN "%s" bool; '''
ind = '''CREATE INDEX "%s_%s" ON %s("%s")'''
vac = '''VACUUM ANALYZE %s'''
try:
self.fromdb.execCommand(mod % (table, self.dirtycolumn))
self.fromdb.execCommand(ind % (table.replace(
".", "_"), self.dirtycolumn, table, self.dirtycolumn))
self.fromdb.commit()
self.fromdb.execCommand(vac % (table, ))
except:
self.fromdb.rollback()
raise
def process(self, data):
try:
tuple = {}
self.GetLatLon(tuple, data)
self.GetTime(tuple, data)
tuple['table'] = "cruise.tsg"
tuple['vessel'] = self.vessel
tuple['cruise'] = self.cruise
tuple['instrument'] = self.instrument
tuple['instrumenttype'] = self.instrumenttype
tuple['salinity'] = float(data['computed_salinity_flothru'])
#tuple['temperature'] = float(data['water_temp_seabird_flothru'])
tuple['temperature'] = float(data['surface_water_temp_seabird'])
tuple['conductivity'] = float(data['conductivity_seabird_flothru'])
tuple['winddirection'] = float(
data.get('wind_heading_ultrasonic_true', -999999))
tuple['windspeed'] = float(
data.get('wind_speed_ultrasonic_true(knots)', -999999))
tuple['atmosphericpressure'] = float(data['barometer'])
tuple['atmospherictemperature'] = float(
data['air_temp_rmyoung_doghouse'])
for k, v in tuple.items():
if str(v) == 'nan':
tuple[k] = None
cmop.debug("WecomaTSGCleaner generated a tuple: %s" % (tuple, ), 8)
return [tuple]
except:
msg = traceback.format_exc()
s = ",".join(["(%s=%s)" % (k, v) for k, v in data.iteritems()])
cmop.info("%s : Skipping bad TSG record: %s" % (msg, s))
return []
elif arg == 'archive':
current = 'all'
elif arg == 'test':
rootdir = './test/'
elif arg == 'level0':
level = 0
elif arg == 'level1':
level = 1
else:
sta = ''
stationsList.append([arg])
print '%s %s %s %s' % (current,rootdir,level,sta)
print stationsList
test = isRunning(current,rootdir,sta,level)
if test[0]!=0:
cmop.info("%s is already running" % test[1])
exit(0)
# extractGliderData()
# extractAUVData()
currentPath = os.getcwd()
sys.path[1:1] = [currentPath]
import stations as station
if sta == 'all':
if current == 'current':
cur = " where currentornull='current'"
else: cur = ''
sql = "select distinct on (station) station from instrument.offeringdetails %s" % cur
db.execute(sql)
stationsList = db.fetchall()
now = datetime.datetime.now()
def info(self, s): cmop.info(s)
def Convert(self, cfg="cmop.cfg"):
'''Use dosemu to run Seabird data conversion.
datcnv must run in the seabird directory, apparently
datcnv is slow; prepare for about 260 scans per second
'''
dat = self.dat
con = self.con
cmop.info("Converting Seabird cast %s, %s" % (dat, con))
root, base, ext = parsepath(self.dat)
# file names must be short
newname = "cast"
# temp dir
d = tempfile.mkdtemp('X', 'X')
cmop.debug("tempdir: \n" + d)
# dat file
datpath = link(d, dat, newname)
dosdat = dosabspath(datpath, self.drive)
# con file
conpath = link(d, con, newname)
doscon = dosabspath(conpath, self.drive)
# cfg file. cfg file path needs to
#be relative to seabird directory, inexplicably
#cfgpath = self.makeconfig(cfg, d)
#doscfg = dosabspath(cfgpath, self.drive)
# output file
out = '%s.cnv' % (newname, )
outpath = os.path.join(d, out)
dosout = dosabspath(outpath, self.drive)
# batch file
bat = '''
%s:
cd %s
datcnv.exe -ax -o%s -i%s -c%s -s -e%s
exitemu
'''
batfile = os.path.join(d, 'seabird.bat')
f = file(batfile, 'w+')
dosseabirddir = dosabspath(self.seabirddir, self.drive)
content = bat % (self.drive, dosseabirddir, dosout, dosdat, doscon,
cfg)
cmop.debug("Batchfile: \n" + content)
f.write(content)
f.close()
# dosemu command
cmd = 'dosemu -t -quiet -5 -E "%s"' % (batfile, )
sout, sin = popen2.popen2(cmd)
response = sout.read()
try:
f = file(outpath)
results = f.read()
os.remove(datpath)
os.remove(conpath)
os.remove(outpath)
os.remove(batfile)
os.rmdir(d)
self.datcnvoutput = results
return results
except:
f = file(os.path.join(d, 'terminal.log'), 'w+')
f.write(response)
raise ValueError(
"Error running datcnv.exe. See terminal.log in %s" % (d, ))
def Transfer(self, table, filter="True", orderby=None, limit=None):
cmop.debug("Transferring %s" % (table, ))
if not self.SubscriptionColumnExists(table):
cmop.debug(
"Subscription does not appear to be active; no subscription column found on %s"
% (table, ))
return
# get the data attributes
keys = self.fromdb.PrimaryKey(table)
if not keys:
raise TypeError(
"Table %s does not have a primary key defined; cannot transfer."
% (table, ))
keyattrsstr = ", ".join(keys)
getattrs = self.fromdb.Attributes(table, self.prefix)
setattrs = getattrs[:]
# if the reverse subscription is set, mark the new tuple as sent
if self.AttributeExists(self.todb, table, self.column(self.fromhost)):
setattrs.append('"%s"' % (self.column(self.fromhost), ))
val = ['False']
else:
val = []
getattrsstr = ", ".join(getattrs)
setattrsstr = ", ".join(setattrs)
get = '''SELECT %s FROM %s WHERE "%s" AND %s'''
if orderby: get += " ORDER BY %s" % (orderby, )
if limit: get += " LIMIT %s" % (limit, )
get = get % (getattrsstr, table, self.dirtycolumn, filter)
xfertable = table.replace(".", "_") + "_xfer"
create = '''CREATE TEMP TABLE %s AS (%s)''' % (xfertable, get)
insert = '''INSERT INTO %s (%s) VALUES %s''' % (xfertable, setattrsstr,
"%s")
drop = "DROP TABLE %s" % (xfertable, )
def equal(col):
return "%s.%s = %s.%s" % (table, col, xfertable, col)
joincond = " AND ".join([equal(col) for col in keys])
update = '''
UPDATE %s SET %s = False
FROM %s
WHERE %s
''' % (table, self.column(self.tohost), xfertable, joincond)
createremote = '''
CREATE TEMP TABLE %s AS (
SELECT * FROM %s LIMIT 0
)''' % (xfertable, table)
qualattrs = ", ".join(["%s.%s" % (xfertable, a) for a in setattrs])
merge = '''
INSERT INTO %s (%s) (
SELECT %s
FROM %s LEFT JOIN %s
ON (%s)
WHERE %s.%s IS NULL
)
''' % (table, setattrsstr, qualattrs, xfertable, table, joincond, table,
keys[0])
try:
self.fromdb.begin()
self.todb.begin()
cmop.debug("Creating xfer table %s on %s" %
(xfertable, self.fromhost))
self.fromdb.execCommand(create)
cmop.debug("Creating xfer table %s on %s" %
(xfertable, self.tohost))
self.todb.execCommand(createremote)
cmop.debug("Extracting values from %s on %s" %
(xfertable, self.fromhost))
values, cnt = self.ValuesClause(xfertable, val)
if cnt > 0:
cmop.debug("Inserting tuples on %s to %s" %
(self.tohost, xfertable))
self.todb.execCommand(insert % (values, ))
cmop.debug("Merging tuples on %s into %s" %
(self.tohost, table))
self.todb.execCommand(merge)
cmop.debug("Marking tuples as sent on %s for %s" %
(self.fromhost, table))
self.fromdb.execCommand(update)
cmop.debug("Dropping temp tables")
self.fromdb.execCommand(drop)
self.todb.execCommand(drop)
cmop.debug("Committing on %s" % (self.tohost, ))
self.todb.commit()
cmop.debug("Committing on %s" % (self.fromhost, ))
self.fromdb.commit()
except:
self.todb.rollback()
self.fromdb.rollback()
raise
cmop.info(
"Transferred %s %s tuples from %s to %s (may include dupes)" %
(cnt, table, self.fromhost, self.tohost))