def Scan(self, filename):
f = file(filename)
hdr = self.ReadHeader(f)
pos, item = self.state.get(filename, (0, 0))
cmop.debug("Scanning %s as RecordStream" % (filename, ), 8)
if pos == 0:
pos = f.tell()
opos, oitem = pos, item
f.seek(pos)
for x in self.ReadBlock(hdr, f):
x['row'] = item
item += 1
# add a column for filename
x['file'] = filename
self.Enqueue(x)
pos = f.tell()
if item != oitem:
cmop.info("Scanned %s records (%s bytes) from %s" %
(item - oitem, pos - opos, filename))
self.state[filename] = (pos, item)
def process(self, fname):
cmop.debug("Seabird file found: %s" % (fname, ))
cast = seabird.SeabirdCast(fname)
cast.GetConverted()
cast.GetBottle()
# uncomment for speedier testing
#f = file('sample.cnv')
#cast.datcnvoutput = f.read()
cast.Parse()
cast.ParseBottle()
record = self.makeCast(cast)
cmop.debug("Parsed Seabird File %s" % (fname, ))
# yield {"sqlcommand":"begin"}
try:
yield record
for o in cast.IterateBottle(self.mapping['depth']):
yield self.makeBottle(record, o)
for o in cast.Iterate():
yield self.makeObservation(record, o)
# yield {"sqlcommand":"commit"}
except:
pass
# yield {"sqlcommand":"rollback"}
cmop.info("Processed %s observations for file %s" %
(len(cast.rows), fname))
def Watch(self):
time.sleep(1)
currdirs = self.dirs.copy()
cmop.debug("Watching directory: %s" % (currdirs, ))
def PhonyEvent(event):
cmop.debug("Triggering Phony Event %s" % (event, ), 6)
for d in currdirs:
dispatch = self.MakeDispatch(d)
try:
cmop.debug("Checking directory %s" % (d, ), 6)
for f in os.listdir(d):
cmop.debug("Found file %s" % (os.path.join(d, f), ), 6)
dispatch(os.path.join(d, f), event)
except:
cmop.error(
"Error watching directory: %s" %
("".join(traceback.format_exception(*sys.exc_info()))))
PhonyEvent(gamin.GAMExists)
while True:
# while self.mon.event_pending():
if ErrorRaised(): sys.exit()
# self.mon.handle_one_event()
time.sleep(self.pollingdelay)
PhonyEvent(gamin.GAMChanged)
def run(self):
uniquemessages = set([])
while True:
try:
# consume an item, but throw an exception if queue is empty
data = self.source.Dequeue(False)
uniquemessages.add(data)
except Queue.Empty:
cmop.debug("Coalesced %s messages" % (len(uniquemessages), ),
8)
for m in uniquemessages:
self.RunProcess(m)
uniquemessages.clear()
time.sleep(self.sleeptime)
def process(self, data):
tup = {}
if 'sqlcommand' in data:
cmd = data['sqlcommand']
self.db.execCommand(cmd)
return []
table = data.pop('table')
for k, v in data.items():
tup[k] = db.quote(v)
tup = self.clean(tup)
try:
yes = self.db.InsertTuple(table, tup.keys(), tup.values())
cmop.debug("Inserted Tuple: %s" % (tup, ), 9)
except cmop.db.DuplicateError:
cmop.debug("Duplicate record: %s" % (tup, ), 8)
return [tup]
def WatchNew(self):
time.sleep(1)
currdirs = self.dirs.copy()
cmop.debug("Watching new directory: %s" % (currdirs, ))
def PhonyEvent(event):
try:
for d in currdirs:
dispatch = self.MakeDispatch(d)
for f in os.listdir(d):
path = os.path.join(d, f)
dispatch(path, event)
except:
cmop.error(
"Error watching new directories: %s" %
("".join(traceback.format_exception(*sys.exc_info()))))
while True:
PhonyEvent(gamin.GAMExists)
cmop.debug("sleeping %s seconds" % (self.pollingdelay, ))
time.sleep(self.pollingdelay)
def ValuesClause(self, xfertable, extravals=[]):
'''
Read all tuples from the given table and prepare a VALUES clause from the results
Returns a tuple of the values clause as a string and the number of tuples involved.
'''
select = '''SELECT * FROM %s''' % (xfertable, )
cmop.debug("Reading tuples to transfer.", 8)
rs = self.fromdb.execQuery(select)
cnt = len(rs)
if not rs:
cmop.info("No tuples to transfer.")
return "()", 0
def preprow(r):
vals = ["%s" % (db.quote(a), ) for a in r]
vals += extravals
return "(%s)" % (", ".join(vals), )
values = ", ".join([preprow(r) for r in rs])
return values, cnt
def process(self, data):
try:
tuple = {}
self.GetLatLon(tuple, data)
self.GetTime(tuple, data)
tuple['table'] = "cruise.tsg"
tuple['vessel'] = self.vessel
tuple['cruise'] = self.cruise
tuple['instrument'] = self.instrument
tuple['instrumenttype'] = self.instrumenttype
tuple['salinity'] = float(data['computed_salinity_flothru'])
#tuple['temperature'] = float(data['water_temp_seabird_flothru'])
tuple['temperature'] = float(data['surface_water_temp_seabird'])
tuple['conductivity'] = float(data['conductivity_seabird_flothru'])
tuple['winddirection'] = float(
data.get('wind_heading_ultrasonic_true', -999999))
tuple['windspeed'] = float(
data.get('wind_speed_ultrasonic_true(knots)', -999999))
tuple['atmosphericpressure'] = float(data['barometer'])
tuple['atmospherictemperature'] = float(
data['air_temp_rmyoung_doghouse'])
for k, v in tuple.items():
if str(v) == 'nan':
tuple[k] = None
cmop.debug("WecomaTSGCleaner generated a tuple: %s" % (tuple, ), 8)
return [tuple]
except:
msg = traceback.format_exc()
s = ",".join(["(%s=%s)" % (k, v) for k, v in data.iteritems()])
cmop.info("%s : Skipping bad TSG record: %s" % (msg, s))
return []
def Parse(self):
'''Parse the content of a datcnv output file. Expects datcnv file to have been read into datcnvoutput attribute.'''
lines = self.datcnvoutput.split('\n')
for line in lines:
line = line.replace('\r', '')
if re.match('^\*', line):
# header line
for key, expr in hdr.items():
d = re.search(expr, line)
if d:
self.__dict__[key] = d.group(1).strip()
cmop.debug("HDR: %s=%s" % (key, self.__dict__[key]))
break
elif re.match('^\#', line):
# metadata line
col = re.search('# (.+) (\d+) = (.+)', line)
if col:
cmop.debug("COL: %s" % (col.groups(), ))
self.__dict__.setdefault(col.group(1).strip(), {})
self.__dict__[col.group(1)][int(
col.group(2))] = col.group(3)
else:
attr = re.search('# (.+) = (.+)', line)
if attr:
cmop.debug("ATTR: %s" % (attr.groups(), ))
self.__dict__[attr.group(1)] = attr.group(2).strip()
else:
#unrecognized line
cmop.debug("unrecognized line format: %s" % (line, ))
else:
# data line
row = re.sub('^\s+', '', line)
row = re.sub('\s+$', '', row)
# make sure the line isn't blank
if row:
row = [r for r in re.split('\s+', row)]
self.rows.append(row)
def PhonyEvent(event):
cmop.debug("Triggering Phony Event %s" % (event, ), 6)
for d in currdirs:
dispatch = self.MakeDispatch(d)
try:
cmop.debug("Checking directory %s" % (d, ), 6)
for f in os.listdir(d):
cmop.debug("Found file %s" % (os.path.join(d, f), ), 6)
dispatch(os.path.join(d, f), event)
except:
cmop.error(
"Error watching directory: %s" %
("".join(traceback.format_exception(*sys.exc_info()))))
def MakeImage():
# save the active figure to a tempfile
# should be able to pass in a file object, but alas
cmop.debug("Creating tempfile...", 5)
(f, name) = tempfile.mkstemp(".png", dir=fullwritedir)
os.close(f)
cmop.debug("Saving image to tempfile %s" % (name, ), 5)
savefig(name, format='png')
# close the active figure
close()
f = file(name)
img = f.read()
f.close()
os.remove(name)
cmop.debug("Returning image...", 5)
return img
def debug(self, s, level=10): cmop.debug(s, level)
def GetTime(self, seacast, cast):
'''
Deduce the time of the cast.
Set the 'time' key on the cast dictionary to an
expression that can be evaluated by Postgresql
'''
if hasattr(seacast, 'NMEAtime'):
# System UpLoad Time is in local time and comes
# from the system clock of the machine running
# seasave
# NMEA time comes from the device, buit has no date
# information.
# must assume a bound on clock skew to guess the right date
test_time1 = dateutil.parser.parse(seacast.NMEAtime,
default=datetime.datetime(
year=1, month=2, day=3))
test_time5 = dateutil.parser.parse(seacast.NMEAtime,
default=datetime.datetime(
year=5, month=1, day=1))
test = test_time1 - test_time5
cmop.debug('NMEAtime %s' % seacast.NMEAtime)
if test == datetime.timedelta(
0): # NMEAtime parses as a full date/time
cast['time'] = "%s %s" % (seacast.NMEAtime, 'UTC')
cmop.debug('full NMEA %s' % cast['time'])
return
if hasattr(seacast, 'start_time'):
cmop.debug('start time %s' % seacast.start_time)
start_time = dateutil.parser.parse(seacast.start_time)
NMEAtime = dateutil.parser.parse(seacast.NMEAtime)
comb_time = datetime.datetime.combine(start_time.date(),
NMEAtime.time())
if start_time > comb_time + datetime.timedelta(0, 43200):
comb_time += datetime.timedelta(1)
elif comb_time > start_time + datetime.timedelta(0, 43200):
comb_time -= datetime.timedelta(1)
cast['time'] = "%s %s" % (comb_time, 'UTC')
cmop.debug('NMEA start %s' % cast['time'])
return
if hasattr(seacast, 'uploadtime'):
cmop.debug('upload time %s' % seacast.uploadtime)
uploadtime = dateutil.parser.parse(seacast.uploadtime)
NMEAtime = dateutil.parser.parse(seacast.NMEAtime)
comb_time = datetime.datetime.combine(uploadtime.date(),
NMEAtime.time())
if uploadtime > comb_time + datetime.timedelta(0, 43200):
comb_time += datetime.timedelta(1)
elif comb_time > uploadtime + datetime.timedelta(0, 43200):
comb_time -= datetime.timedelta(1)
cast['time'] = "%s %s" % (comb_time, 'UTC')
cmop.debug('NMEA upload %s' % cast['time'])
return
if hasattr(seacast, 'GMTtime') and hasattr(seacast, 'GMTdate'):
time = seacast.GMTtime
time.replace(' ', '')
cmop.debug('trying to merge %s and %s' %
(seacast.GMTdate, seacast.GMTtime))
if re.search('^\d{4}', seacast.GMTtime.strip()):
time = datetime.time(int(time[0:2]), int(time[2:4]))
date = seacast.GMTdate
try:
date = int(date)
date = datetime.date.fromordinal(date)
except:
date = dateutil.parser.parse(date)
try:
cmop.debug('preparing to merge %s and %s' % (date, time))
date = datetime.datetime.combine(date, time)
cmop.debug('converted %s to %s' % (seacast.GMTdate, date))
if hasattr(seacast, 'start_time'):
cmop.debug('start time %s' % seacast.start_time)
altdate = dateutil.parser.parse(seacast.start_time)
elif hasattr(seacast, 'uploadtime'):
cmop.debug('upload time %s' % seacast.uploadtime)
altdate = dateutil.parser.parse(seacast.uploadtime)
date = date.replace(year=altdate.year)
if altdate > date + datetime.timedelta(100):
date.replace(year=date.year - 1)
elif date > altdate + datetime.timedelta(100):
date.replace(year=date.year + 1)
cast['time'] = "%s %s" % (date, 'UTC')
cmop.debug('GMT time %s' % cast['time'])
return
except:
raise TypeError('could not convert %s' % seacast.GMTdate)
if hasattr(seacast, 'uploadtime'):
cast['time'] = "%s %s" % (seacast.uploadtime, self.timezone)
cmop.debug('upload time %s' % cast['time'])
return
if hasattr(seacast, 'start_time'):
cast['time'] = "%s %s" % (seacast.start_time, self.timezone)
cmop.debug('start time %s' % cast['time'])
return
raise ValueError(
"Cannot calculate cast date: Need either GMTdate and GMTtime, or System Upload Time and NMEA Time. \n%s"
% (seacast.__dict__.keys(), ))
def Convert(self, cfg="cmop.cfg"):
'''Use dosemu to run Seabird data conversion.
datcnv must run in the seabird directory, apparently
datcnv is slow; prepare for about 260 scans per second
'''
dat = self.dat
con = self.con
cmop.info("Converting Seabird cast %s, %s" % (dat, con))
root, base, ext = parsepath(self.dat)
# file names must be short
newname = "cast"
# temp dir
d = tempfile.mkdtemp('X', 'X')
cmop.debug("tempdir: \n" + d)
# dat file
datpath = link(d, dat, newname)
dosdat = dosabspath(datpath, self.drive)
# con file
conpath = link(d, con, newname)
doscon = dosabspath(conpath, self.drive)
# cfg file. cfg file path needs to
#be relative to seabird directory, inexplicably
#cfgpath = self.makeconfig(cfg, d)
#doscfg = dosabspath(cfgpath, self.drive)
# output file
out = '%s.cnv' % (newname, )
outpath = os.path.join(d, out)
dosout = dosabspath(outpath, self.drive)
# batch file
bat = '''
%s:
cd %s
datcnv.exe -ax -o%s -i%s -c%s -s -e%s
exitemu
'''
batfile = os.path.join(d, 'seabird.bat')
f = file(batfile, 'w+')
dosseabirddir = dosabspath(self.seabirddir, self.drive)
content = bat % (self.drive, dosseabirddir, dosout, dosdat, doscon,
cfg)
cmop.debug("Batchfile: \n" + content)
f.write(content)
f.close()
# dosemu command
cmd = 'dosemu -t -quiet -5 -E "%s"' % (batfile, )
sout, sin = popen2.popen2(cmd)
response = sout.read()
try:
f = file(outpath)
results = f.read()
os.remove(datpath)
os.remove(conpath)
os.remove(outpath)
os.remove(batfile)
os.rmdir(d)
self.datcnvoutput = results
return results
except:
f = file(os.path.join(d, 'terminal.log'), 'w+')
f.write(response)
raise ValueError(
"Error running datcnv.exe. See terminal.log in %s" % (d, ))
def Transfer(self, table, filter="True", orderby=None, limit=None):
cmop.debug("Transferring %s" % (table, ))
if not self.SubscriptionColumnExists(table):
cmop.debug(
"Subscription does not appear to be active; no subscription column found on %s"
% (table, ))
return
# get the data attributes
keys = self.fromdb.PrimaryKey(table)
if not keys:
raise TypeError(
"Table %s does not have a primary key defined; cannot transfer."
% (table, ))
keyattrsstr = ", ".join(keys)
getattrs = self.fromdb.Attributes(table, self.prefix)
setattrs = getattrs[:]
# if the reverse subscription is set, mark the new tuple as sent
if self.AttributeExists(self.todb, table, self.column(self.fromhost)):
setattrs.append('"%s"' % (self.column(self.fromhost), ))
val = ['False']
else:
val = []
getattrsstr = ", ".join(getattrs)
setattrsstr = ", ".join(setattrs)
get = '''SELECT %s FROM %s WHERE "%s" AND %s'''
if orderby: get += " ORDER BY %s" % (orderby, )
if limit: get += " LIMIT %s" % (limit, )
get = get % (getattrsstr, table, self.dirtycolumn, filter)
xfertable = table.replace(".", "_") + "_xfer"
create = '''CREATE TEMP TABLE %s AS (%s)''' % (xfertable, get)
insert = '''INSERT INTO %s (%s) VALUES %s''' % (xfertable, setattrsstr,
"%s")
drop = "DROP TABLE %s" % (xfertable, )
def equal(col):
return "%s.%s = %s.%s" % (table, col, xfertable, col)
joincond = " AND ".join([equal(col) for col in keys])
update = '''
UPDATE %s SET %s = False
FROM %s
WHERE %s
''' % (table, self.column(self.tohost), xfertable, joincond)
createremote = '''
CREATE TEMP TABLE %s AS (
SELECT * FROM %s LIMIT 0
)''' % (xfertable, table)
qualattrs = ", ".join(["%s.%s" % (xfertable, a) for a in setattrs])
merge = '''
INSERT INTO %s (%s) (
SELECT %s
FROM %s LEFT JOIN %s
ON (%s)
WHERE %s.%s IS NULL
)
''' % (table, setattrsstr, qualattrs, xfertable, table, joincond, table,
keys[0])
try:
self.fromdb.begin()
self.todb.begin()
cmop.debug("Creating xfer table %s on %s" %
(xfertable, self.fromhost))
self.fromdb.execCommand(create)
cmop.debug("Creating xfer table %s on %s" %
(xfertable, self.tohost))
self.todb.execCommand(createremote)
cmop.debug("Extracting values from %s on %s" %
(xfertable, self.fromhost))
values, cnt = self.ValuesClause(xfertable, val)
if cnt > 0:
cmop.debug("Inserting tuples on %s to %s" %
(self.tohost, xfertable))
self.todb.execCommand(insert % (values, ))
cmop.debug("Merging tuples on %s into %s" %
(self.tohost, table))
self.todb.execCommand(merge)
cmop.debug("Marking tuples as sent on %s for %s" %
(self.fromhost, table))
self.fromdb.execCommand(update)
cmop.debug("Dropping temp tables")
self.fromdb.execCommand(drop)
self.todb.execCommand(drop)
cmop.debug("Committing on %s" % (self.tohost, ))
self.todb.commit()
cmop.debug("Committing on %s" % (self.fromhost, ))
self.fromdb.commit()
except:
self.todb.rollback()
self.fromdb.rollback()
raise
cmop.info(
"Transferred %s %s tuples from %s to %s (may include dupes)" %
(cnt, table, self.fromhost, self.tohost))