def process(self, fname):
        cmop.debug("Seabird file found: %s" % (fname, ))
        cast = seabird.SeabirdCast(fname)
        cast.GetConverted()
        cast.GetBottle()

        # uncomment for speedier testing
        #f = file('sample.cnv')
        #cast.datcnvoutput = f.read()
        cast.Parse()
        cast.ParseBottle()
        record = self.makeCast(cast)

        cmop.debug("Parsed Seabird File %s" % (fname, ))
        # yield {"sqlcommand":"begin"}
        try:
            yield record
            for o in cast.IterateBottle(self.mapping['depth']):
                yield self.makeBottle(record, o)
            for o in cast.Iterate():
                yield self.makeObservation(record, o)

        #  yield {"sqlcommand":"commit"}
        except:
            pass
        # yield {"sqlcommand":"rollback"}

        cmop.info("Processed %s observations for file %s" %
                  (len(cast.rows), fname))
def isRunning(current,rootdir,sta,level):

  import glob
  if rootdir != 'archive':
     rootdir = 'test'
  testname = "/tmp/flat-cache-db2cdf-level%s-%s-%s-%s" % (level,current,sta,rootdir)
  pids = glob.glob("%s.*" % testname)
  instanceCount = 0

  for pid in pids:
    pid = pid.split(".")[1]

    # is running
    if os.path.exists("/proc/%s" % (pid)):
      instanceCount = instanceCount + 1
    # not running
    else:
      cmop.info("instance [%s] is death" % (pid))
      os.remove("%s.%s" % (testname,pid))

  if instanceCount==0:
    pidfile = open("%s.%d" % (testname,os.getpid()), "w")
    pidfile.close()

  return (instanceCount,testname)
    def Scan(self, filename):
        f = file(filename)
        hdr = self.ReadHeader(f)
        pos, item = self.state.get(filename, (0, 0))

        cmop.debug("Scanning %s as RecordStream" % (filename, ), 8)

        if pos == 0:
            pos = f.tell()

        opos, oitem = pos, item

        f.seek(pos)
        for x in self.ReadBlock(hdr, f):
            x['row'] = item
            item += 1
            # add a column for filename
            x['file'] = filename
            self.Enqueue(x)

        pos = f.tell()
        if item != oitem:
            cmop.info("Scanned %s records (%s bytes) from %s" %
                      (item - oitem, pos - opos, filename))

        self.state[filename] = (pos, item)
Example #4
0
 def Subscribe(self, table):
     # all new records will be marked for broadcast to current host
     if not self.Subscribed(table):
         msg = "SubscriptionBroker: %s subscribing to table %s on %s"
         cmop.info(msg % (self.tohost, table, self.fromhost))
         self.Configure(table)
         default = '''ALTER TABLE %s ALTER COLUMN "%s" SET DEFAULT True'''
         self.fromdb.execCommand(default % (table, self.dirtycolumn))
Example #5
0
 def UnSubscribe(self, table):
     # new records will not be marked for broadcast,
     # but the subscription column will not be dropped
     # no effect if subscription column does not exist
     if self.SubscriptionColumnExists(table):
         msg = "SubscriptionBroker: %s UNsubscribing from table %s on %s"
         cmop.info(msg % (self.tohost, table, self.fromhost))
         mod = '''ALTER TABLE %s ALTER COLUMN "%s" DROP DEFAULT'''
         self.fromdb.execCommand(mod % (table, self.dirtycolumn))
Example #6
0
 def Clean(self, table):
     # Remove the subscription column
     # Don't do this as part of normal operation;
     # dropped columns are not permanently removed!
     msg = "SubscriptionBroker: CLEAN CALLED: %s dropping column from table %s on %s"
     cmop.info(msg % (self.tohost, table, self.fromhost))
     mod = '''ALTER TABLE %s DROP COLUMN "%s"'''
     modind = '''DROP INDEX %s_%s'''
     self.fromdb.begin()
     try:
         self.fromdb.execCommand(mod % (table, self.dirtycolumn))
         self.fromdb.execCommand(
             modind % (table.replace(".", "_"), self.dirtycolumn))
         self.fromdb.commit()
     except:
         self.fromdb.rollback()
Example #7
0
    def ValuesClause(self, xfertable, extravals=[]):
        '''
Read all tuples from the given table and prepare a VALUES clause from the results
Returns a tuple of the values clause as a string and the number of tuples involved.
'''
        select = '''SELECT * FROM %s''' % (xfertable, )
        cmop.debug("Reading tuples to transfer.", 8)
        rs = self.fromdb.execQuery(select)
        cnt = len(rs)
        if not rs:
            cmop.info("No tuples to transfer.")
            return "()", 0

        def preprow(r):
            vals = ["%s" % (db.quote(a), ) for a in r]
            vals += extravals
            return "(%s)" % (", ".join(vals), )

        values = ", ".join([preprow(r) for r in rs])

        return values, cnt
Example #8
0
    def Configure(self, table):
        # Add a column on source table representing this subscription
        # also add an index
        if not self.SubscriptionColumnExists(table):
            if not self.tohost.Primarykey(table):
                msg = "Table %s does not have a primary key defined; cannot transfer."
                raise TypeError(msg % (table, ))

            msg = "SubscriptionBroker: Configuring %s subscribing to table %s on %s"
            cmop.info(msg % (self.tohost, table, self.fromhost))
            self.fromdb.begin()
            mod = '''ALTER TABLE %s ADD COLUMN "%s" bool; '''
            ind = '''CREATE INDEX "%s_%s" ON %s("%s")'''
            vac = '''VACUUM ANALYZE %s'''
            try:
                self.fromdb.execCommand(mod % (table, self.dirtycolumn))
                self.fromdb.execCommand(ind % (table.replace(
                    ".", "_"), self.dirtycolumn, table, self.dirtycolumn))
                self.fromdb.commit()
                self.fromdb.execCommand(vac % (table, ))
            except:
                self.fromdb.rollback()
                raise
    def process(self, data):
        try:
            tuple = {}

            self.GetLatLon(tuple, data)
            self.GetTime(tuple, data)

            tuple['table'] = "cruise.tsg"
            tuple['vessel'] = self.vessel
            tuple['cruise'] = self.cruise
            tuple['instrument'] = self.instrument
            tuple['instrumenttype'] = self.instrumenttype
            tuple['salinity'] = float(data['computed_salinity_flothru'])
            #tuple['temperature'] = float(data['water_temp_seabird_flothru'])
            tuple['temperature'] = float(data['surface_water_temp_seabird'])
            tuple['conductivity'] = float(data['conductivity_seabird_flothru'])
            tuple['winddirection'] = float(
                data.get('wind_heading_ultrasonic_true', -999999))
            tuple['windspeed'] = float(
                data.get('wind_speed_ultrasonic_true(knots)', -999999))
            tuple['atmosphericpressure'] = float(data['barometer'])
            tuple['atmospherictemperature'] = float(
                data['air_temp_rmyoung_doghouse'])

            for k, v in tuple.items():
                if str(v) == 'nan':
                    tuple[k] = None

            cmop.debug("WecomaTSGCleaner generated a tuple: %s" % (tuple, ), 8)
            return [tuple]

        except:
            msg = traceback.format_exc()
            s = ",".join(["(%s=%s)" % (k, v) for k, v in data.iteritems()])
            cmop.info("%s : Skipping bad TSG record: %s" % (msg, s))
            return []
     elif arg == 'archive':
        current = 'all'
     elif arg == 'test':
        rootdir = './test/'
     elif arg == 'level0':
        level = 0
     elif arg == 'level1':
        level = 1
     else:
        sta = ''
        stationsList.append([arg])
  print '%s %s %s %s' % (current,rootdir,level,sta)
  print stationsList
  test = isRunning(current,rootdir,sta,level)
  if test[0]!=0:
    cmop.info("%s is already running" % test[1])
    exit(0)
#  extractGliderData()
#  extractAUVData()
  currentPath = os.getcwd()
  sys.path[1:1] = [currentPath] 
  import stations as station 
  if sta == 'all':
     if current == 'current':
        cur = " where currentornull='current'"
     else: cur = ''
     sql = "select distinct on (station) station from instrument.offeringdetails %s" % cur
     db.execute(sql)
     stationsList = db.fetchall()  
  
  now = datetime.datetime.now()
Example #11
0
 def info(self, s):
   cmop.info(s)
Example #12
0
    def Convert(self, cfg="cmop.cfg"):
        '''Use dosemu to run Seabird data conversion.
       datcnv must run in the seabird directory, apparently
       datcnv is slow; prepare for about 260 scans per second
    '''

        dat = self.dat
        con = self.con

        cmop.info("Converting Seabird cast %s, %s" % (dat, con))

        root, base, ext = parsepath(self.dat)

        # file names must be short
        newname = "cast"

        # temp dir
        d = tempfile.mkdtemp('X', 'X')
        cmop.debug("tempdir: \n" + d)

        # dat file
        datpath = link(d, dat, newname)
        dosdat = dosabspath(datpath, self.drive)

        # con file
        conpath = link(d, con, newname)
        doscon = dosabspath(conpath, self.drive)

        # cfg file. cfg file path needs to
        #be relative to seabird directory, inexplicably

        #cfgpath = self.makeconfig(cfg, d)
        #doscfg = dosabspath(cfgpath, self.drive)

        # output file
        out = '%s.cnv' % (newname, )
        outpath = os.path.join(d, out)
        dosout = dosabspath(outpath, self.drive)

        # batch file
        bat = '''
%s:
cd %s 
datcnv.exe -ax -o%s -i%s -c%s -s -e%s
exitemu
'''
        batfile = os.path.join(d, 'seabird.bat')
        f = file(batfile, 'w+')
        dosseabirddir = dosabspath(self.seabirddir, self.drive)
        content = bat % (self.drive, dosseabirddir, dosout, dosdat, doscon,
                         cfg)
        cmop.debug("Batchfile: \n" + content)
        f.write(content)
        f.close()

        # dosemu command
        cmd = 'dosemu -t -quiet -5 -E "%s"' % (batfile, )

        sout, sin = popen2.popen2(cmd)
        response = sout.read()

        try:
            f = file(outpath)
            results = f.read()

            os.remove(datpath)
            os.remove(conpath)
            os.remove(outpath)
            os.remove(batfile)
            os.rmdir(d)

            self.datcnvoutput = results
            return results

        except:
            f = file(os.path.join(d, 'terminal.log'), 'w+')
            f.write(response)
            raise ValueError(
                "Error running datcnv.exe.  See terminal.log in %s" % (d, ))
Example #13
0
    def Transfer(self, table, filter="True", orderby=None, limit=None):
        cmop.debug("Transferring %s" % (table, ))

        if not self.SubscriptionColumnExists(table):
            cmop.debug(
                "Subscription does not appear to be active; no subscription column found on %s"
                % (table, ))
            return

        # get the data attributes
        keys = self.fromdb.PrimaryKey(table)
        if not keys:
            raise TypeError(
                "Table %s does not have a primary key defined; cannot transfer."
                % (table, ))

        keyattrsstr = ", ".join(keys)
        getattrs = self.fromdb.Attributes(table, self.prefix)
        setattrs = getattrs[:]

        # if the reverse subscription is set, mark the new tuple as sent
        if self.AttributeExists(self.todb, table, self.column(self.fromhost)):
            setattrs.append('"%s"' % (self.column(self.fromhost), ))
            val = ['False']
        else:
            val = []

        getattrsstr = ", ".join(getattrs)
        setattrsstr = ", ".join(setattrs)

        get = '''SELECT %s FROM %s WHERE "%s" AND %s'''
        if orderby: get += " ORDER BY %s" % (orderby, )
        if limit: get += " LIMIT %s" % (limit, )
        get = get % (getattrsstr, table, self.dirtycolumn, filter)

        xfertable = table.replace(".", "_") + "_xfer"

        create = '''CREATE TEMP TABLE %s AS  (%s)''' % (xfertable, get)

        insert = '''INSERT INTO %s (%s) VALUES %s''' % (xfertable, setattrsstr,
                                                        "%s")

        drop = "DROP TABLE %s" % (xfertable, )

        def equal(col):
            return "%s.%s = %s.%s" % (table, col, xfertable, col)

        joincond = " AND ".join([equal(col) for col in keys])
        update = '''
UPDATE %s SET %s = False
  FROM %s
 WHERE %s
''' % (table, self.column(self.tohost), xfertable, joincond)

        createremote = '''
CREATE TEMP TABLE %s AS (
SELECT * FROM %s LIMIT 0
)''' % (xfertable, table)

        qualattrs = ", ".join(["%s.%s" % (xfertable, a) for a in setattrs])
        merge = '''
INSERT INTO %s (%s) ( 
  SELECT %s
    FROM %s LEFT JOIN %s
         ON (%s) 
   WHERE %s.%s IS NULL
)
''' % (table, setattrsstr, qualattrs, xfertable, table, joincond, table,
        keys[0])

        try:
            self.fromdb.begin()
            self.todb.begin()

            cmop.debug("Creating xfer table %s on %s" %
                       (xfertable, self.fromhost))
            self.fromdb.execCommand(create)

            cmop.debug("Creating xfer table %s on %s" %
                       (xfertable, self.tohost))
            self.todb.execCommand(createremote)

            cmop.debug("Extracting values from %s on %s" %
                       (xfertable, self.fromhost))
            values, cnt = self.ValuesClause(xfertable, val)

            if cnt > 0:
                cmop.debug("Inserting tuples on %s to %s" %
                           (self.tohost, xfertable))
                self.todb.execCommand(insert % (values, ))

                cmop.debug("Merging tuples on %s into %s" %
                           (self.tohost, table))
                self.todb.execCommand(merge)

                cmop.debug("Marking tuples as sent on %s for %s" %
                           (self.fromhost, table))
                self.fromdb.execCommand(update)

                cmop.debug("Dropping temp tables")
                self.fromdb.execCommand(drop)
                self.todb.execCommand(drop)

                cmop.debug("Committing on %s" % (self.tohost, ))
                self.todb.commit()

                cmop.debug("Committing on %s" % (self.fromhost, ))
                self.fromdb.commit()

        except:
            self.todb.rollback()
            self.fromdb.rollback()
            raise

        cmop.info(
            "Transferred %s %s tuples from %s to %s (may include dupes)" %
            (cnt, table, self.fromhost, self.tohost))