Exemplo n.º 1
0
def getNameMMSI(logfile,outfile):
    for line in logfile:
        fields = line.split(',')[:6]
        if '1'!=fields[2]: # Must be the start of a sequence
            continue
        if len(fields[5])<39: continue
        bv = binary.ais6tobitvec(fields[5][:39]) # Hacked for speed

        mmsi = ais_msg_5.decodeUserID(bv)
        name = aisstring.unpad(ais_msg_5.decodename(bv))
        outfile.write(str(mmsi)+' '+str(name)+'\n')
Exemplo n.º 2
0
def parse_msgs(infile, verbose=False):
    for line in infile:
        line = line.strip()

        try:
            match = uscg_ais_nmea_regex.search(line).groupdict()
        except AttributeError:
            continue

        msg_type = match['body'][0]
        if msg_type not in ('6', '8'):
            continue

        if msg_type == '6' and len(match['body']) < 15:
            continue
        if msg_type == '8' and len(match['body']) < 10:
            continue

        try:
            bv = binary.ais6tobitvec(match['body'][:15])
        except ValueError:
            sys.stderr.write('bad msg: %s\n' % line.strip())
            continue

        r = {}
        r['MessageID']=int(bv[0:6])
        r['UserID']=int(bv[8:38])

        if '6' == msg_type:
            dac = int(bv[72:82])
            fi = int(bv[82:88])
        elif '8' == msg_type:
            dac = int(bv[40:50])
            fi = int(bv[50:56])
        elif verbose:
            print 'not a bbm:', line

        if verbose:
            print msg_type, dac, fi, r['UserID'], line.rstrip()
        else:
            print msg_type, dac, fi, r['UserID'], match['station']
Exemplo n.º 3
0
def getPosition(logfile, outfile, minDist=None):
    '''
    Pull the positions from the log file
    @param logfile: file like object
    @param outfile: file like object destination
    @param minDist: how far apart points must be apart to be considered unique
    '''
    # FIX: use the right utm zone.  14 is the central US so it will kind of work
    params = {'proj':'utm', 'zone':14} #int(options.zone)}
    proj = None
    if minDist != None:
        proj = Proj(params)

    positions = {} # Last recoded ship position

    for line in logfile:
        fields = line.split(',')
        # FIX: use regex instead
        if '!AIVDM' != fields[0]:
            continue
        if '1' != fields[1] and '1' != fields[2]: # Must be the start of a sequence
            continue
        if fields[5][0] not in ('1','2','3'):
            continue

        bv = binary.ais6tobitvec(fields[5][:39]) # Hacked for speed

        timestamp = fields[-1].strip()
        mmsi = str(ais_msg_1.decodeUserID(bv))
        lon = ais_msg_1.decodelongitude(bv)
        lat = ais_msg_1.decodelatitude(bv)

        d = None
        if mmsi not in positions:
            positions[mmsi] = (lon, lat)
        elif minDist != None:
            lonOld, latOld = positions[mmsi]
            oldUTM = proj(lonOld, latOld)
            newUTM = proj(lon, lat)
            d = dist(oldUTM[0], oldUTM[1], newUTM[0], newUTM[1])
            if str(d)=='nan':
                continue #pass  # FIX: Print but do not save nan values???
            elif d < minDist:
                continue
            else:
                positions[mmsi] = (lon, lat)

        lon = str(lon)
        lat = str(lat)

        if len(mmsi) < 9:
            mmsi += ' '*(9-len(mmsi))

        fLen = 12 # field length ... how much space

        if len(lon)>fLen:
            lon = lon[:fLen]
        if len(lon)<fLen:
            lon += ' '*(fLen-len(lon))

        if len(lat)>fLen:
            lat = lat[:fLen]
        if len(lat)<fLen:
            lat += ' '*(fLen-len(lat))

        outfile.write(timestamp+' '+str(mmsi)+' '+lon+' '+lat+'\n')
Exemplo n.º 4
0
def build_dist_database(database_filename, log_files, verbose=False):

    cx = sqlite3.connect(database_filename)

    print 'WARNING: not saving the station name'
    cx.execute('''
    CREATE TABLE IF NOT EXISTS distance (
       -- Save space, no key
       -- ts INTEGER, -- Save more space
       julian_day INTEGER,
       -- x REAL,
       -- y REAL,
       dist_km REAL
       --,
       --station VARCHAR(15)
       );
    ''')

    cu = cx.cursor()

    counts = {'nogps': 0}

    for filename in log_files:
        if verbose:
            print 'file:', filename
            sys.stdout.flush()
        for line_num, line in enumerate(file(filename)):
            if 'AIVDM,1,1' not in line: continue
            match = uscg_ais_nmea_regex.search(line).groupdict()
            message_id = match['body'][0]  # First letter is the message type
            if message_id not in ('1', '2', '3'): continue

            if len(match['body']) != 28:  # 6 bits per character
                raise AisErrorBadNumBits('expected 168, got %d' %
                                         len(match['body']) / 6)

            bits = binary.ais6tobitvec(
                match['body'][:20]
            )  # Don't need any of the other bits, so do not waste time

            x = binary.signedIntFromBV(bits[61:89]) / 600000.
            y = binary.signedIntFromBV(bits[89:116]) / 600000.

            if x > 180 or y > 90:
                counts['nogps'] += 1
                continue

            station = match['station']

            julian_day = int(
                datetime.datetime.utcfromtimestamp(int(
                    match['timeStamp'])).strftime('%j'))

            d_km = dist_utm_km((x, y), station_locations[station])
            #cu.execute('INSERT INTO distance VALUES (:julian_day, :x, :y, :dist_km, :station)',
            #{'julian_day': julian_day, 'x':x, 'y':y, 'dist_km': d_km, 'station':station} )
            #cu.execute('INSERT INTO distance VALUES (:julian_day, :x, :y, :dist_km)',
            #           {'julian_day': julian_day, 'x':x, 'y':y, 'dist_km': d_km, } )
            cu.execute('INSERT INTO distance VALUES (:julian_day, :dist_km)', {
                'julian_day': julian_day,
                'dist_km': d_km,
            })
            if line_num % 10000 == 9999:
                cx.commit()

        cx.commit()

    if False:
        print 'Creating indexes'
        try:
            cx.execute('CREATE INDEX idx_dist_day ON distance(julian_day);')
            cx.execute('CREATE INDEX idx_dist_dist ON distance(dist_km);')
            #cx.execute('CREATE INDEX idx_dist_station ON distance(station);')
            cx.commit()
        except sqlite3.OperationalError:
            print 'Appears indexes were already created'

    return cx, counts
Exemplo n.º 5
0
def decode_aivdm(msg):
    bv = binary.ais6tobitvec(msg.split(',')[5])
    print decode(bv)
Exemplo n.º 6
0
        buf = StringIO.StringIO()
        for field in options.fieldList:
            buf.write(field + ',')
        result = buf.getvalue()
        if result[-1] == ',': print result[:-1]
        else: print result

    if options.doDecode:
        if len(args) == 0: args = sys.stdin
        for msg in args:
            bv = None

            if msg[0] in ('$', '!') and msg[3:6] in ('VDM', 'VDO'):
                # Found nmea
                # FIX: do checksum
                bv = binary.ais6tobitvec(msg.split(',')[5])
            else:  # either binary or nmeapayload... expect mostly nmeapayloads
                # assumes that an all 0 and 1 string can not be a nmeapayload
                binaryMsg = True
                for c in msg:
                    if c not in ('0', '1'):
                        binaryMsg = False
                        break
                if binaryMsg:
                    bv = BitVector(bitstring=msg)
                else:  # nmeapayload
                    bv = binary.ais6tobitvec(msg)

            params = decode(bv)
            #print 'params:',params
            printFields(params,
Exemplo n.º 7
0
def main():
    from optparse import OptionParser
    parser = OptionParser(usage="%prog [options]")

    parser.add_option('--doc-test',dest='doctest',default=False,action='store_true',
        help='run the documentation tests')
    parser.add_option('--unit-test',dest='unittest',default=False,action='store_true',
        help='run the unit tests')
    parser.add_option('-v','--verbose',dest='verbose',default=False,action='store_true',
        help='Make the test output verbose')

    # FIX: remove nmea from binary messages.  No way to build the whole packet?
    # FIX: or build the surrounding msg 8 for a broadcast?
    typeChoices = ('binary','nmeapayload','nmea') # FIX: what about a USCG type message?
    parser.add_option('-t', '--type', choices=typeChoices, type='choice',
        dest='ioType', default='nmeapayload',
        help='What kind of string to write for encoding ('+', '.join(typeChoices)+') [default: %default]')


    outputChoices = ('std','html','csv','sql' , 'kml','kml-full')
    parser.add_option('-T', '--output-type', choices=outputChoices,
        type='choice', dest='outputType', default='std',
        help='What kind of string to output ('+', '.join(outputChoices)+') '
        '[default: %default]')

    parser.add_option('-o','--output',dest='outputFileName',default=None,
        help='Name of the python file to write [default: stdout]')

    parser.add_option('-f', '--fields', dest='fieldList', default=None,
        action='append', choices=fieldList,
        help='Which fields to include in the output.  Currently only for csv '
        'output [default: all]')

    parser.add_option('-p', '--print-csv-field-list', dest='printCsvfieldList',
        default=False,action='store_true',
        help='Print the field name for csv')

    parser.add_option('-c', '--sql-create', dest='sqlCreate', default=False,
        action='store_true',
        help='Print out an sql create command for the table.')

    parser.add_option('--latex-table', dest='latexDefinitionTable',
        default=False,action='store_true',
        help='Print a LaTeX table of the type')

    parser.add_option('--text-table', dest='textDefinitionTable', default=False,
        action='store_true',
        help='Print delimited table of the type (for Word table importing)')

    parser.add_option('--delimt-text-table', dest='delimTextDefinitionTable',
        default='    ',
        help='Delimiter for text table [default: \'%default\'] '
        '(for Word table importing)')

    dbChoices = ('sqlite','postgres')
    parser.add_option('-D', '--db-type', dest='dbType', default='postgres',
        choices=dbChoices,type='choice',
        help='What kind of database ('+', '.join(dbChoices)+') '
        '[default: %default]')

    addMsgOptions(parser)

    (options,args) = parser.parse_args()
    success = True

    if options.doctest:
            import os; print os.path.basename(sys.argv[0]), 'doctests ...',
            sys.argv = [sys.argv[0]]
            if options.verbose:
              sys.argv.append('-v')

            numfail, numtests = doctest.testmod()
            if not numfail:
                print 'ok'
            else:
                print 'FAILED'
                success = False

    if not success: sys.exit('Something Failed')
    del success # Hide success from epydoc

    if options.unittest:
            sys.argv = [sys.argv[0]]
            if options.verbose: sys.argv.append('-v')
            unittest.main()

    outfile = sys.stdout
    if None!=options.outputFileName:
            outfile = file(options.outputFileName,'w')


    if options.doEncode:
        # Make sure all non required options are specified.
        if None==options.time_monthField: parser.error("missing value for time_monthField")
        if None==options.time_dayField: parser.error("missing value for time_dayField")
        if None==options.time_hourField: parser.error("missing value for time_hourField")
        if None==options.time_minField: parser.error("missing value for time_minField")
        if None==options.stationidField: parser.error("missing value for stationidField")
        if None==options.pos_longitudeField: parser.error("missing value for pos_longitudeField")
        if None==options.pos_latitudeField: parser.error("missing value for pos_latitudeField")
        if None==options.flowField: parser.error("missing value for flowField")
    msgDict = {
        'time_month': options.time_monthField,
        'time_day': options.time_dayField,
        'time_hour': options.time_hourField,
        'time_min': options.time_minField,
        'stationid': options.stationidField,
        'pos_longitude': options.pos_longitudeField,
        'pos_latitude': options.pos_latitudeField,
        'flow': options.flowField,
        'reserved': '0',
    }

    bits = encode(msgDict)
    if 'binary' == options.ioType:
        print str(bits)
    elif 'nmeapayload'==options.ioType:
        # FIX: figure out if this might be necessary at compile time
        bitLen=len(bits)
        if bitLen % 6 != 0:
            bits = bits + BitVector(size=(6 - (bitLen%6)))  # Pad out to multiple of 6
        print binary.bitvectoais6(bits)[0]

    # FIX: Do not emit this option for the binary message payloads.  Does not make sense.
    elif 'nmea' == options.ioType:
        nmea = uscg.create_nmea(bits)
        print nmea
    else:
        sys.exit('ERROR: unknown ioType.  Help!')


        if options.sqlCreate:
                sqlCreateStr(outfile,options.fieldList,dbType=options.dbType)

        if options.latexDefinitionTable:
                latexDefinitionTable(outfile)

        # For conversion to word tables
        if options.textDefinitionTable:
                textDefinitionTable(outfile,options.delimTextDefinitionTable)

        if options.printCsvfieldList:
                # Make a csv separated list of fields that will be displayed for csv
                if None == options.fieldList: options.fieldList = fieldList
                import StringIO
                buf = StringIO.StringIO()
                for field in options.fieldList:
                        buf.write(field+',')
                result = buf.getvalue()
                if result[-1] == ',': print result[:-1]
                else: print result

        if options.doDecode:
                if len(args)==0: args = sys.stdin
                for msg in args:
                        bv = None

                        if msg[0] in ('$','!') and msg[3:6] in ('VDM','VDO'):
                                # Found nmea
                                # FIX: do checksum
                                bv = binary.ais6tobitvec(msg.split(',')[5])
                        else: # either binary or nmeapayload... expect mostly nmeapayloads
                                # assumes that an all 0 and 1 string can not be a nmeapayload
                                binaryMsg=True
                                for c in msg:
                                        if c not in ('0','1'):
                                                binaryMsg=False
                                                break
                                if binaryMsg:
                                        bv = BitVector(bitstring=msg)
                                else: # nmeapayload
                                        bv = binary.ais6tobitvec(msg)

                        printFields(decode(bv)
                                    ,out=outfile
                                    ,format=options.outputType
                                    ,fieldList=options.fieldList
                                    ,dbType=options.dbType
                                    )
Exemplo n.º 8
0
def load_data(cx, datafile=sys.stdin, verbose=False, uscg=True):
    """Try to read data from an open file object.

    Not yet well tested.

    @param cx: database connection
    @param verbose: pring out more if true
    @param uscg: Process uscg tail information to get timestamp and receive station
    @rtype: None
    @return: Nothing

    @note: can not handle multiline AIS messages.  They must be normalized first.
    """

    v = verbose # Hmm... "v"... the irony

    cu = cx.cursor()
    lineNum = 0

    next_key = 0
    max_key = get_max_key(cx)
    if max_key is not None:
        next_key = max_key + 1
    print 'keys_starting_at:',next_key

    message_set = (1,2,3,4,5,18,19)
    counts = {}
    for msg_num in message_set:
        counts[msg_num] = 0

    counts['checksum_failed'] = 0

    track_dups = TrackDuplicates(lookback_length=1000)

    for line in datafile:
        lineNum += 1
        if lineNum%1000==0:
            print lineNum
            cx.commit()

        if len(line)<15 or line[3:6] not in ('VDM|VDO'): continue # Not an AIS VHF message

        #print 'FIX: validate checksum'
        if not nmea.checksum.isChecksumValid(line):
            print >> sys.stderr, 'WARNING: invalid checksum:\n\t',line,
            print >> sys.stderr, '   ',nmea.checksum.checksumStr(line)
            counts['checksum_failed'] += 1

        fields=line.split(',') # FIX: use this split throughout below...

        try:
            msg_num = int(binary.ais6tobitvec(fields[5][0]))
        except:
            print 'line would not decode',line
            continue
        if verbose: print 'msg_num:',msg_num
        if msg_num not in message_set:
            if verbose:
                print 'skipping',line
                print '  not in msg set:',str(message_set)
            continue

        try:
            bv = binary.ais6tobitvec(fields[5])
        except:
            print >> sys.stderr, 'ERROR: Unable to decode bits in line:\n\t',line
            traceback.print_exc(file=sys.stderr)
            continue

        # FIX: need to take padding into account ... right before the *
        if msg_num in (1,2,3,4,18):
            if len(bv) != 168:
                print 'ERROR: skipping bad one slot message, line:',lineNum
                print '  ',line,
                print '   Got length',len(bv), 'expected', 168
                continue
        elif msg_num == 5:
            # 426 has 2 pad bits
            if len(bv) not in (424,426):
                print 'ERROR: skipping bad shipdata message, line:',lineNum
                print '  ',line,
                print '   Got length',len(bv), 'expected', 424
                continue

        ins = None

        try:
            if   msg_num== 1: ins = ais.ais_msg_1_handcoded.sqlInsert(ais.ais_msg_1_handcoded.decode(bv),dbType='sqlite')
            elif msg_num== 2: ins = ais.ais_msg_2_handcoded.sqlInsert(ais.ais_msg_2_handcoded.decode(bv),dbType='sqlite')
            elif msg_num== 3: ins = ais.ais_msg_3_handcoded.sqlInsert(ais.ais_msg_3_handcoded.decode(bv),dbType='sqlite')
            elif msg_num== 4: ins = ais.ais_msg_4_handcoded.sqlInsert(ais.ais_msg_4_handcoded.decode(bv),dbType='sqlite')
            elif msg_num== 5: ins = ais.ais_msg_5.sqlInsert(ais.ais_msg_5.decode(bv),dbType='sqlite')
            elif msg_num==18: ins = ais.ais_msg_18.sqlInsert(ais.ais_msg_18.decode(bv),dbType='sqlite') # Class B position
            elif msg_num==19: ins = ais.ais_msg_19.sqlInsert(ais.ais_msg_19.decode(bv),dbType='sqlite') # Class B position
            else:
                print 'Warning... not handling type',msg_num,'line:',lineNum
                continue
        except:
            print 'ERROR:  some decode error?','line:',lineNum
            print '  ',line
            continue

        counts[msg_num] += 1

        if uscg:
            from aisutils.uscg import uscg_ais_nmea_regex
            match = uscg_ais_nmea_regex.search(line).groupdict()

            try:
                cg_sec = int(float(match['timeStamp']))
                ins.add('cg_sec', cg_sec)
                #ins.add('cg_sec', int(float(match['timeStamp'])) )
                ins.add('cg_timestamp', str(datetime.datetime.utcfromtimestamp(float(match['timeStamp']))) )
                ins.add('cg_r', match['station'] )
            except:
                print >> sys.stderr, match
                print >> sys.stderr, 'bad uscg sections',line,
                continue

            # Optional fields that are not always there

            if match['time_of_arrival'] is not None:
                try:
                    ins.add('cg_t_arrival',  float(match['time_of_arrival']))
                except:
                    print >> sys.stderr, 'WARNING: corrupted time of arrival (T) in line.  T ignored\n\t',line
                    pass # Not critical if corrupted


            if match['slot'] is not None:
                ins.add('cg_s_slotnum',  int(match['slot']) )

        if msg_num in (1,2,3,4):
            pkt_id,dup_flag = track_dups.check_packet(cg_sec,fields[5]) # Pass in the NMEA payload string of data
            if v:
                print 'dup_check:',pkt_id,dup_flag,fields[5]
            ins.add('pkt_id',pkt_id)
            ins.add('dup_flag',dup_flag)


        ins.add('key',next_key)
        next_key += 1

        if verbose:
            print str(ins)
        try:
            cu.execute(str(ins))
        except pysqlite2.dbapi2.OperationalError, params:
        #except OperationalError,  params:
            if -1 != params.message.find('no such table'):
                print 'ERROR:',params.message
                sys.exit('You probably need to run with --with-create')
            print 'params',params
            print type(params)
            print 'ERROR: sql error?','line:',lineNum
            print '  ', str(ins)
            print '  ',line

            if False:
                # Give some debugging flexibility
                from IPython.Shell import IPShellEmbed
                ipshell = IPShellEmbed(argv=[])
                ipshell()
                sys.exit('Gave up')
Exemplo n.º 9
0
'''

import sys

from aisutils.BitVector import BitVector
from aisutils import binary

import ais_msg_8
import sls.waterlevel

if __name__=='__main__':

    print 'FIX: broken'
    vdm='!AIVDM,1,1,4,A,8030ot1?0@>PSpPPPC<2<oURAU=>T08f@02PSpPPP3C3<oU=d5<U00BH@02PSpPPP3C3EoU:A5<TwPPO@02PSpPPP2hk<oRWU5;si0Pl@02O<0PPPP3D<oPPEU;M418g@02PSpPPP2hlEoRQgU;j@17p@00,2*32'
    msg=vdm.split(',')[5]
    bvMsg = binary.ais6tobitvec(msg)

    msg8 = ais_msg_8.decode(bvMsg)
    bvMsg8 = msg8['BinaryData']
    del msg8['BinaryData']
    ais_msg_8.printFields(msg8)

    print

    # Now deal with the St Lawrence Seaway Header
    slsHdr = sls_header.decode(bvMsg8)
    bvHdr = slsHdr['BinaryData']
    del slsHdr['BinaryData']
    sls_header.printFields(slsHdr)

    #print slsHdr.keys()
Exemplo n.º 10
0
def loadData(cx, datafile, verbose=False, uscg=True):
    '''
    Try to read data from an open file object.  Not yet well tested.

    @param cx: database connection
    @param verbose: pring out more if true
    @param uscg: Process uscg tail information to get timestamp and receive station
    @rtype: None
    @return: Nothing

    @note: can not handle multiline AIS messages.  They must be normalized first.
    '''
    cu = cx.cursor()
    lineNum = 0

    #counts = {1:0,2:0,3:0,4:0,5:0}
    counts = {}
    countsTotal = {}  # Includes ignored messages

    for i in range(64):
        counts[i] = 0
        countsTotal[i] = 0

    for line in datafile:
        lineNum += 1
        if lineNum % 1000 == 0:
            print lineNum
            cx.commit()
#           if lineNum>3000:
#		print 'Early exit from load'
#		break

        if line[3:6] not in ('VDM|VDO'):
            #            if verbose:
            #                sys.stderr.write
            continue  # Not an AIS VHF message
        try:
            msgNum = int(binary.ais6tobitvec(line.split(',')[5][0]))
        except:
            print 'line would not decode', line
            continue

        countsTotal[msgNum] += 1

        if verbose: print 'msgNum:', msgNum
        #	if msgNum not in (1,2,3,4,5):
        #           if verbose: print 'skipping',line
        #           continue

        payload = bv = binary.ais6tobitvec(line.split(',')[5])

        # FIX: need to take badding into account ... right before the *
        if msgNum in (1, 2, 3):
            #           if len(bv) != 168:
            if len(bv) < 168:
                print 'ERROR: skipping bad position message, line:', lineNum
                print '  ', line,
                print '   Got length', len(bv), 'expected', 168
                continue
#	elif msgNum == 4:
        elif msgNum == 5:
            #           if len(bv) != 424:
            if len(bv) < 424:
                print 'ERROR: skipping bad shipdata message, line:', lineNum
                print '  ', line,
                print '   Got length', len(bv), 'expected', 424
                continue

        fields = line.split(',')

        cg_timestamp = None
        cg_station = None
        if uscg:
            try:
                cg_sec = int(float(fields[-1]))  # US Coast Guard time stamp.
                print 'cg_sec:', cg_sec, type(cg_sec)
                cg_timestamp = datetime.datetime.utcfromtimestamp(
                    float(cg_sec))
            except:
                print 'ERROR getting timestamp for', lineNum, line
            #print len(fields),fields
            for i in range(len(fields) - 1, 5, -1):
                if 0 < len(fields[i]) and 'r' == fields[i][0]:
                    cg_station = fields[i]
                    break  # Found it so ditch the for loop

        #print station
        #sys.exit('stations please work')

        ins = None

        try:
            if msgNum == 1:
                ins = ais.ais_msg_1.sqlInsert(ais.ais_msg_1.decode(bv),
                                              dbType='sqlite')
            elif msgNum == 2:
                ins = ais.ais_msg_2.sqlInsert(ais.ais_msg_2.decode(bv),
                                              dbType='sqlite')
            elif msgNum == 3:
                ins = ais.ais_msg_3.sqlInsert(ais.ais_msg_3.decode(bv),
                                              dbType='sqlite')
            elif msgNum == 4:
                ins = ais.ais_msg_4.sqlInsert(ais.ais_msg_4.decode(bv),
                                              dbType='sqlite')
            elif msgNum == 5:
                ins = ais.ais_msg_5.sqlInsert(ais.ais_msg_5.decode(bv),
                                              dbType='sqlite')

            elif msgNum == 18:
                ins = ais.ais_msg_18.sqlInsert(ais.ais_msg_18.decode(bv),
                                               dbType='sqlite')
            elif msgNum == 19:
                ins = ais.ais_msg_19.sqlInsert(ais.ais_msg_19.decode(bv),
                                               dbType='sqlite')
            else:
                if verbose:
                    print 'Warning... not handling type', msgNum, 'line:', lineNum
                continue
        except:
            print 'ERROR:  some decode error?', 'line:', lineNum
            print '  ', line
            continue

        counts[msgNum] += 1

        if uscg:
            # FIX: make cg_timestamp work
            if None != cg_timestamp: ins.add('cg_timestamp', cg_timestamp)
            if None != cg_station: ins.add('cg_r', cg_station)
        if verbose: print str(ins)

        # FIX: redo this correctly???
        #try:
        print str(ins)
        cu.execute(str(ins))
        #except:
        #    sys.stderr.write('FIX: write some sort of exception handler\n')
#         except pysqlite2.dbapi2.OperationalError, params:
#       #except OperationalError,  params:
#             if -1 != params.message.find('no such table'):
#                 print 'ERROR:',params.message
#                 sys.exit('You probably need to run with --with-create')
#             print 'params',params
#             print type(params)
#           print 'ERROR: sql error?','line:',lineNum
#           print '  ', str(ins)
#           print '  ',line

#             if False:
#                 # Give some debugging flexibility
#                 from IPython.Shell import IPShellEmbed
#                 ipshell = IPShellEmbed(argv=[])
#                 ipshell()
#                 sys.exit('Gave up')

#print counts
    print '\nMessages found:'
    for key in countsTotal:
        if countsTotal[key] > 0:
            print str(key) + ':', countsTotal[key]

    print '\nMessages processed:'
    for key in counts:
        if counts[key] > 0:
            print str(key) + ':', counts[key]
    cx.commit()
Exemplo n.º 11
0
def main():
    from optparse import OptionParser

    parser = OptionParser(usage="%prog [options]")

    parser.add_option("--unit-test", dest="unittest", default=False, action="store_true", help="run the unit tests")
    parser.add_option(
        "-v", "--verbose", dest="verbose", default=False, action="store_true", help="Make the test output verbose"
    )

    # FIX: remove nmea from binary messages.  No way to build the whole packet?
    # FIX: or build the surrounding msg 8 for a broadcast?
    typeChoices = ("binary", "nmeapayload", "nmea")  # FIX: what about a USCG type message?
    parser.add_option(
        "-t",
        "--type",
        choices=typeChoices,
        type="choice",
        dest="ioType",
        default="nmeapayload",
        help="What kind of string to write for encoding (" + ", ".join(typeChoices) + ") [default: %default]",
    )

    outputChoices = ("std", "html", "csv", "sql")
    parser.add_option(
        "-T",
        "--output-type",
        choices=outputChoices,
        type="choice",
        dest="outputType",
        default="std",
        help="What kind of string to output (" + ", ".join(outputChoices) + ") " "[default: %default]",
    )

    parser.add_option(
        "-o", "--output", dest="outputFileName", default=None, help="Name of the python file to write [default: stdout]"
    )

    parser.add_option(
        "-f",
        "--fields",
        dest="fieldList",
        default=None,
        action="append",
        choices=fieldList,
        help="Which fields to include in the output.  Currently only for csv " "output [default: all]",
    )

    parser.add_option(
        "-p",
        "--print-csv-field-list",
        dest="printCsvfieldList",
        default=False,
        action="store_true",
        help="Print the field name for csv",
    )

    parser.add_option(
        "-c",
        "--sql-create",
        dest="sqlCreate",
        default=False,
        action="store_true",
        help="Print out an sql create command for the table.",
    )

    parser.add_option(
        "--latex-table",
        dest="latexDefinitionTable",
        default=False,
        action="store_true",
        help="Print a LaTeX table of the type",
    )

    parser.add_option(
        "--text-table",
        dest="textDefinitionTable",
        default=False,
        action="store_true",
        help="Print delimited table of the type (for Word table importing)",
    )

    parser.add_option(
        "--delimt-text-table",
        dest="delimTextDefinitionTable",
        default="    ",
        help="Delimiter for text table [default: '%default'] " "(for Word table importing)",
    )

    dbChoices = ("sqlite", "postgres")
    parser.add_option(
        "-D",
        "--db-type",
        dest="dbType",
        default="postgres",
        choices=dbChoices,
        type="choice",
        help="What kind of database (" + ", ".join(dbChoices) + ") " "[default: %default]",
    )

    addMsgOptions(parser)

    options, args = parser.parse_args()

    if options.unittest:
        sys.argv = [sys.argv[0]]
        if options.verbose:
            sys.argv.append("-v")
        unittest.main()

    outfile = sys.stdout
    if None != options.outputFileName:
        outfile = file(options.outputFileName, "w")

    if options.doEncode:
        # Make sure all non required options are specified.
        if None == options.RepeatIndicatorField:
            parser.error("missing value for RepeatIndicatorField")
        if None == options.UserIDField:
            parser.error("missing value for UserIDField")
        if None == options.monthField:
            parser.error("missing value for monthField")
        if None == options.dayField:
            parser.error("missing value for dayField")
        if None == options.hourField:
            parser.error("missing value for hourField")
        if None == options.minField:
            parser.error("missing value for minField")
        if None == options.stationidField:
            parser.error("missing value for stationidField")
        if None == options.waterlevelField:
            parser.error("missing value for waterlevelField")
        if None == options.datumField:
            parser.error("missing value for datumField")
        if None == options.sigmaField:
            parser.error("missing value for sigmaField")
        if None == options.sourceField:
            parser.error("missing value for sourceField")
    msgDict = {
        "MessageID": "8",
        "RepeatIndicator": options.RepeatIndicatorField,
        "UserID": options.UserIDField,
        "Spare": "0",
        "dac": "366",
        "fid": "63",
        "month": options.monthField,
        "day": options.dayField,
        "hour": options.hourField,
        "min": options.minField,
        "stationid": options.stationidField,
        "waterlevel": options.waterlevelField,
        "datum": options.datumField,
        "sigma": options.sigmaField,
        "source": options.sourceField,
    }

    bits = encode(msgDict)
    if "binary" == options.ioType:
        print str(bits)
    elif "nmeapayload" == options.ioType:
        # FIX: figure out if this might be necessary at compile time
        bitLen = len(bits)
        if bitLen % 6 != 0:
            bits = bits + BitVector(size=(6 - (bitLen % 6)))  # Pad out to multiple of 6
        print binary.bitvectoais6(bits)[0]

    # FIX: Do not emit this option for the binary message payloads.  Does not make sense.
    elif "nmea" == options.ioType:
        nmea = uscg.create_nmea(bits)
        print nmea
    else:
        sys.exit("ERROR: unknown ioType.  Help!")

        if options.sqlCreate:
            sqlCreateStr(outfile, options.fieldList, dbType=options.dbType)

        if options.latexDefinitionTable:
            latexDefinitionTable(outfile)

        # For conversion to word tables
        if options.textDefinitionTable:
            textDefinitionTable(outfile, options.delimTextDefinitionTable)

        if options.printCsvfieldList:
            # Make a csv separated list of fields that will be displayed for csv
            if None == options.fieldList:
                options.fieldList = fieldList
            import StringIO

            buf = StringIO.StringIO()
            for field in options.fieldList:
                buf.write(field + ",")
            result = buf.getvalue()
            if result[-1] == ",":
                print result[:-1]
            else:
                print result

        if options.doDecode:
            if len(args) == 0:
                args = sys.stdin
            for msg in args:
                bv = None

                if msg[0] in ("$", "!") and msg[3:6] in ("VDM", "VDO"):
                    # Found nmea
                    # FIX: do checksum
                    bv = binary.ais6tobitvec(msg.split(",")[5])
                else:  # either binary or nmeapayload... expect mostly nmeapayloads
                    # assumes that an all 0 and 1 string can not be a nmeapayload
                    binaryMsg = True
                    for c in msg:
                        if c not in ("0", "1"):
                            binaryMsg = False
                            break
                    if binaryMsg:
                        bv = BitVector(bitstring=msg)
                    else:  # nmeapayload
                        bv = binary.ais6tobitvec(msg)

                printFields(
                    decode(bv),
                    out=outfile,
                    format=options.outputType,
                    fieldList=options.fieldList,
                    dbType=options.dbType,
                )
Exemplo n.º 12
0
def load_data(cx, datafile=sys.stdin, verbose=False, uscg=True):
    """Try to read data from an open file object.

    Not yet well tested.

    @param cx: database connection
    @param verbose: pring out more if true
    @param uscg: Process uscg tail information to get timestamp and receive station
    @rtype: None
    @return: Nothing

    @note: can not handle multiline AIS messages.  They must be normalized first.
    """

    v = verbose  # Hmm... "v"... the irony

    cu = cx.cursor()
    lineNum = 0

    next_key = 0
    max_key = get_max_key(cx)
    if max_key is not None:
        next_key = max_key + 1
    print 'keys_starting_at:', next_key

    message_set = (1, 2, 3, 4, 5, 18, 19)
    counts = {}
    for msg_num in message_set:
        counts[msg_num] = 0

    counts['checksum_failed'] = 0

    track_dups = TrackDuplicates(lookback_length=1000)

    for line in datafile:
        lineNum += 1
        if lineNum % 1000 == 0:
            print lineNum
            cx.commit()

        if len(line) < 15 or line[3:6] not in ('VDM|VDO'):
            continue  # Not an AIS VHF message

        #print 'FIX: validate checksum'
        if not nmea.checksum.isChecksumValid(line):
            print >> sys.stderr, 'WARNING: invalid checksum:\n\t', line,
            print >> sys.stderr, '   ', nmea.checksum.checksumStr(line)
            counts['checksum_failed'] += 1

        fields = line.split(',')  # FIX: use this split throughout below...

        try:
            msg_num = int(binary.ais6tobitvec(fields[5][0]))
        except:
            print 'line would not decode', line
            continue
        if verbose: print 'msg_num:', msg_num
        if msg_num not in message_set:
            if verbose:
                print 'skipping', line
                print '  not in msg set:', str(message_set)
            continue

        try:
            bv = binary.ais6tobitvec(fields[5])
        except:
            print >> sys.stderr, 'ERROR: Unable to decode bits in line:\n\t', line
            traceback.print_exc(file=sys.stderr)
            continue

        # FIX: need to take padding into account ... right before the *
        if msg_num in (1, 2, 3, 4, 18):
            if len(bv) != 168:
                print 'ERROR: skipping bad one slot message, line:', lineNum
                print '  ', line,
                print '   Got length', len(bv), 'expected', 168
                continue
        elif msg_num == 5:
            # 426 has 2 pad bits
            if len(bv) not in (424, 426):
                print 'ERROR: skipping bad shipdata message, line:', lineNum
                print '  ', line,
                print '   Got length', len(bv), 'expected', 424
                continue

        ins = None

        try:
            if msg_num == 1:
                ins = ais.ais_msg_1_handcoded.sqlInsert(
                    ais.ais_msg_1_handcoded.decode(bv), dbType='sqlite')
            elif msg_num == 2:
                ins = ais.ais_msg_2_handcoded.sqlInsert(
                    ais.ais_msg_2_handcoded.decode(bv), dbType='sqlite')
            elif msg_num == 3:
                ins = ais.ais_msg_3_handcoded.sqlInsert(
                    ais.ais_msg_3_handcoded.decode(bv), dbType='sqlite')
            elif msg_num == 4:
                ins = ais.ais_msg_4_handcoded.sqlInsert(
                    ais.ais_msg_4_handcoded.decode(bv), dbType='sqlite')
            elif msg_num == 5:
                ins = ais.ais_msg_5.sqlInsert(ais.ais_msg_5.decode(bv),
                                              dbType='sqlite')
            elif msg_num == 18:
                ins = ais.ais_msg_18.sqlInsert(
                    ais.ais_msg_18.decode(bv),
                    dbType='sqlite')  # Class B position
            elif msg_num == 19:
                ins = ais.ais_msg_19.sqlInsert(
                    ais.ais_msg_19.decode(bv),
                    dbType='sqlite')  # Class B position
            else:
                print 'Warning... not handling type', msg_num, 'line:', lineNum
                continue
        except:
            print 'ERROR:  some decode error?', 'line:', lineNum
            print '  ', line
            continue

        counts[msg_num] += 1

        if uscg:
            from aisutils.uscg import uscg_ais_nmea_regex
            match = uscg_ais_nmea_regex.search(line).groupdict()

            try:
                cg_sec = int(float(match['timeStamp']))
                ins.add('cg_sec', cg_sec)
                #ins.add('cg_sec', int(float(match['timeStamp'])) )
                ins.add(
                    'cg_timestamp',
                    str(
                        datetime.datetime.utcfromtimestamp(
                            float(match['timeStamp']))))
                ins.add('cg_r', match['station'])
            except:
                print >> sys.stderr, match
                print >> sys.stderr, 'bad uscg sections', line,
                continue

            # Optional fields that are not always there

            if match['time_of_arrival'] is not None:
                try:
                    ins.add('cg_t_arrival', float(match['time_of_arrival']))
                except:
                    print >> sys.stderr, 'WARNING: corrupted time of arrival (T) in line.  T ignored\n\t', line
                    pass  # Not critical if corrupted

            if match['slot'] is not None:
                ins.add('cg_s_slotnum', int(match['slot']))

        if msg_num in (1, 2, 3, 4):
            pkt_id, dup_flag = track_dups.check_packet(
                cg_sec, fields[5])  # Pass in the NMEA payload string of data
            if v:
                print 'dup_check:', pkt_id, dup_flag, fields[5]
            ins.add('pkt_id', pkt_id)
            ins.add('dup_flag', dup_flag)

        ins.add('key', next_key)
        next_key += 1

        if verbose:
            print str(ins)
        try:
            cu.execute(str(ins))
        except pysqlite2.dbapi2.OperationalError, params:
            #except OperationalError,  params:
            if -1 != params.message.find('no such table'):
                print 'ERROR:', params.message
                sys.exit('You probably need to run with --with-create')
            print 'params', params
            print type(params)
            print 'ERROR: sql error?', 'line:', lineNum
            print '  ', str(ins)
            print '  ', line

            if False:
                # Give some debugging flexibility
                from IPython.Shell import IPShellEmbed
                ipshell = IPShellEmbed(argv=[])
                ipshell()
                sys.exit('Gave up')
Exemplo n.º 13
0
def loadData(cx,datafile,verbose=False
             , uscg=True):
    '''
    Try to read data from an open file object.  Not yet well tested.

    @param cx: database connection
    @param verbose: pring out more if true
    @param uscg: Process uscg tail information to get timestamp and receive station
    @rtype: None
    @return: Nothing

    @note: can not handle multiline AIS messages.  They must be normalized first.
    '''
    cu = cx.cursor()
    lineNum = 0

    import psycopg2 # For ProgrammingError exception


    counts = {1:0,2:0,3:0,5:0}

#    buf=[]


    for line in datafile:
        lineNum += 1
        if lineNum%1000==0:
            print lineNum

        if line[3:6] not in ('VDM|VDO'): continue # Not an AIS VHF message
        try:
            msgNum = int(binary.ais6tobitvec(line.split(',')[5][0]))
        except:
            print '# line would not decode',line
            continue
        if verbose: print '# msgNum:',msgNum
        if msgNum not in (1,2,3,5):
            if verbose: print '# skipping',line
            continue

        payload = bv = binary.ais6tobitvec(line.split(',')[5])

        # TODO(schwehr): Need to take padding into account.
        if msgNum in (1,2,3):
            if len(bv) < 168:
                print '# ERROR: skipping bad position message, line:',lineNum
                print '#  ',line,
                print '#   Got length',len(bv), 'expected', 168
                continue
        elif msgNum == 5:
            if len(bv) < 424:
                print '# ERROR: skipping bad shipdata message, line:',lineNum
                print '#  ',line,
                print '#   Got length',len(bv), 'expected', 424
                continue



        fields=line.split(',')

        cg_sec = None
        cg_station   = None
        if uscg:
            cg_sec = int(float(fields[-1])) # US Coast Guard time stamp.

            #print len(fields),fields
            for i in range(len(fields)-1,5,-1):
                if 0<len(fields[i]) and 'r' == fields[i][0]:
                    cg_station = fields[i]
                    break # Found it so ditch the for loop

        ins = None

        # FIX: redo this for all messages using the new aisutils structure
        if True:
            if   msgNum==1: ins = ais.ais_msg_1.sqlInsert(ais.ais_msg_1.decode(bv),dbType='postgres')
            elif msgNum==2: ins = ais.ais_msg_2.sqlInsert(ais.ais_msg_2.decode(bv),dbType='postgres')
            elif msgNum==3: ins = ais.ais_msg_3.sqlInsert(ais.ais_msg_3.decode(bv),dbType='postgres')
            elif msgNum==5:
                params = ais.ais_msg_5.decode(bv)
                #print params
                # FIX: make this a command line option
                params['name'] = params['name'].replace('"','').replace('\\','').strip('@').strip()
                params['callsign'] = params['callsign'].replace('"','').replace('\\','').strip('@').strip()
                params['destination'] = params['destination'].replace('"','').replace('\\','').strip('@').strip()
                #params.callsign = params.callsign.strip()
                #params. = params..strip()
                ins = ais.ais_msg_5.sqlInsert(params,dbType='postgres')
            else:
                print '# Warning... not handling type',msgNum,'line:',lineNum
                continue

        counts[msgNum] += 1

        if uscg:
            if None != cg_sec:       ins.add('cg_sec',       cg_sec)
            if None != cg_timestamp: ins.add('cg_timestamp', cg_timestamp)
            if None != cg_station:   ins.add('cg_r',         cg_station)
        if verbose:
            print str(ins)
            print '# line:',line
        try:
            cu.execute(str(ins))
        except Exception, e:
            print params
            # TODO(schwehr): Give a better error message.
            print '# exception:',str(type(Exception)), str(e)
            print '# ERROR: sql error?','line:',lineNum
            print '#  ', str(ins)
            print '#  ',line
            sys.exit('EARLY!!!')

        if lineNum%5000==0:
            if verbose:
                print '# committing batch'
            cx.commit()
Exemplo n.º 14
0
                else:
                    print 'FAILED'
                    success=False

        if not success: sys.exit('Something Failed')
        del success # Hide success from epydoc

        if options.unittest:
                sys.argv = [sys.argv[0]]
                if options.verbose: sys.argv.append('-v')
                unittest.main()


        outfile = sys.stdout
        if None!=options.outputFileName:
                outfile = file(options.outputFileName,'w')

        bv=None
        for msg in args:
            if   'binary'      == options.inputType:  bv = BitVector(bitstring=msg)
            elif 'nmeapayload' == options.inputType:  bv = binary.ais6tobitvec(msg)
            elif 'nmea'        == options.inputType:  bv = binary.ais6tobitvec(msg.split(',')[5])
            else: sys.exit('ERROR: unknown inputType.  Help!')

            #import ais.ais_msg_8 as m8
            #m8dict = m8.decode(bv)
            import ais.waterlevel as wl

            #wl.printFields(wl.decode(m8dict['BinaryData']),out=outfile,format=options.outputType)
            wl.printFields(wl.decode(bv),out=outfile,format=options.outputType)
Exemplo n.º 15
0
            dt_time_of_arrival = 'N/A'.rjust(8)
        time_of_arrival_prev = time_of_arrival

    try:
        slot_num = int(match['slot'])
        slot_t = slot_num / 2250. * 60
        slot_t = '%5.2f' % slot_t
    except:
        slot_num = 'N/A'
        slot_t = 'N/A'

    print '|', uscg, '|', cg_s, '|', dt, '|', time_of_arrival, '|', dt_time_of_arrival, '|', match[
        't_recver_hhmmss'], '|', slot_num, '|', slot_t, '|',

    if match['body'][0] in ('1', '2', '3'):
        bits = binary.ais6tobitvec(match['body'])
        msg = ais_msg_1.decode(bits)
        #print msg.keys()
        #all_keys.update(set(msg.keys()))
        msg_slot = 'N/A'
        if 'slot_number' not in msg:
            msg['slot_number'] = 'N/A'
            msg['slot_time'] = 'N/A'
        else:
            msg['slot_time'] = msg['slot_number'] / 2250. * 60
        if 'commstate_utc_hour' not in msg:
            msg['commstate_utc_hour'] = msg['commstate_utc_min'] = 'N/A'

        print '{slot_number}|{slot_time}|{commstate_utc_hour}|{commstate_utc_min}|{TimeStamp}|{UserID}|'.format(
            **msg)
    elif match['body'][0] == '4':
Exemplo n.º 16
0
from aisutils.BitVector import BitVector
from aisutils import binary
import ais_msg_1
import ais_msg_8
import sls.waterlevel

if __name__ == '__main__':

    # Try to parse some binary message
    if False:
        nmeaStr = '!AIVDM,1,1,,A,85OpLV1Kf98p96dWWPLSViUfJlU@SV>cDF2Wq5>`=u8CnEFGCIOq,0*70,r003669983,1165795916'

        msgPayload = nmeaStr.split(',')[5]
        print 'nmea string:    ', nmeaStr
        print 'message payload:', msgPayload
        bv = binary.ais6tobitvec(msgPayload)
        print len(bv), bv

        msgDict = ais_msg_8.bin_broadcastDecode(bv)
        ais_msg_8.bin_broadcastPrintFields(msgDict)

        bv = bv[39:]

        print 'dac: ', bv[:10], int(bv[:10])
        bv = bv[10:]
        print 'fid: ', bv[:6], int(bv[:6])
        bv = bv[6:]
        print 'bits:', bv[:16], int(bv[:10])
        bv = bv[10:]
        print 'len: ', len(bv)
Exemplo n.º 17
0
        'Append the nmea string after the line [default: do not print the line]'
    )

    parser.add_option('-o',
                      '--output',
                      dest='outputFilename',
                      default=None,
                      help='Name of the file to write [default: stdout]')

    (options, args) = parser.parse_args()
    o = sys.stdout
    if None != options.outputFilename: o = open(options.outFilename, 'w')

    print args
    for filename in args:
        print filename
        for line in file(filename):
            if line[0] == '#':
                continue
            fields = line.split(',')[:6]
            if '1' != fields[2]:  # Must be the start of a sequence
                continue
            if len(fields[5]) < 7: continue
            bv = binary.ais6tobitvec(fields[5][:7])  # Hacked for speed
            int(bv[8:38])
            mmsi = str(int(bv[8:38]))
            o.write(mmsi)
            if options.dumpLine:
                o.write(' ' + line.strip())
            o.write('\n')
Exemplo n.º 18
0
def loadData(cx,datafile,verbose=False
             , uscg=True):
    '''
    Try to read data from an open file object.  Not yet well tested.

    @param cx: database connection
    @param verbose: pring out more if true
    @param uscg: Process uscg tail information to get timestamp and receive station
    @rtype: None
    @return: Nothing

    @note: can not handle multiline AIS messages.  They must be normalized first.
    '''
    cu = cx.cursor()
    lineNum = 0

    #counts = {1:0,2:0,3:0,4:0,5:0}
    counts = {}
    countsTotal = {} # Includes ignored messages

    for i in range(64):
        counts[i]=0
        countsTotal[i]=0

    for line in datafile:
        lineNum += 1
        if lineNum%1000==0:
            print lineNum
            cx.commit()

        if line[3:6] not in ('VDM|VDO'):
            continue  # Not an AIS VHF message.
        try:
            msgNum = int(binary.ais6tobitvec(line.split(',')[5][0]))
        except:
            print 'line would not decode',line
            continue

        countsTotal[msgNum]+=1

        if verbose: print 'msgNum:',msgNum

        payload = bv = binary.ais6tobitvec(line.split(',')[5])

        # TODO(schwehr): Take padding into account.x
        if msgNum in (1,2,3):
            if len(bv) < 168:
                print 'ERROR: skipping bad position message, line:',lineNum
                print '  ',line,
                print '   Got length',len(bv), 'expected', 168
                continue
        elif msgNum == 5:
            if len(bv) < 424:
                print 'ERROR: skipping bad shipdata message, line:',lineNum
                print '  ',line,
                print '   Got length',len(bv), 'expected', 424
                continue

        fields=line.split(',')

        cg_timestamp = None
        cg_station   = None
        if uscg:
            try:
                cg_sec = int(float(fields[-1])) # US Coast Guard time stamp.
                print 'cg_sec:',cg_sec,type(cg_sec)
                cg_timestamp = datetime.datetime.utcfromtimestamp(float(cg_sec))
            except:
            #print len(fields),fields
            for i in range(len(fields)-1,5,-1):
                if 0<len(fields[i]) and 'r' == fields[i][0]:
                    cg_station = fields[i]
                    break # Found it so ditch the for loop

        ins = None

        try:
            if   msgNum==1: ins = ais.ais_msg_1.sqlInsert(ais.ais_msg_1.decode(bv),dbType='sqlite')
            elif msgNum==2: ins = ais.ais_msg_2.sqlInsert(ais.ais_msg_2.decode(bv),dbType='sqlite')
            elif msgNum==3: ins = ais.ais_msg_3.sqlInsert(ais.ais_msg_3.decode(bv),dbType='sqlite')
            elif msgNum==4: ins = ais.ais_msg_4.sqlInsert(ais.ais_msg_4.decode(bv),dbType='sqlite')
            elif msgNum==5: ins = ais.ais_msg_5.sqlInsert(ais.ais_msg_5.decode(bv),dbType='sqlite')

            elif msgNum==18: ins = ais.ais_msg_18.sqlInsert(ais.ais_msg_18.decode(bv),dbType='sqlite')
            elif msgNum==19: ins = ais.ais_msg_19.sqlInsert(ais.ais_msg_19.decode(bv),dbType='sqlite')
            else:
                if verbose:
                    print 'Warning... not handling type',msgNum,'line:',lineNum
                continue
        except:
            print 'ERROR:  some decode error?','line:',lineNum
            print '  ',line
            continue

        counts[msgNum] += 1

        if uscg:
            # FIX: make cg_timestamp work
            if None != cg_timestamp: ins.add('cg_timestamp',cg_timestamp)
            if None != cg_station:   ins.add('cg_r',        cg_station)
        if verbose: print str(ins)

        # FIX: redo this correctly???
        #try:
        print str(ins)
        cu.execute(str(ins))
    print '\nMessages found:'
    for key in countsTotal:
        if countsTotal[key]>0:
            print str(key)+':',countsTotal[key]

    print '\nMessages processed:'
    for key in counts:
        if counts[key]>0:
            print str(key)+':',counts[key]
    cx.commit()


############################################################
if __name__=='__main__':
    from optparse import OptionParser
    parser = OptionParser(usage="%prog [options] file1.ais [file2.ais ...]",
                          version='%prog ')

    parser.add_option('-d','--database-file',dest='databaseFilename',default='ais.db3'
                      ,help='Name of the sqlite3 database file to write [default: %default]')
    parser.add_option('-C','--with-create',dest='createTables',default=False, action='store_true'
                      ,help='Create the tables in the database')

    parser.add_option('--without-uscg',dest='uscgTail',default=True,action='store_false'
                      ,help='Do not look for timestamp and receive station at the end of each line [default: with-uscg]')

    parser.add_option('-v','--verbose',dest='verbose',default=False,action='store_true'
                      ,help='Make program output more verbose info as it runs')

    (options,args) = parser.parse_args()
    cx = sqlite3.connect(options.databaseFilename)

    print 'FIX: cg_sec and cg_timestamp are currently broken'

    if options.createTables:
        createTables(cx,verbose=options.verbose)

    if len(args)==0:
        print 'processing from stdin'

        loadData(cx,sys.stdin,verbose=options.verbose,uscg=options.uscgTail)
    else:
        for filename in args:
            print 'processing file:',filename
            loadData(cx,file(filename,'r'),verbose=options.verbose,uscg=options.uscgTail)
Exemplo n.º 19
0
    ws_report.write(ws_report_row,col,'destination'); col += 1
    ws_report.write(ws_report_row,col,'first reported (UTCsec)'); col += 1

    ws_report_row += 1

    msgsByShip={}
    # FIX: error checking?
    for filename in args:
        linenum=0
        for line in file(filename):
            line=line.strip()
            linenum +=1
            if linenum%1000==0:
                print linenum
            fields = line.split(',')
            bv = binary.ais6tobitvec(fields[5][:38])
            mmsi = m5.decodeUserID(bv)
            timestamp=fields[-1]

            if mmsi in msgsByShip:
                #if line not in msgsByShip:
                msgsByShip[mmsi].append(line)
            #if line in timeByMsgs
            else:
                msgsByShip[mmsi]=[line]

    print 'Finished scan.  Now processing ships.\n'

    ships = msgsByShip.keys()
    ships.sort()
Exemplo n.º 20
0
from aisutils import binary
import ais_msg_1
import ais_msg_8
import sls.waterlevel


if __name__ == "__main__":

    # Try to parse some binary message
    if False:
        nmeaStr = "!AIVDM,1,1,,A,85OpLV1Kf98p96dWWPLSViUfJlU@SV>cDF2Wq5>`=u8CnEFGCIOq,0*70,r003669983,1165795916"

        msgPayload = nmeaStr.split(",")[5]
        print "nmea string:    ", nmeaStr
        print "message payload:", msgPayload
        bv = binary.ais6tobitvec(msgPayload)
        print len(bv), bv

        msgDict = ais_msg_8.bin_broadcastDecode(bv)
        ais_msg_8.bin_broadcastPrintFields(msgDict)

        bv = bv[39:]

        print "dac: ", bv[:10], int(bv[:10])
        bv = bv[10:]
        print "fid: ", bv[:6], int(bv[:6])
        bv = bv[6:]
        print "bits:", bv[:16], int(bv[:10])
        bv = bv[10:]
        print "len: ", len(bv)
Exemplo n.º 21
0
'''

import sys

from aisutils.BitVector import BitVector
from aisutils import binary

import ais_msg_8
import sls.waterlevel

if __name__ == '__main__':

    print 'FIX: broken'
    vdm = '!AIVDM,1,1,4,A,8030ot1?0@>PSpPPPC<2<oURAU=>T08f@02PSpPPP3C3<oU=d5<U00BH@02PSpPPP3C3EoU:A5<TwPPO@02PSpPPP2hk<oRWU5;si0Pl@02O<0PPPP3D<oPPEU;M418g@02PSpPPP2hlEoRQgU;j@17p@00,2*32'
    msg = vdm.split(',')[5]
    bvMsg = binary.ais6tobitvec(msg)

    msg8 = ais_msg_8.decode(bvMsg)
    bvMsg8 = msg8['BinaryData']
    del msg8['BinaryData']
    ais_msg_8.printFields(msg8)

    print

    # Now deal with the St Lawrence Seaway Header
    slsHdr = sls_header.decode(bvMsg8)
    bvHdr = slsHdr['BinaryData']
    del slsHdr['BinaryData']
    sls_header.printFields(slsHdr)

    #print slsHdr.keys()
Exemplo n.º 22
0
def build_dist_database(database_filename, log_files, verbose=False):

    cx = sqlite3.connect(database_filename)

    print 'WARNING: not saving the station name'
    cx.execute('''
    CREATE TABLE IF NOT EXISTS distance (
       -- Save space, no key
       -- ts INTEGER, -- Save more space
       julian_day INTEGER,
       -- x REAL,
       -- y REAL,
       dist_km REAL
       --,
       --station VARCHAR(15)
       );
    ''')

    cu = cx.cursor()

    counts = {'nogps': 0}

    for filename in log_files:
        if verbose:
            print 'file:',filename
            sys.stdout.flush()
        for line_num, line in enumerate(file(filename)):
            if 'AIVDM,1,1' not in line: continue
            match = uscg_ais_nmea_regex.search(line).groupdict()
            message_id = match['body'][0] # First letter is the message type
            if message_id not in ('1','2','3'): continue

            if len(match['body']) != 28: # 6 bits per character
                raise AisErrorBadNumBits('expected 168, got %d' % len(match['body']) / 6)

            bits = binary.ais6tobitvec(match['body'][:20]) # Don't need any of the other bits, so do not waste time

            x = binary.signedIntFromBV(bits[61:89]) / 600000.
            y = binary.signedIntFromBV(bits[89:116]) / 600000.

            if x > 180 or y > 90:
                counts['nogps'] += 1
                continue

            station = match['station']

            julian_day = int(datetime.datetime.utcfromtimestamp(int(match['timeStamp'])).strftime('%j'))

            d_km =  dist_utm_km( (x,y), station_locations[station] )
            #cu.execute('INSERT INTO distance VALUES (:julian_day, :x, :y, :dist_km, :station)',
                       #{'julian_day': julian_day, 'x':x, 'y':y, 'dist_km': d_km, 'station':station} )
            #cu.execute('INSERT INTO distance VALUES (:julian_day, :x, :y, :dist_km)',
            #           {'julian_day': julian_day, 'x':x, 'y':y, 'dist_km': d_km, } )
            cu.execute('INSERT INTO distance VALUES (:julian_day, :dist_km)',
                       {'julian_day': julian_day, 'dist_km': d_km, } )
            if line_num % 10000 == 9999:
                cx.commit()


        cx.commit()

    if False:
        print 'Creating indexes'
        try:
            cx.execute('CREATE INDEX idx_dist_day ON distance(julian_day);')
            cx.execute('CREATE INDEX idx_dist_dist ON distance(dist_km);')
            #cx.execute('CREATE INDEX idx_dist_station ON distance(station);')
            cx.commit()
        except sqlite3.OperationalError:
            print 'Appears indexes were already created'

    return cx, counts
Exemplo n.º 23
0
def decode_aivdm(msg):
    bv = binary.ais6tobitvec(msg.split(',')[5])
    print decode(bv)
Exemplo n.º 24
0
def loadData(cx, datafile, verbose=False, uscg=True):
    '''
    Try to read data from an open file object.  Not yet well tested.

    @param cx: database connection
    @param verbose: pring out more if true
    @param uscg: Process uscg tail information to get timestamp and receive station
    @rtype: None
    @return: Nothing

    @note: can not handle multiline AIS messages.  They must be normalized first.
    '''
    cu = cx.cursor()
    lineNum = 0

    import psycopg2  # For ProgrammingError exception

    counts = {1: 0, 2: 0, 3: 0, 5: 0}

    #    buf=[]

    for line in datafile:
        lineNum += 1
        if lineNum % 1000 == 0:
            print lineNum

        if line[3:6] not in ('VDM|VDO'): continue  # Not an AIS VHF message
        try:
            msgNum = int(binary.ais6tobitvec(line.split(',')[5][0]))
        except:
            print '# line would not decode', line
            continue
        if verbose: print '# msgNum:', msgNum
        if msgNum not in (1, 2, 3, 5):
            if verbose: print '# skipping', line
            continue

        payload = bv = binary.ais6tobitvec(line.split(',')[5])

        # TODO(schwehr): Need to take padding into account.
        if msgNum in (1, 2, 3):
            if len(bv) < 168:
                print '# ERROR: skipping bad position message, line:', lineNum
                print '#  ', line,
                print '#   Got length', len(bv), 'expected', 168
                continue
        elif msgNum == 5:
            if len(bv) < 424:
                print '# ERROR: skipping bad shipdata message, line:', lineNum
                print '#  ', line,
                print '#   Got length', len(bv), 'expected', 424
                continue

        fields = line.split(',')

        cg_sec = None
        cg_station = None
        if uscg:
            cg_sec = int(float(fields[-1]))  # US Coast Guard time stamp.

            #print len(fields),fields
            for i in range(len(fields) - 1, 5, -1):
                if 0 < len(fields[i]) and 'r' == fields[i][0]:
                    cg_station = fields[i]
                    break  # Found it so ditch the for loop

        ins = None

        # FIX: redo this for all messages using the new aisutils structure
        if True:
            if msgNum == 1:
                ins = ais.ais_msg_1.sqlInsert(ais.ais_msg_1.decode(bv),
                                              dbType='postgres')
            elif msgNum == 2:
                ins = ais.ais_msg_2.sqlInsert(ais.ais_msg_2.decode(bv),
                                              dbType='postgres')
            elif msgNum == 3:
                ins = ais.ais_msg_3.sqlInsert(ais.ais_msg_3.decode(bv),
                                              dbType='postgres')
            elif msgNum == 5:
                params = ais.ais_msg_5.decode(bv)
                #print params
                # FIX: make this a command line option
                params['name'] = params['name'].replace('"', '').replace(
                    '\\', '').strip('@').strip()
                params['callsign'] = params['callsign'].replace(
                    '"', '').replace('\\', '').strip('@').strip()
                params['destination'] = params['destination'].replace(
                    '"', '').replace('\\', '').strip('@').strip()
                #params.callsign = params.callsign.strip()
                #params. = params..strip()
                ins = ais.ais_msg_5.sqlInsert(params, dbType='postgres')
            else:
                print '# Warning... not handling type', msgNum, 'line:', lineNum
                continue

        counts[msgNum] += 1

        if uscg:
            if None != cg_sec: ins.add('cg_sec', cg_sec)
            if None != cg_timestamp: ins.add('cg_timestamp', cg_timestamp)
            if None != cg_station: ins.add('cg_r', cg_station)
        if verbose:
            print str(ins)
            print '# line:', line
        try:
            cu.execute(str(ins))
        except Exception, e:
            print params
            # TODO(schwehr): Give a better error message.
            print '# exception:', str(type(Exception)), str(e)
            print '# ERROR: sql error?', 'line:', lineNum
            print '#  ', str(ins)
            print '#  ', line
            sys.exit('EARLY!!!')

        if lineNum % 5000 == 0:
            if verbose:
                print '# committing batch'
            cx.commit()
Exemplo n.º 25
0
		buf = StringIO.StringIO()
		for field in options.fieldList:
			buf.write(field+',')
		result = buf.getvalue()
		if result[-1] == ',': print result[:-1]
		else: print result

	if options.doDecode:
		if len(args)==0: args = sys.stdin
		for msg in args:
			bv = None

			if msg[0] in ('$','!') and msg[3:6] in ('VDM','VDO'):
				# Found nmea
				# FIX: do checksum
				bv = binary.ais6tobitvec(msg.split(',')[5])
			else: # either binary or nmeapayload... expect mostly nmeapayloads
				# assumes that an all 0 and 1 string can not be a nmeapayload
				binaryMsg=True
				for c in msg:
					if c not in ('0','1'):
						binaryMsg=False
						break
				if binaryMsg:
					bv = BitVector(bitstring=msg)
				else: # nmeapayload
					bv = binary.ais6tobitvec(msg)

			printFields(decode(bv)
				    ,out=outfile
				    ,format=options.outputType
Exemplo n.º 26
0
            #try:

            match_obj = uscg_ais_nmea_regex.search(line)
            if match_obj is None:
                sys.stderr.write(line)
                continue
            station = match_obj.group('station')

            #except:
            #    sys.stderr.write('bad line: %s\n' %line)
            #    continue


            fields = line.split(',')[:6]
            if '1'!=fields[2]: # Must be the start of a sequence
                #if verbose:
                #    print 'skipping based on field 2',line
                continue
            if len(fields[5])<39:
                #if verbose:
                #    print 'skipping',line
                continue
            bv = binary.ais6tobitvec(fields[5][:39]) # Hacked for speed
            #print int(bv[8:38]),aisstring.decode(bv[112:232],True)
            name = aisstring.decode(bv[112:232],True).strip('@ ')
            mmsi = str(int(bv[8:38]))
            imo = str(int(bv[40:70]))
            #if len(name)<1 or name[0]=='X': print 'TROUBLE with line:',line
            if len(name)<1: print 'TROUBLE with line:',line
            o.write (mmsi+' '+imo+' '+station+' '+name+'\n')
Exemplo n.º 27
0
def main():
    from optparse import OptionParser
    parser = OptionParser(usage="%prog [options]")

    parser.add_option('--unit-test',dest='unittest',default=False,action='store_true',
        help='run the unit tests')
    parser.add_option('-v','--verbose',dest='verbose',default=False,action='store_true',
        help='Make the test output verbose')

    # FIX: remove nmea from binary messages.  No way to build the whole packet?
    # FIX: or build the surrounding msg 8 for a broadcast?
    typeChoices = ('binary','nmeapayload','nmea') # FIX: what about a USCG type message?
    parser.add_option('-t', '--type', choices=typeChoices, type='choice',
        dest='ioType', default='nmeapayload',
        help='What kind of string to write for encoding ('+', '.join(typeChoices)+') [default: %default]')


    outputChoices = ('std','html','csv','sql' )
    parser.add_option('-T', '--output-type', choices=outputChoices,
        type='choice', dest='outputType', default='std',
        help='What kind of string to output ('+', '.join(outputChoices)+') '
        '[default: %default]')

    parser.add_option('-o','--output',dest='outputFileName',default=None,
        help='Name of the python file to write [default: stdout]')

    parser.add_option('-f', '--fields', dest='fieldList', default=None,
        action='append', choices=fieldList,
        help='Which fields to include in the output.  Currently only for csv '
        'output [default: all]')

    parser.add_option('-p', '--print-csv-field-list', dest='printCsvfieldList',
        default=False,action='store_true',
        help='Print the field name for csv')

    parser.add_option('-c', '--sql-create', dest='sqlCreate', default=False,
        action='store_true',
        help='Print out an sql create command for the table.')

    parser.add_option('--latex-table', dest='latexDefinitionTable',
        default=False,action='store_true',
        help='Print a LaTeX table of the type')

    parser.add_option('--text-table', dest='textDefinitionTable', default=False,
        action='store_true',
        help='Print delimited table of the type (for Word table importing)')

    parser.add_option('--delimt-text-table', dest='delimTextDefinitionTable',
        default='    ',
        help='Delimiter for text table [default: \'%default\'] '
        '(for Word table importing)')

    dbChoices = ('sqlite','postgres')
    parser.add_option('-D', '--db-type', dest='dbType', default='postgres',
        choices=dbChoices,type='choice',
        help='What kind of database ('+', '.join(dbChoices)+') '
        '[default: %default]')

    addMsgOptions(parser)

    options, args = parser.parse_args()

    if options.unittest:
            sys.argv = [sys.argv[0]]
            if options.verbose: sys.argv.append('-v')
            unittest.main()

    outfile = sys.stdout
    if None!=options.outputFileName:
            outfile = file(options.outputFileName,'w')


    if options.doEncode:
        # Make sure all non required options are specified.
        if None==options.unavail_uintField: parser.error("missing value for unavail_uintField")
        if None==options.anUIntField: parser.error("missing value for anUIntField")
        if None==options.anIntField: parser.error("missing value for anIntField")
        if None==options.aBoolField: parser.error("missing value for aBoolField")
        if None==options.aStrField: parser.error("missing value for aStrField")
        if None==options.anUDecimalField: parser.error("missing value for anUDecimalField")
        if None==options.aDecimalField: parser.error("missing value for aDecimalField")
        if None==options.aFloatField: parser.error("missing value for aFloatField")
    msgDict = {
        'dac': '366',
        'reqDecimal': '122',
        'unavail_uint': options.unavail_uintField,
        'anUInt': options.anUIntField,
        'anInt': options.anIntField,
        'aBool': options.aBoolField,
        'aStr': options.aStrField,
        'anUDecimal': options.anUDecimalField,
        'aDecimal': options.aDecimalField,
        'aFloat': options.aFloatField,
    }

    bits = encode(msgDict)
    if 'binary' == options.ioType:
        print str(bits)
    elif 'nmeapayload'==options.ioType:
        # FIX: figure out if this might be necessary at compile time
        bitLen=len(bits)
        if bitLen % 6 != 0:
            bits = bits + BitVector(size=(6 - (bitLen%6)))  # Pad out to multiple of 6
        print binary.bitvectoais6(bits)[0]

    # FIX: Do not emit this option for the binary message payloads.  Does not make sense.
    elif 'nmea' == options.ioType:
        nmea = uscg.create_nmea(bits)
        print nmea
    else:
        sys.exit('ERROR: unknown ioType.  Help!')


        if options.sqlCreate:
                sqlCreateStr(outfile,options.fieldList,dbType=options.dbType)

        if options.latexDefinitionTable:
                latexDefinitionTable(outfile)

        # For conversion to word tables
        if options.textDefinitionTable:
                textDefinitionTable(outfile,options.delimTextDefinitionTable)

        if options.printCsvfieldList:
                # Make a csv separated list of fields that will be displayed for csv
                if None == options.fieldList: options.fieldList = fieldList
                import StringIO
                buf = StringIO.StringIO()
                for field in options.fieldList:
                        buf.write(field+',')
                result = buf.getvalue()
                if result[-1] == ',': print result[:-1]
                else: print result

        if options.doDecode:
                if len(args)==0: args = sys.stdin
                for msg in args:
                        bv = None

                        if msg[0] in ('$','!') and msg[3:6] in ('VDM','VDO'):
                                # Found nmea
                                # FIX: do checksum
                                bv = binary.ais6tobitvec(msg.split(',')[5])
                        else: # either binary or nmeapayload... expect mostly nmeapayloads
                                # assumes that an all 0 and 1 string can not be a nmeapayload
                                binaryMsg=True
                                for c in msg:
                                        if c not in ('0','1'):
                                                binaryMsg=False
                                                break
                                if binaryMsg:
                                        bv = BitVector(bitstring=msg)
                                else: # nmeapayload
                                        bv = binary.ais6tobitvec(msg)

                        printFields(decode(bv)
                                    ,out=outfile
                                    ,format=options.outputType
                                    ,fieldList=options.fieldList
                                    ,dbType=options.dbType
                                    )
Exemplo n.º 28
0
            dt_time_of_arrival = 'N/A'.rjust(8)
        time_of_arrival_prev = time_of_arrival

    try:
        slot_num = int(match['slot'])
        slot_t = slot_num / 2250. * 60
        slot_t = '%5.2f' % slot_t
    except:
        slot_num = 'N/A'
        slot_t = 'N/A'


    print '|',uscg,'|',cg_s,'|',dt,'|',time_of_arrival,'|', dt_time_of_arrival,'|', match['t_recver_hhmmss'], '|',slot_num, '|',slot_t , '|',

    if match['body'][0] in ('1','2','3'):
        bits = binary.ais6tobitvec(match['body'])
        msg = ais_msg_1.decode(bits)
        #print msg.keys()
        #all_keys.update(set(msg.keys()))
        msg_slot = 'N/A'
        if 'slot_number' not in msg:
            msg['slot_number'] = 'N/A'
            msg['slot_time'] = 'N/A'
        else:
            msg['slot_time'] = msg['slot_number'] / 2250. * 60
        if 'commstate_utc_hour' not in msg:
            msg['commstate_utc_hour'] = msg['commstate_utc_min'] = 'N/A'

        print '{slot_number}|{slot_time}|{commstate_utc_hour}|{commstate_utc_min}|{TimeStamp}|{UserID}|'.format(**msg)
    elif match['body'][0] == '4':
        bits = binary.ais6tobitvec(match['body'])