def filter_file(infile, outfile, polygonWKT, verbose=False):
    '''
    For messages 1,2, and 3, see if the message is within the bounding box and send it to outfile if it is.

    Polygon should look something like this... 'POLYGON ((-1.0 50.5, -0.5 51.2, 0.3 50.9, -1 50.5))'

    param polygon: bounding region for the query
    type polygon: WKT polygon string
    '''
    import ais.ais_msg_1 as ais_msg_1
    import ais.binary as binary
    try:
        from cartography.geometry import Geometry
    except:
        sys.exit('ERROR: need to install pcl-core for cartography.geometry.Geometry')

    poly = Geometry.fromWKT(polygonWKT)
    bbox = poly.envelope()
    minx = bbox.minx  # for speed, throw out points as soon as possible
    maxx = bbox.maxx
    miny = bbox.miny
    maxy = bbox.maxy

    if verbose:
        print 'minLon maxLon minLat maxLat filename'
        print minx, maxx, miny, maxy

    count = 0
    linenum=0
    for line in infile:
        linenum += 1
        if linenum%1000==0:
            sys.stderr.write('line '+str(linenum)+' -- count='+str(count)+'\n')
        # Trick: Only handle the first 19 characters since that contains the lon/lat
        txt = line.split(',')[5][:25]
        #print txt
        bv = binary.ais6tobitvec(txt) #line[5][:19]

        # Try to throw out points as soon as possible.  Use float rather than decimal.  faster??  Maybe not
        #lon = ais_msg_1.decodelongitude(bv)
        lon = binary.signedIntFromBV(bv[61:89])/600000.0
        if lon<minx or lon>maxx: continue
        #print 'close1:',lon
        #lat = ais_msg_1.decodelatitude(bv)
        lat = binary.signedIntFromBV(bv[89:116])/600000.0
        if lat<miny or lat>maxy: continue

        #print 'close2: POINT ('+str(lon)+' '+str(lat)+')'

        point = Geometry.fromWKT('POINT ('+str(lon)+' '+str(lat)+')')
        inside = point.within(poly)
        if 1==inside:
            outfile.write(line)
            count+= 1

    return count
Esempio n. 2
0
    def add_message(self, ais_msg_dict, bits):
        '''Takes a USCG message dict pulled with the regex and a bit vector
        FIX... optionally ingest class b messages
        '''
        #match = uscg_ais_nmea_regex.search(line).groupdict()
        message_id = ais_msg_dict['body'][
            0]  # First letter is the message type
        if message_id in ('1', '2', '3'):
            if len(bits) != 168:
                print 'bad_bit_count_for_pos:', len(bits)
                self.count_bad_num_bits += 1
                raise AisErrorBadNumBits('expected 168, got 54')
                #return

            x = lon = binary.signedIntFromBV(bits[61:89]) / 600000.
            y = lat = binary.signedIntFromBV(bits[89:116]) / 600000.

            #
            # Check for bad messages
            #
            if lon > 180 or lat > 90:
                self.count_no_gps += 1
                return

            if self.station_location is not None:
                dist = distance_km_unit_sphere(x, y, self.station_location[0],
                                               self.station_location[1])
                #print 'dist:', dist
                if self.max_dist_km < dist:
                    #print 'bbox_dropping_point:',x,y,dist,'km'
                    self.count_bad_pos += 1
                    raise AisErrorPositionTooFar(
                        '%.2f %.2f -> %.2f km' % (x, y, dist))
                    #return

                self.dist_hist.add_point(dist)

            self.positions.append((lon, lat))
            self.bbox.add_point(lon, lat)
Esempio n. 3
0
def build_dist_database(database_filename, log_files, verbose=False):

    cx = sqlite3.connect(database_filename)

    print 'WARNING: not saving the station name'
    cx.execute('''
    CREATE TABLE IF NOT EXISTS distance (
       -- Save space, no key
       -- ts INTEGER, -- Save more space
       julian_day INTEGER,
       -- x REAL,
       -- y REAL,
       dist_km REAL
       --,
       --station VARCHAR(15)
       );
    ''')

    cu = cx.cursor()

    counts = {'nogps': 0}

    for filename in log_files:
        if verbose:
            print 'file:',filename
            sys.stdout.flush()
        for line_num, line in enumerate(file(filename)):
            if 'AIVDM,1,1' not in line: continue
            match = uscg_ais_nmea_regex.search(line).groupdict()
            message_id = match['body'][0] # First letter is the message type
            if message_id not in ('1','2','3'): continue

            if len(match['body']) != 28: # 6 bits per character
                raise AisErrorBadNumBits('expected 168, got %d' % len(match['body']) / 6)

            bits = binary.ais6tobitvec(match['body'][:20]) # Don't need any of the other bits, so do not waste time

            x = binary.signedIntFromBV(bits[61:89]) / 600000.
            y = binary.signedIntFromBV(bits[89:116]) / 600000.

            if x > 180 or y > 90:
                counts['nogps'] += 1
                continue

            station = match['station']

            julian_day = int(datetime.datetime.utcfromtimestamp(int(match['timeStamp'])).strftime('%j'))

            d_km =  dist_utm_km( (x,y), station_locations[station] )
            #cu.execute('INSERT INTO distance VALUES (:julian_day, :x, :y, :dist_km, :station)',
                       #{'julian_day': julian_day, 'x':x, 'y':y, 'dist_km': d_km, 'station':station} )
            #cu.execute('INSERT INTO distance VALUES (:julian_day, :x, :y, :dist_km)',
            #           {'julian_day': julian_day, 'x':x, 'y':y, 'dist_km': d_km, } )
            cu.execute('INSERT INTO distance VALUES (:julian_day, :dist_km)',
                       {'julian_day': julian_day, 'dist_km': d_km, } )
            if line_num % 10000 == 9999:
                cx.commit()


        cx.commit()

    if False:
        print 'Creating indexes'
        try:
            cx.execute('CREATE INDEX idx_dist_day ON distance(julian_day);')
            cx.execute('CREATE INDEX idx_dist_dist ON distance(dist_km);')
            #cx.execute('CREATE INDEX idx_dist_station ON distance(station);')
            cx.commit()
        except sqlite3.OperationalError:
            print 'Appears indexes were already created'

    return cx, counts