def filter_file(infile, outfile, polygonWKT, verbose=False):
'''
For messages 1,2, and 3, see if the message is within the bounding box and send it to outfile if it is.
Polygon should look something like this... 'POLYGON ((-1.0 50.5, -0.5 51.2, 0.3 50.9, -1 50.5))'
param polygon: bounding region for the query
type polygon: WKT polygon string
'''
import ais.ais_msg_1 as ais_msg_1
import ais.binary as binary
try:
from cartography.geometry import Geometry
except:
sys.exit('ERROR: need to install pcl-core for cartography.geometry.Geometry')
poly = Geometry.fromWKT(polygonWKT)
bbox = poly.envelope()
minx = bbox.minx # for speed, throw out points as soon as possible
maxx = bbox.maxx
miny = bbox.miny
maxy = bbox.maxy
if verbose:
print 'minLon maxLon minLat maxLat filename'
print minx, maxx, miny, maxy
count = 0
linenum=0
for line in infile:
linenum += 1
if linenum%1000==0:
sys.stderr.write('line '+str(linenum)+' -- count='+str(count)+'\n')
# Trick: Only handle the first 19 characters since that contains the lon/lat
txt = line.split(',')[5][:25]
#print txt
bv = binary.ais6tobitvec(txt) #line[5][:19]
# Try to throw out points as soon as possible. Use float rather than decimal. faster?? Maybe not
#lon = ais_msg_1.decodelongitude(bv)
lon = binary.signedIntFromBV(bv[61:89])/600000.0
if lon<minx or lon>maxx: continue
#print 'close1:',lon
#lat = ais_msg_1.decodelatitude(bv)
lat = binary.signedIntFromBV(bv[89:116])/600000.0
if lat<miny or lat>maxy: continue
#print 'close2: POINT ('+str(lon)+' '+str(lat)+')'
point = Geometry.fromWKT('POINT ('+str(lon)+' '+str(lat)+')')
inside = point.within(poly)
if 1==inside:
outfile.write(line)
count+= 1
return count
def add_message(self, ais_msg_dict, bits):
'''Takes a USCG message dict pulled with the regex and a bit vector
FIX... optionally ingest class b messages
'''
#match = uscg_ais_nmea_regex.search(line).groupdict()
message_id = ais_msg_dict['body'][
0] # First letter is the message type
if message_id in ('1', '2', '3'):
if len(bits) != 168:
print 'bad_bit_count_for_pos:', len(bits)
self.count_bad_num_bits += 1
raise AisErrorBadNumBits('expected 168, got 54')
#return
x = lon = binary.signedIntFromBV(bits[61:89]) / 600000.
y = lat = binary.signedIntFromBV(bits[89:116]) / 600000.
#
# Check for bad messages
#
if lon > 180 or lat > 90:
self.count_no_gps += 1
return
if self.station_location is not None:
dist = distance_km_unit_sphere(x, y, self.station_location[0],
self.station_location[1])
#print 'dist:', dist
if self.max_dist_km < dist:
#print 'bbox_dropping_point:',x,y,dist,'km'
self.count_bad_pos += 1
raise AisErrorPositionTooFar(
'%.2f %.2f -> %.2f km' % (x, y, dist))
#return
self.dist_hist.add_point(dist)
self.positions.append((lon, lat))
self.bbox.add_point(lon, lat)
def build_dist_database(database_filename, log_files, verbose=False):
cx = sqlite3.connect(database_filename)
print 'WARNING: not saving the station name'
cx.execute('''
CREATE TABLE IF NOT EXISTS distance (
-- Save space, no key
-- ts INTEGER, -- Save more space
julian_day INTEGER,
-- x REAL,
-- y REAL,
dist_km REAL
--,
--station VARCHAR(15)
);
''')
cu = cx.cursor()
counts = {'nogps': 0}
for filename in log_files:
if verbose:
print 'file:',filename
sys.stdout.flush()
for line_num, line in enumerate(file(filename)):
if 'AIVDM,1,1' not in line: continue
match = uscg_ais_nmea_regex.search(line).groupdict()
message_id = match['body'][0] # First letter is the message type
if message_id not in ('1','2','3'): continue
if len(match['body']) != 28: # 6 bits per character
raise AisErrorBadNumBits('expected 168, got %d' % len(match['body']) / 6)
bits = binary.ais6tobitvec(match['body'][:20]) # Don't need any of the other bits, so do not waste time
x = binary.signedIntFromBV(bits[61:89]) / 600000.
y = binary.signedIntFromBV(bits[89:116]) / 600000.
if x > 180 or y > 90:
counts['nogps'] += 1
continue
station = match['station']
julian_day = int(datetime.datetime.utcfromtimestamp(int(match['timeStamp'])).strftime('%j'))
d_km = dist_utm_km( (x,y), station_locations[station] )
#cu.execute('INSERT INTO distance VALUES (:julian_day, :x, :y, :dist_km, :station)',
#{'julian_day': julian_day, 'x':x, 'y':y, 'dist_km': d_km, 'station':station} )
#cu.execute('INSERT INTO distance VALUES (:julian_day, :x, :y, :dist_km)',
# {'julian_day': julian_day, 'x':x, 'y':y, 'dist_km': d_km, } )
cu.execute('INSERT INTO distance VALUES (:julian_day, :dist_km)',
{'julian_day': julian_day, 'dist_km': d_km, } )
if line_num % 10000 == 9999:
cx.commit()
cx.commit()
if False:
print 'Creating indexes'
try:
cx.execute('CREATE INDEX idx_dist_day ON distance(julian_day);')
cx.execute('CREATE INDEX idx_dist_dist ON distance(dist_km);')
#cx.execute('CREATE INDEX idx_dist_station ON distance(station);')
cx.commit()
except sqlite3.OperationalError:
print 'Appears indexes were already created'
return cx, counts
