sys.stderr.write("Web server running on port %d\n" % args.http_port)
if args.verbose:
sys.stdout.write("\n")
oldtime = time.time()
file_path = None
f_all = None
for message in consumer:
if message is not None:
(timestamp, source, data) = message.value.split('|', 3)
if source.startswith("20"): # wrong order on some early belmullet
tmp = timestamp
timestamp = source
source = tmp
if (data.startswith("!AIVDM")):
new_file_path = "/data/ais/{0}/{1}/{0}-{2}.txt".format(
source, timestamp[:4], timestamp[:7])
if new_file_path != file_path:
if f_all:
f_all.close()
file_path = new_file_path
if not os.path.exists(os.path.dirname(file_path)):
os.mkdir(os.path.dirname(file_path))
f_all = open(file_path, 'a')
f_all.write("{0}\n".format(message.value))
killer.ping()
webserver.update(message.value)
if args.verbose:
sys.stdout.write(message.value)
sys.stdout.flush()
values = message.value.split(',')
if(len(values) >= 11) and all(is_number(i) for i in values[2:10]):
(clock_date, clock_time, temp, cond, salinity, depth, ph, ph_mv, turbidity, do_mgl, battery,dummy) = values
(dd,mm,yyyy) = clock_date.split('/')
timestamp = "{0}-{1}-{2}T{3}Z".format(yyyy,mm,dd,clock_time)
with conn.cursor() as cursor:
try:
cursor.execute("""
INSERT INTO mestech (instrument_id, time, site_id, lat, lon, clock_date, clock_time, temp, cond, salinity, depth, ph, ph_mv, turbidity, do_mgl, battery)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)
""",
(instrument_id,timestamp,site_id,lat,lon,clock_date,clock_time,float(temp),float(cond),float(salinity),
float(depth), float(ph), float(ph_mv), float(turbidity), float(do_mgl), float(battery))
)
except pymssql.IntegrityError:
cursor.execute("""
update mestech set site_id=%s, lat=%s, lon=%s, clock_date=%s, clock_time=%s, temp=%s, cond=%s,
salinity=%s, depth=%s, ph=%s, ph_mv=%s, turbidity=%s, do_mgl=%s, battery=%s
where instrument_id=%s and time=%s
""",
(site_id,lat,lon,clock_date,clock_time,float(temp),float(cond),float(salinity),
float(depth), float(ph), float(ph_mv), float(turbidity), float(do_mgl), float(battery),instrument_id,timestamp)
)
consumer.commit_offsets()
killer.ping()
webserver.update(message.value)
if args.verbose:
print json.dumps({"clock_date": clock_date, "clock_time": clock_time, "temp": temp, "cond": cond,
"salinity": salinity, "depth": depth, "ph":ph, "ph_mv": ph_mv, "turbidity": turbidity, "do_mgl": do_mgl, "battery": battery})
def write_to_file(b):
global f
global filename
if not f:
filename = next_filename()
f = open(filename + '.tmp', 'wb')
f.write(b)
while True:
try:
write_to_file(s.recv(4096))
killer.ping()
webserver.update("(binary)")
nodata = False
except socket.error, e:
err = e.args[0]
if err == errno.EAGAIN or err == errno.EWOULDBLOCK:
if f and nodata and next_filename() != filename:
f.close()
f = None
os.rename(filename + '.tmp', filename)
if first_time or os.stat(filename).st_size < 211:
os.unlink(filename)
first_time = False
else:
try:
subprocess.call([
args.kafkacat, '-P', '-b', args.kafka_server, '-t',
import argparse
import subprocess
import os
from midas import WebServer, KillerMonitor
parser = argparse.ArgumentParser(description='Reads ais data from kafka, converts to json using gpsd, and writes to kafka')
parser.add_argument('--timeout', type=int, default=600, help='Number of seconds to wait for messages before giving up, default=600 (10 minutes)')
parser.add_argument('--http-port', type=int, default=8078, help='HTTP web server port showing latest message, default is 8078')
args = parser.parse_args()
killer = KillerMonitor(args.timeout)
killer.setDaemon(True)
killer.start()
sys.stderr.write("Monitor will kill application if unable to process a message for %d seconds\n" % args.timeout)
webserver = WebServer(args.http_port)
webserver.setDaemon(True)
webserver.start()
sys.stderr.write("Web server running on port %d\n" % args.http_port)
offsets_dir = "/var/lib/consumer-offsets/ais-rinville-1-2gpsdjson"
if not os.path.exists(offsets_dir):
os.makedirs(offsets_dir)
command = """kafkacat -X topic.offset.store.path=/var/lib/consumer-offsets/ais-rinville-1-2gpsdjson -o stored -C -u -b kafka01,kafka02,kafka03 -t ais-rinville-1 | stdbuf -oL sed -e 's/^[^|]*|[^|]*|//' | stdbuf -oL gpsdecode | stdbuf -oL kafkacat -P -T -b kafka01,kafka02,kafka03 -p 0 -t ais-rinville-1-gpsdjson """
process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
for line in iter(process.stdout.readline, ''):
killer.ping()
webserver.update(line)
parser.add_argument('--timeout', type=int, default=300, help='Number of seconds to wait for messages before giving up, default=300 (5 minutes)')
parser.add_argument('--http-port', type=int, default=8082, help='HTTP web server port showing latest message, default is 8082')
args = parser.parse_args()
if "rfc2217://" in args.device:
ser = rfc2217.Serial(args.device,args.baud,timeout=args.timeout)
elif "://" in args.device:
ser = serial_for_url(args.device,baudrate=args.baud,timeout=args.timeout)
else:
ser = Serial(args.device,baudrate=args.baud,timeout=args.timeout)
killer = KillerMonitor(args.timeout)
killer.setDaemon(True)
killer.start()
sys.stderr.write("Monitoring %s will kill application if unable to process a message for %d seconds\n" % (args.device, args.timeout))
webserver = WebServer(args.http_port)
webserver.setDaemon(True)
webserver.start()
sys.stderr.write("Web server running on port %d\n" % args.http_port)
while 1:
line = ser.readline().rstrip()
if line:
timestamp = datetime.datetime.utcnow().isoformat()[:-3]+"Z"
output = u"{1}{0}{2}{0}{3}".format(args.separator,timestamp,args.source,line)
print output
sys.stdout.flush()
killer.ping()
webserver.update(output)
return path
def write_to_file(b):
global f
global filename
if not f:
filename = next_filename()
f = open(filename+'.tmp', 'wb')
f.write(b)
while True:
try:
write_to_file(s.recv(4096))
killer.ping()
webserver.update("(binary)")
nodata = False
except socket.error, e:
err = e.args[0]
if err == errno.EAGAIN or err == errno.EWOULDBLOCK:
if f and nodata:
f.close()
f = None
os.rename(filename+'.tmp',filename)
if first_time or os.stat(filename).st_size < 211:
os.unlink(filename)
first_time = False
else:
try:
subprocess.call( [args.kafkacat, '-P', '-b', args.kafka_server, '-t',
args.kafka_topic, '-p', '0', filename])
o = json.loads(message.value)
except ValueError,e:
continue
if "mmsi" in o:
mmsi = o["mmsi"]
if not mmsi in cache:
cache[mmsi] = {
"type": "Feature",
"geometry": {
"type": "Point",
"coordinates": [0, 0]
},
"properties": {
"mmsi": mmsi
}
}
feature = cache[mmsi]
feature["properties"].update(o)
emitting = False
if "lat" in o and "lon" in o:
feature["properties"]["id"] = mmsi
feature["geometry"]["coordinates"] = [float(o["lon"]),float(o["lat"])]
emitting = True
cache[mmsi] = feature
if emitting:
answer = json.dumps(feature)
producer.produce(answer)
webserver.update(feature)
killer.ping()
