def main():
import optparse
parser = optparse.OptionParser('usage: %prog')
ZmqSubscriber.addOptions(parser, 'rasterMapIndexer')
parser.add_option('--timeSeries',
help='Specify a comma-separated subset of time series to index')
parser.add_option('-c', '--clean',
action='store_true', default=False,
help='Clean out old raster map data at startup')
parser.add_option('-q', '--quit',
action='store_true', default=False,
help='Quit after initial indexing is complete')
opts, args = parser.parse_args()
if args:
parser.error('expected no args')
logging.basicConfig(level=logging.INFO)
# insure atexit handlers are called on receiving SIGINT or SIGTERM
def sigHandler(signo, frame):
logging.warn('caught signal %s, exiting', signo)
sys.exit(0)
for sig in (signal.SIGINT, signal.SIGTERM):
signal.signal(sig, sigHandler)
rmi = RasterMapIndexer(opts)
if opts.clean:
rmi.clean()
rmi.start()
atexit.register(rmi.stop)
if not opts.quit:
zmqLoop()
def __init__(self, opts):
self.opts = opts
self.subscriber = ZmqSubscriber(
**ZmqSubscriber.getOptionValues(self.opts))
self.cacheDir = os.path.join(DATA_PATH, 'cache')
self.store = None
self.delayBox = DelayBox(self.writeOutputTile,
maxDelaySeconds=5,
numBuckets=10)
if opts.timeSeries:
timeSeriesList = opts.timeSeries.split(',')
print 'indexing only the following time series:'
for valueCode in timeSeriesList:
print ' %s' % valueCode
timeSeriesSet = set(opts.timeSeries.split(','))
else:
timeSeriesSet = None
self.indexes = {}
for m in meta.TIME_SERIES:
if 'map' in m:
if timeSeriesSet and m['valueCode'] not in timeSeriesSet:
continue
index = TileIndex(m, self)
self.indexes[m['valueCode']] = index
def main():
import optparse
parser = optparse.OptionParser('usage: %prog')
ZmqSubscriber.addOptions(parser, 'xgdsPlotIndexer')
parser.add_option('-c', '--clean',
action='store_true', default=False,
help='Delete indexes and start from scratch')
parser.add_option('-q', '--quit',
action='store_true', default=False,
help='Quit after initial indexing is complete')
parser.add_option('--timeSeries',
help='Comma-separated list of time series to index (by default index all)')
opts, args = parser.parse_args()
if args:
parser.error('expected no arguments')
logging.basicConfig(level=logging.INFO)
# insure atexit handlers are called on receiving SIGINT or SIGTERM
def sigHandler(signo, frame):
logging.warn('caught signal %s, exiting', signo)
sys.exit(0)
for sig in (signal.SIGINT, signal.SIGTERM):
signal.signal(sig, sigHandler)
x = XgdsPlotIndexer(opts)
if opts.clean:
x.clean()
x.start()
atexit.register(x.stop)
if not opts.quit:
zmqLoop()
def main():
import optparse
parser = optparse.OptionParser('usage: %prog OPTIONS\n' + __doc__)
parser.add_option('-f', '--watchFilePattern',
action='append', default=[],
help='Watch for files matching the specified UNIX glob, like "somedir/*.jpg"')
parser.add_option('-d', '--watchDirectory',
action='append', default=[],
help='Watch for files in the specified directory')
parser.add_option('-s', '--watchSymlink',
action='append', default=[],
help='Watch for files pointed to by the specified symlink. This assumes a cooperative file writer that updates a symlink whenever it writes a new file. This may be more efficient than the other approaches when new files are appearing in a directory containing many files.')
parser.add_option('-t', '--subtopic',
default='standard',
help='Subtopic to use when publishing zmq message [%default]')
ZmqPublisher.addOptions(parser, 'filePublisher.{subtopic}')
ZmqSubscriber.addOptions(parser, 'filePublisher.{subtopic}')
opts, args = parser.parse_args()
if args:
parser.error('expected no args')
logging.basicConfig(level=logging.DEBUG)
p = FilePublisher(opts)
p.start()
zmqLoop()
def __init__(self, opts):
self.opts = opts
self.subscriber = ZmqSubscriber(**ZmqSubscriber.getOptionValues(self.opts))
self.cacheDir = os.path.join(DATA_PATH, 'cache')
self.store = None
self.delayBox = DelayBox(self.writeOutputTile,
maxDelaySeconds=5,
numBuckets=10)
if opts.timeSeries:
timeSeriesList = opts.timeSeries.split(',')
print 'indexing only the following time series:'
for valueCode in timeSeriesList:
print ' %s' % valueCode
timeSeriesSet = set(opts.timeSeries.split(','))
else:
timeSeriesSet = None
self.indexes = {}
for m in meta.TIME_SERIES:
if 'map' in m:
if timeSeriesSet and m['valueCode'] not in timeSeriesSet:
continue
index = TileIndex(m, self)
self.indexes[m['valueCode']] = index
def __init__(self, opts):
self.opts = opts
self.subscriber = ZmqSubscriber(**ZmqSubscriber.getOptionValues(self.opts))
self.application = tornado.web.Application([
(r"^/?$", MainHandler),
(r"^/zmq/$", ClientSocket),
])
def __init__(self, opts):
self.opts = opts
self.subscriber = ZmqSubscriber(
**ZmqSubscriber.getOptionValues(self.opts))
self.application = tornado.web.Application([
(r"^/?$", MainHandler),
(r"^/zmq/$", ClientSocket),
])
class RasterMapIndexer(object):
def __init__(self, opts):
self.opts = opts
self.subscriber = ZmqSubscriber(
**ZmqSubscriber.getOptionValues(self.opts))
self.cacheDir = os.path.join(DATA_PATH, 'cache')
self.store = None
self.delayBox = DelayBox(self.writeOutputTile,
maxDelaySeconds=5,
numBuckets=10)
if opts.timeSeries:
timeSeriesList = opts.timeSeries.split(',')
print 'indexing only the following time series:'
for valueCode in timeSeriesList:
print ' %s' % valueCode
timeSeriesSet = set(opts.timeSeries.split(','))
else:
timeSeriesSet = None
self.indexes = {}
for m in meta.TIME_SERIES:
if 'map' in m:
if timeSeriesSet and m['valueCode'] not in timeSeriesSet:
continue
index = TileIndex(m, self)
self.indexes[m['valueCode']] = index
def writeOutputTile(self, info):
indexName, tileParams = info
self.indexes[indexName].writeOutputTile(tileParams)
def start(self):
self.store = LruCacheStore(FileStore(self.cacheDir),
MAX_TILES_IN_MEMORY)
self.delayBox.start()
self.subscriber.start()
for index in self.indexes.itervalues():
print
print '###################################################'
print '# initializing map index:', index.valueCode
print '###################################################'
index.start()
def stop(self):
logging.info('cleaning up indexer...')
self.store.sync()
self.delayBox.stop()
for index in self.indexes.itervalues():
index.stop()
logging.info(' ... done')
def clean(self):
plotUtil.rmIfPossible(self.cacheDir)
for index in self.indexes.itervalues():
index.clean()
class RasterMapIndexer(object):
def __init__(self, opts):
self.opts = opts
self.subscriber = ZmqSubscriber(**ZmqSubscriber.getOptionValues(self.opts))
self.cacheDir = os.path.join(DATA_PATH, 'cache')
self.store = None
self.delayBox = DelayBox(self.writeOutputTile,
maxDelaySeconds=5,
numBuckets=10)
if opts.timeSeries:
timeSeriesList = opts.timeSeries.split(',')
print 'indexing only the following time series:'
for valueCode in timeSeriesList:
print ' %s' % valueCode
timeSeriesSet = set(opts.timeSeries.split(','))
else:
timeSeriesSet = None
self.indexes = {}
for m in meta.TIME_SERIES:
if 'map' in m:
if timeSeriesSet and m['valueCode'] not in timeSeriesSet:
continue
index = TileIndex(m, self)
self.indexes[m['valueCode']] = index
def writeOutputTile(self, info):
indexName, tileParams = info
self.indexes[indexName].writeOutputTile(tileParams)
def start(self):
self.store = LruCacheStore(FileStore(self.cacheDir),
MAX_TILES_IN_MEMORY)
self.delayBox.start()
self.subscriber.start()
for index in self.indexes.itervalues():
print
print '###################################################'
print '# initializing map index:', index.valueCode
print '###################################################'
index.start()
def stop(self):
logging.info('cleaning up indexer...')
self.store.sync()
self.delayBox.stop()
for index in self.indexes.itervalues():
index.stop()
logging.info(' ... done')
def clean(self):
plotUtil.rmIfPossible(self.cacheDir)
for index in self.indexes.itervalues():
index.clean()
def main():
import optparse
parser = optparse.OptionParser('usage: %prog')
ZmqSubscriber.addOptions(parser, 'tracLinkTelemetryCleanup')
ZmqPublisher.addOptions(parser, 'tracLinkTelemetryCleanup')
opts, args = parser.parse_args()
if args:
parser.error('expected no args')
logging.basicConfig(level=logging.DEBUG)
d = TracLinkTelemetryCleanup(opts)
d.start()
atexit.register(d.flush)
zmqLoop()
def __init__(self, opts):
self.sources = ([PatternFileSource(x) for x in opts.watchFilePattern] +
[DirectoryFileSource(x) for x in opts.watchDirectory] +
[SymlinkFileSource(x) for x in opts.watchSymlink])
self.subtopic = opts.subtopic
self.pollTimer = None
self.stopPollingTime = None
self.imageProcessor = None
self.tmpDir = tempfile.mkdtemp(prefix='filePublisher')
opts.moduleName = opts.moduleName.format(subtopic=opts.subtopic)
self.publisher = ZmqPublisher(**ZmqPublisher.getOptionValues(opts))
self.subscriber = ZmqSubscriber(**ZmqSubscriber.getOptionValues(opts))
def main():
import optparse
parser = optparse.OptionParser('usage: %prog')
parser.add_option('-p', '--port',
type='int', default=8001,
help='TCP port where websocket server should listen [%default]')
ZmqSubscriber.addOptions(parser, 'zmqProxy')
opts, args = parser.parse_args()
if args:
parser.error('expected no args')
global proxyG
proxyG = ZmqProxy(opts)
proxyG.start()
zmqLoop()
class ZmqProxy(object):
def __init__(self, opts):
self.opts = opts
self.subscriber = ZmqSubscriber(**ZmqSubscriber.getOptionValues(self.opts))
self.application = tornado.web.Application([
(r"^/?$", MainHandler),
(r"^/zmq/$", ClientSocket),
])
def start(self):
# initialize zmq
self.subscriber.start()
# start serving web clients
print 'binding to port %d' % self.opts.port
self.application.listen(self.opts.port)
class ZmqProxy(object):
def __init__(self, opts):
self.opts = opts
self.subscriber = ZmqSubscriber(
**ZmqSubscriber.getOptionValues(self.opts))
self.application = tornado.web.Application([
(r"^/?$", MainHandler),
(r"^/zmq/$", ClientSocket),
])
def start(self):
# initialize zmq
self.subscriber.start()
# start serving web clients
print 'binding to port %d' % self.opts.port
self.application.listen(self.opts.port)
def __init__(self, opts):
self.opts = opts
if self.opts.timeSeries:
names = self.opts.timeSeries.split(',')
for name in names:
if name not in TIME_SERIES_LOOKUP:
print ('unknown time series %s, expected one of %s'
% (name, TIME_SERIES_LOOKUP.keys()))
sys.exit(1)
timeSeriesList = [TIME_SERIES_LOOKUP[name]
for name in names]
else:
timeSeriesList = TIME_SERIES
self.subscriber = ZmqSubscriber(**ZmqSubscriber.getOptionValues(self.opts))
self.indexes = {}
for meta in timeSeriesList:
index = SegmentIndex(meta, self.subscriber)
self.indexes[meta['valueCode']] = index
def main():
import optparse
parser = optparse.OptionParser('usage: %prog')
parser.add_option(
'-p',
'--port',
type='int',
default=8001,
help='TCP port where websocket server should listen [%default]')
ZmqSubscriber.addOptions(parser, 'zmqProxy')
opts, args = parser.parse_args()
if args:
parser.error('expected no args')
global proxyG
proxyG = ZmqProxy(opts)
proxyG.start()
zmqLoop()
def main():
import optparse
parser = optparse.OptionParser('usage: %prog')
ZmqSubscriber.addOptions(parser, 'testSubscriber')
opts, args = parser.parse_args()
if args:
parser.error('expected no args')
logging.basicConfig(level=logging.DEBUG)
# set up networking
s = ZmqSubscriber(**ZmqSubscriber.getOptionValues(opts))
s.start()
# subscribe to the message we want
s.subscribeRaw('geocamUtil.greeting:', handleGreeting)
zmqLoop()
class XgdsPlotIndexer(object):
def __init__(self, opts):
self.opts = opts
if self.opts.timeSeries:
names = self.opts.timeSeries.split(',')
for name in names:
if name not in TIME_SERIES_LOOKUP:
print ('unknown time series %s, expected one of %s'
% (name, TIME_SERIES_LOOKUP.keys()))
sys.exit(1)
timeSeriesList = [TIME_SERIES_LOOKUP[name]
for name in names]
else:
timeSeriesList = TIME_SERIES
self.subscriber = ZmqSubscriber(**ZmqSubscriber.getOptionValues(self.opts))
self.indexes = {}
for meta in timeSeriesList:
index = SegmentIndex(meta, self.subscriber)
self.indexes[meta['valueCode']] = index
def start(self):
self.subscriber.start()
for index in self.indexes.itervalues():
print
print '###################################################'
print '# initializing time series:', index.valueCode
print '###################################################'
index.start()
def stop(self):
logging.info('cleaning up indexer...')
for index in self.indexes.itervalues():
index.stop()
logging.info(' ... done')
def clean(self):
for index in self.indexes.itervalues():
index.clean()
def main():
import optparse
parser = optparse.OptionParser('usage: %prog OPTIONS\n' + __doc__)
parser.add_option('-o', '--output',
default='.',
help='Directory to write received files to [%default]')
parser.add_option('-p', '--pollPeriod',
type='float', default=1.0,
help='Period between checks for a new latest file (seconds) [%default]')
parser.add_option('-s', '--timestampSpacing',
type='float', default=5.0,
help='Ignore latest file unless it is this much newer than the last file sent (seconds) [%default]')
parser.add_option('--timeout',
type='float', default=30.0,
help='Keep publishing files for this long after each request from fileReceiver (seconds) [%default]')
parser.add_option('-r', '--imageResize',
help='Subsample any images to specified size (e.g. "640x480")')
parser.add_option('-c', '--imageCrop',
help='Crop any images to specified size and location (e.g. "100x100+300+400")')
parser.add_option('-f', '--imageFormat',
help='Output any images in the specified format (e.g. "jpg")')
parser.add_option('-t', '--subtopic',
default='standard',
help='Subtopic to use when receiving zmq message [%default]')
parser.add_option('-n', '--noRequest',
action='store_true', default=False,
help='Do not request any files from filePublisher, but write any files that are received, for example because they are requested by other receivers. May be useful for debugging.')
ZmqPublisher.addOptions(parser, 'fileReceiver.{subtopic}')
ZmqSubscriber.addOptions(parser, 'fileReceiver.{subtopic}')
opts, args = parser.parse_args()
if args:
parser.error('expected no args')
logging.basicConfig(level=logging.DEBUG)
p = FileReceiver(opts)
p.start()
zmqLoop()
def main():
import optparse
parser = optparse.OptionParser('usage: %prog OPTIONS\n' + __doc__)
parser.add_option('-o', '--output',
default='.',
help='Directory to write received files to [%default]')
parser.add_option('-p', '--pollPeriod',
type='float', default=1.0,
help='Period between checks for a new latest file (seconds) [%default]')
parser.add_option('-s', '--timestampSpacing',
type='float', default=5.0,
help='Ignore latest file unless it is this much newer than the last file sent (seconds) [%default]')
parser.add_option('--timeout',
type='float', default=30.0,
help='Keep publishing files for this long after each request from fileReceiver (seconds) [%default]')
parser.add_option('-r', '--imageResize',
help='Subsample any images to specified size (e.g. "640x480")')
parser.add_option('-c', '--imageCrop',
help='Crop any images to specified size and location (e.g. "100x100+300+400")')
parser.add_option('-f', '--imageFormat',
help='Output any images in the specified format (e.g. "jpg")')
parser.add_option('-t', '--subtopic',
default='standard',
help='Subtopic to use when receiving zmq message [%default]')
parser.add_option('-n', '--noRequest',
action='store_true', default=False,
help='Do not request any files from filePublisher, but write any files that are received, for example because they are requested by other receivers. May be useful for debugging.')
ZmqPublisher.addOptions(parser, 'fileReceiver.{subtopic}')
ZmqSubscriber.addOptions(parser, 'fileReceiver.{subtopic}')
opts, args = parser.parse_args()
if args:
parser.error('expected no args')
logging.basicConfig(level=logging.DEBUG)
p = FileReceiver(opts)
p.start()
zmqLoop()
def __init__(self, opts):
self.request = {
'timeout': opts.timeout,
'pollPeriod': opts.pollPeriod,
}
if opts.timestampSpacing:
self.request['timestampSpacing'] = opts.timestampSpacing
if opts.imageResize:
self.request['imageResize'] = parseImageResize(opts.imageResize)
if opts.imageCrop:
self.request['imageCrop'] = parseImageCrop(opts.imageCrop)
if opts.imageFormat:
self.request['imageFormat'] = opts.imageFormat
self.outputDirectory = opts.output
self.subtopic = opts.subtopic
self.noRequest = opts.noRequest
opts.moduleName = opts.moduleName.format(subtopic=opts.subtopic)
self.publisher = ZmqPublisher(**ZmqPublisher.getOptionValues(opts))
self.subscriber = ZmqSubscriber(**ZmqSubscriber.getOptionValues(opts))
self.requestPeriod = 0.5 * opts.timeout
def __init__(self, opts):
self.request = {
'timeout': opts.timeout,
'pollPeriod': opts.pollPeriod,
}
if opts.timestampSpacing:
self.request['timestampSpacing'] = opts.timestampSpacing
if opts.imageResize:
self.request['imageResize'] = parseImageResize(opts.imageResize)
if opts.imageCrop:
self.request['imageCrop'] = parseImageCrop(opts.imageCrop)
if opts.imageFormat:
self.request['imageFormat'] = opts.imageFormat
self.outputDirectory = opts.output
self.subtopic = opts.subtopic
self.noRequest = opts.noRequest
opts.moduleName = opts.moduleName.format(subtopic=opts.subtopic)
self.publisher = ZmqPublisher(**ZmqPublisher.getOptionValues(opts))
self.subscriber = ZmqSubscriber(**ZmqSubscriber.getOptionValues(opts))
self.requestPeriod = 0.5 * opts.timeout
def main():
import optparse
parser = optparse.OptionParser('usage: %prog')
ZmqSubscriber.addOptions(parser, 'rasterMapIndexer')
parser.add_option(
'--timeSeries',
help='Specify a comma-separated subset of time series to index')
parser.add_option('-c',
'--clean',
action='store_true',
default=False,
help='Clean out old raster map data at startup')
parser.add_option('-q',
'--quit',
action='store_true',
default=False,
help='Quit after initial indexing is complete')
opts, args = parser.parse_args()
if args:
parser.error('expected no args')
logging.basicConfig(level=logging.INFO)
# insure atexit handlers are called on receiving SIGINT or SIGTERM
def sigHandler(signo, frame):
logging.warn('caught signal %s, exiting', signo)
sys.exit(0)
for sig in (signal.SIGINT, signal.SIGTERM):
signal.signal(sig, sigHandler)
rmi = RasterMapIndexer(opts)
if opts.clean:
rmi.clean()
rmi.start()
atexit.register(rmi.stop)
if not opts.quit:
zmqLoop()
def main():
import optparse
parser = optparse.OptionParser('usage: %prog')
ZmqSubscriber.addOptions(parser, 'testSubscriber')
opts, args = parser.parse_args()
if args:
parser.error('expected no args')
logging.basicConfig(level=logging.DEBUG)
# set up networking
s = ZmqSubscriber(**ZmqSubscriber.getOptionValues(opts))
s.start()
# subscribe to the message we want
s.subscribeRaw('geocamUtil.greeting:', handleGreeting)
zmqLoop()
def main():
import optparse
parser = optparse.OptionParser('usage: %prog <prefix>')
parser.add_option('-p', '--pretty',
action='store_true', default=False,
help='Pretty-print JSON objects')
ZmqSubscriber.addOptions(parser, 'zmqGrep')
opts, args = parser.parse_args()
if len(args) != 1:
parser.error('expected exactly 1 arg')
logging.basicConfig(level=logging.DEBUG)
# set up networking
s = ZmqSubscriber(**ZmqSubscriber.getOptionValues(opts))
s.start()
# subscribe to the message we want
topic = args[0]
if opts.pretty:
s.subscribeJson(topic, handleMessagePretty)
else:
s.subscribeRaw(topic, handleMessageSimple)
zmqLoop()
class GpsTelemetryCleanup(object):
def __init__(self, opts):
self.opts = opts
self.subscriber = ZmqSubscriber(
**ZmqSubscriber.getOptionValues(self.opts))
self.publisher = ZmqPublisher(
**ZmqPublisher.getOptionValues(self.opts))
def start(self):
self.publisher.start()
self.subscriber.start()
topics = ['gpsposition', 'compass']
for topic in topics:
self.subscriber.subscribeRaw(topic + ':',
getattr(self, 'handle_' + topic))
def flush(self):
# flush bulk saves to db if needed. currently no-op.
pass
def handle_gpsposition(self, topic, body):
try:
self.handle_gpsposition0(topic, body)
except: # pylint: disable=W0702
logging.warning('%s', traceback.format_exc())
logging.warning('exception caught, continuing')
def handle_compass(self, topic, body):
try:
self.handle_compass0(topic, body)
except: # pylint: disable=W0702
logging.warning('%s', traceback.format_exc())
logging.warning('exception caught, continuing')
def adjustHeading(self, compassRecord):
cc = settings.COMPASS_CORRECTION
compassRecord['compass'] = compassRecord['compass'] + cc
if compassRecord['compass'] > 360:
compassRecord['compass'] -= 360
def parseCompassData(self, compassSentence):
# Sample compass NMEA sentence: $R92.3P-0.3C359.8X219.4Y-472.8Z19.7T35.4D270.1A87.7*6F
compassReParsed = re.match(
"\$(?P<rollLbl>[A-Z])(?P<roll>-*[0-9\.]+)(?P<pitchLbl>[A-Z])(?P<pitch>-*[0-9\.]+)(?P<compassLbl>[A-Z])(?P<compass>-*[0-9\.]+)(?P<xLbl>[A-Z])(?P<x>-*[0-9\.]+)(?P<yLbl>[A-Z])(?P<y>-*[0-9\.]+)(?P<zLbl>[A-Z])(?P<z>-*[0-9\.]+)(?P<tempLbl>[A-Z])(?P<temp>-*[0-9\.]+)(?P<drillDLbl>[A-Z])(?P<drillD>-*[0-9\.]+)(?P<drillALbl>[A-Z])(?P<drillA>-*[0-9\.]+)",
compassSentence)
compassRecord = {
"roll": float(compassReParsed.group('roll')),
"pitch": float(compassReParsed.group('pitch')),
"compass": float(compassReParsed.group('compass')),
"x": float(compassReParsed.group('x')),
"y": float(compassReParsed.group('y')),
"z": float(compassReParsed.group('z')),
"temp": float(compassReParsed.group('temp')),
"drillD": float(compassReParsed.group('drillD')),
"drillA": float(compassReParsed.group('drillA'))
}
return compassRecord
def handle_compass0(self, topic, body):
# example: 2:$GPRMC,225030.00,A,3725.1974462,N,12203.8994696,W,,,220216,0.0,E,A*2B
serverTimestamp = datetime.datetime.now(pytz.utc)
if body == 'NO DATA':
logging.info('NO DATA')
return
# parse record
resourceIdStr, compassStr, content = body.split(":", 2)
resourceId = int(resourceIdStr)
if not checkDataQuality(resourceId, content):
logging.info('UNRECOGNIZED OR CORRUPT COMPASS SENTENCE: %s',
content)
return
compassRecord = self.parseCompassData(content)
self.adjustHeading(compassRecord)
sourceTimestamp = serverTimestamp # Compass has no independent clock
# save subsystem status to cache
myKey = "compassCleanupEV%s" % resourceIdStr
status = {
'name': myKey,
'displayName': 'Compass Cleanup EV%s' % str(resourceIdStr),
'statusColor': '#00ff00',
'lastUpdated': datetime.datetime.utcnow().isoformat(),
'elapsedTime': ''
}
cache.set(myKey, json.dumps(status))
# save latest compass reading in memcache for GPS use
cacheKey = 'compass.%s' % resourceId
cacheRecordDict = {
"timestamp": sourceTimestamp,
"compassRecord": compassRecord
}
cache.set(cacheKey, json.dumps(cacheRecordDict,
cls=DatetimeJsonEncoder))
def handle_gpsposition0(self, topic, body):
# example: 2:$GPRMC,225030.00,A,3725.1974462,N,12203.8994696,W,,,220216,0.0,E,A*2B
serverTimestamp = datetime.datetime.now(pytz.utc)
if body == 'NO DATA':
logging.info('NO DATA')
return
# parse record
resourceIdStr, trackName, content = body.split(":", 2)
resourceId = int(resourceIdStr)
if not checkDataQuality(resourceId, content):
logging.info('UNRECOGNIZED OR CORRUPT GPS SENTENCE: %s', content)
return
sentenceType, utcTime, activeVoid, lat, latHemi, lon,\
lonHemi, speed, heading, date, declination, declinationDir,\
modeAndChecksum = content.split(",")
if OVERRIDE_GPS_DATE:
serverTime = datetime.datetime.now(pytz.utc)
overrideDate = serverTime.strftime("%d%m%y")
sourceTimestamp = datetime.datetime.strptime(
'%s %s' % (overrideDate, utcTime), '%d%m%y %H%M%S.%f')
else:
sourceTimestamp = datetime.datetime.strptime(
'%s %s' % (date, utcTime), '%d%m%y %H%M%S.%f')
sourceTimestamp = sourceTimestamp.replace(tzinfo=pytz.utc)
lat = parseTracLinkDM(lat, latHemi)
lon = parseTracLinkDM(lon, lonHemi)
# Get compass heading from compass record
# TODO this clobbers heading read from GPS every time. but this is for basalt. do we care?
heading = None
compassCacheKey = 'compass.%s' % resourceId
compassInfoString = cache.get(compassCacheKey)
try:
if compassInfoString:
compassInfo = json.loads(compassInfoString)
compassRecord = compassInfo["compassRecord"]
# sanity check the timestamp in the compass record
compassTimeString = compassInfo['timestamp']
compassTimestamp = dateparser(compassTimeString)
tdelta = serverTimestamp - compassTimestamp
if tdelta.total_seconds() <= MAX_COMPASS_TIME_SECONDS:
heading = float(compassRecord["compass"])
except:
traceback.print_exc()
# save subsystem status to cache
myKey = "telemetryCleanup"
status = {
'name': myKey,
'displayName': 'Telemetry Cleanup',
'statusColor': OKAY_COLOR,
'lastUpdated': datetime.datetime.utcnow().isoformat(),
'elapsedTime': ''
}
cache.set(myKey, json.dumps(status))
# calculate which track record belongs to
cacheKey = 'gpstrack.%s' % resourceId
pickledTrack = cache.get(cacheKey)
if pickledTrack:
# cache hit, great
track = pickle.loads(pickledTrack)
else:
# check db for a track matching this resourceId
try:
basaltVehicle = BasaltVehicle.objects.get(
resourceId=resourceId)
except ObjectDoesNotExist:
logging.warning('%s', traceback.format_exc())
raise KeyError(
'Received GPS position for the EV with resourceId %s. Please ensure there is a vehicle with that id in the BasaltVehicle table.'
% resourceId)
# Check for track name. We use explicit name if specified, otherwise
# we check for an active flight and finally use the resourceId
if len(trackName):
logging.info("Using track name from listener: %s" % trackName)
if len(trackName
) == 0: # I.e. we were not given a name for track already
try:
activeFlight = BasaltActiveFlight.objects.get(
flight__vehicle=basaltVehicle)
trackName = activeFlight.flight.name
logging.info(
"Using track name from BasaltActiveFlight: %s" %
trackName)
except ObjectDoesNotExist:
trackName = basaltVehicle.name
logging.info("Using track name from EV arg: %s" %
trackName)
tracks = BasaltTrack.objects.filter(name=trackName)
assert len(tracks) in (0, 1)
if tracks:
# we already have a valid track, use that
track = tracks[0]
else:
# must start a new track
track = BasaltTrack(name=trackName,
vehicle=basaltVehicle,
iconStyle=DEFAULT_ICON_STYLE,
lineStyle=DEFAULT_LINE_STYLE,
dataType=RAW_DATA_TYPE)
track.save()
# set cache for next time
pickledTrack = pickle.dumps(track, pickle.HIGHEST_PROTOCOL)
cache.set(cacheKey, pickledTrack, TRACK_CACHE_TIMEOUT)
######################################################################
# asset position
######################################################################
# create a NewAssetPosition row
params = {
'track': track,
'timestamp': sourceTimestamp,
'serverTimestamp': serverTimestamp,
'latitude': lat,
'longitude': lon,
'heading': heading,
'altitude': None,
}
pos = PastPosition(**params)
pos.save() # note: could queue for bulk save instead
cpos = CurrentPosition(**params)
cpos.saveCurrent()
pos.broadcast()
self.publisher.sendDjango(cpos)
class TracLinkTelemetryCleanup(object):
def __init__(self, opts):
self.opts = opts
self.subscriber = ZmqSubscriber(**ZmqSubscriber.getOptionValues(self.opts))
self.publisher = ZmqPublisher(**ZmqPublisher.getOptionValues(self.opts))
def start(self):
self.publisher.start()
self.subscriber.start()
topics = ['traclink']
for topic in topics:
self.subscriber.subscribeRaw(topic + ':', getattr(self, 'handle_' + topic))
def flush(self):
# flush bulk saves to db if needed. currently no-op.
pass
def handle_traclink(self, topic, body):
try:
self.handle_traclink0(topic, body)
except: # pylint: disable=W0702
logging.warning('%s', traceback.format_exc())
logging.warning('exception caught, continuing')
def handle_traclink0(self, topic, body):
# example: 13,09/19/2013,05:17:39, 2831.3070, -8038.8460, 2831.3068, -8038.8459,205.2, 0.9
serverTimestamp = datetime.datetime.now(pytz.utc)
if body == 'NO DATA':
logging.info('NO DATA')
return
# parse record
targetId, d, t, shipLat, shipLon, lat, lon, shipHeading, depth = body.split(',')
targetId = int(targetId)
sourceTimestamp = datetime.datetime.strptime('%s %s' % (d, t), '%m/%d/%y %H:%M:%S')
lat = parseTracLinkDM(lat)
lon = parseTracLinkDM(lon)
shipLat = parseTracLinkDM(shipLat)
shipLon = parseTracLinkDM(shipLon)
shipHeading = float(shipHeading)
depth = float(depth)
# calculate which track record belongs to
cacheKey = 'traclink.track.%s' % targetId
pickledTrack = cache.get(cacheKey)
if pickledTrack:
# cache hit, great
track = pickle.loads(pickledTrack)
else:
# check db for a track matching this targetId
try:
beacon = Beacon.objects.get(targetId=targetId)
except ObjectDoesNotExist:
logging.warning('%s', traceback.format_exc())
raise KeyError('Received TracLink position for the beacon with targetId %s. Please ensure there is a beacon with that targetId in the plrpExplorer Beacon table.' % targetId)
try:
activeFlight = ActiveFlight.objects.get(flight__beacon=beacon)
except ObjectDoesNotExist:
raise KeyError('Received TracLink position for the beacon with targetId %s (named "%s"). Please ensure there is an active flight using that beacon in the plrpExplorer NewFlight table.' % (targetId, beacon.name))
flight = activeFlight.flight
tracks = Track.objects.filter(vehicle=flight.vehicle)
assert len(tracks) in (0, 1)
if tracks:
# we already have a valid track, use that
track = tracks[0]
else:
# must start a new track
track = Track(name=flight.name,
resource=flight,
iconStyle=DEFAULT_ICON_STYLE,
lineStyle=DEFAULT_LINE_STYLE,
dataType=RAW_DATA_TYPE)
track.save()
# set cache for next time
pickledTrack = pickle.dumps(track, pickle.HIGHEST_PROTOCOL)
cache.set(cacheKey, pickledTrack,
settings.PLRP_TRACK_CACHE_TIMEOUT_SECONDS)
######################################################################
# asset position
######################################################################
# create a NewAssetPosition row
params = {
'track': track,
'timestamp': serverTimestamp,
'latitude': lat,
'longitude': lon,
'heading': None, # traclink doesn't provide heading for tracked object
'depthMeters': depth,
'sourceTimestamp': sourceTimestamp,
'serverTimestamp': serverTimestamp,
}
pos = NewAssetPosition(**params)
pos.save() # note: could queue for bulk save instead
cpos = NewAssetCurrentPosition(**params)
cpos.saveCurrent()
self.publisher.sendDjango(cpos)
# add fields to create a NewAssetPositionTracLink row
params.update({
'summary': pos,
'targetId': targetId,
'shipLatitude': shipLat,
'shipLongitude': shipLon,
'shipHeading': shipHeading,
})
posTracLink = NewAssetPositionTracLink(**params)
posTracLink.save() # note: could queue for bulk save instead
######################################################################
# boat position
######################################################################
params = {
'track': BOAT_TRACK,
'timestamp': serverTimestamp,
'latitude': shipLat,
'longitude': shipLon,
'heading': shipHeading,
'depthMeters': 0.0,
'sourceTimestamp': sourceTimestamp,
'serverTimestamp': serverTimestamp,
}
boatPos = NewAssetPosition(**params)
boatPos.save()
boatCPos = NewAssetCurrentPosition(**params)
boatCPos.saveCurrent()
self.publisher.sendDjango(boatCPos)
class GpsTelemetryCleanup(object):
def __init__(self, opts):
self.opts = opts
self.subscriber = ZmqSubscriber(
**ZmqSubscriber.getOptionValues(self.opts))
self.publisher = ZmqPublisher(
**ZmqPublisher.getOptionValues(self.opts))
def start(self):
self.publisher.start()
self.subscriber.start()
topics = ['gpsposition']
for topic in topics:
self.subscriber.subscribeRaw(topic + ':',
getattr(self, 'handle_' + topic))
def flush(self):
# flush bulk saves to db if needed. currently no-op.
pass
def handle_gpsposition(self, topic, body):
try:
self.handle_gpsposition0(topic, body)
except: # pylint: disable=W0702
logging.warning('%s', traceback.format_exc())
logging.warning('exception caught, continuing')
def handle_gpsposition0(self, topic, body):
# example: 2:$GPRMC,225030.00,A,3725.1974462,N,12203.8994696,W,,,220216,0.0,E,A*2B
serverTimestamp = datetime.datetime.now(pytz.utc)
if body == 'NO DATA':
logging.info('NO DATA')
return
# parse record
resourceIdStr, trackName, content = body.split(":", 2)
resourceId = int(resourceIdStr)
if not checkDataQuality(resourceId, content):
logging.info('UNRECOGNIZED OR CORRUPT GPS SENTENCE: %s', content)
return
sentenceType, utcTime, activeVoid, lat, latHemi, lon,\
lonHemi, speed, heading, date, declination, declinationDir,\
modeAndChecksum = content.split(",")
sourceTimestamp = datetime.datetime.strptime('%s %s' % (date, utcTime),
'%d%m%y %H%M%S.%f')
sourceTimestamp = sourceTimestamp.replace(tzinfo=pytz.utc)
lat = parseTracLinkDM(lat, latHemi)
lon = parseTracLinkDM(lon, lonHemi)
# save subsystem status to cache
myKey = "telemetryCleanup"
status = {'lastUpdated': datetime.datetime.utcnow().isoformat()}
cache.set(myKey, json.dumps(status))
# calculate which track record belongs to
cacheKey = 'gpstrack.%s' % resourceId
pickledTrack = cache.get(cacheKey)
if pickledTrack:
# cache hit, great
track = pickle.loads(pickledTrack)
else:
# check db for a track matching this resourceId
try:
basaltResource = BasaltResource.objects.get(
resourceId=resourceId)
except ObjectDoesNotExist:
logging.warning('%s', traceback.format_exc())
raise KeyError(
'Received GPS position for the EV with id %s. Please ensure there is a vehicle with that id in the BasaltResource table.'
% resourceId)
# Check for track name. We use explicit name if specified, otherwise
# we check for an active flight and finally use the resourceId
if len(trackName):
logging.info("Using track name from listener: %s" % trackName)
if len(trackName
) == 0: # I.e. we were not given a name for track already
try:
activeFlight = BasaltActiveFlight.objects.get(
flight__vehicle__basaltresource=basaltResource)
trackName = activeFlight.flight.name
logging.info(
"Using track name from BasaltActiveFlight: %s" %
trackName)
except ObjectDoesNotExist:
trackName = basaltResource.name
logging.info("Using track name from EV arg: %s" %
trackName)
tracks = BasaltTrack.objects.filter(name=trackName)
assert len(tracks) in (0, 1)
if tracks:
# we already have a valid track, use that
track = tracks[0]
else:
# must start a new track
track = BasaltTrack(name=trackName,
resource=basaltResource,
iconStyle=DEFAULT_ICON_STYLE,
lineStyle=DEFAULT_LINE_STYLE,
dataType=RAW_DATA_TYPE)
track.save()
# set cache for next time
pickledTrack = pickle.dumps(track, pickle.HIGHEST_PROTOCOL)
cache.set(cacheKey, pickledTrack, TRACK_CACHE_TIMEOUT)
######################################################################
# asset position
######################################################################
# create a NewAssetPosition row
params = {
'track': track,
'timestamp': sourceTimestamp,
'serverTimestamp': serverTimestamp,
'latitude': lat,
'longitude': lon,
# may not have heading, but we'll try...
'heading': float(heading) if len(heading) else None,
'altitude': None,
}
pos = PastPosition(**params)
pos.save() # note: could queue for bulk save instead
cpos = CurrentPosition(**params)
cpos.saveCurrent()
self.publisher.sendDjango(cpos)
class FilePublisher(object):
def __init__(self, opts):
self.sources = ([PatternFileSource(x) for x in opts.watchFilePattern] +
[DirectoryFileSource(x) for x in opts.watchDirectory] +
[SymlinkFileSource(x) for x in opts.watchSymlink])
self.subtopic = opts.subtopic
self.pollTimer = None
self.stopPollingTime = None
self.imageProcessor = None
self.tmpDir = tempfile.mkdtemp(prefix='filePublisher')
opts.moduleName = opts.moduleName.format(subtopic=opts.subtopic)
self.publisher = ZmqPublisher(**ZmqPublisher.getOptionValues(opts))
self.subscriber = ZmqSubscriber(**ZmqSubscriber.getOptionValues(opts))
def getTopic(self, msgType):
return 'geocamUtil.filePublisher.%s.%s' % (self.subtopic, msgType)
def start(self):
self.publisher.start()
self.subscriber.start()
self.subscriber.subscribeJson(self.getTopic('request'),
self.handleRequest)
def handleRequest(self, topic, requestDict):
logging.debug('handleRequest %s', json.dumps(requestDict))
self.requestTimeout = requestDict['timeout']
self.pollPeriod = requestDict['pollPeriod']
self.timestampSpacing = requestDict.get('timestampSpacing')
if 'imageResize' in requestDict or 'imageCrop' in requestDict or 'imageFormat' in requestDict:
self.imageProcessor = ImageProcessor(resize=requestDict.get('imageResize'),
crop=requestDict.get('imageCrop'),
fmt=requestDict.get('imageFormat'),
tmpDir=self.tmpDir)
else:
self.imageProcessor = None
self.stopPollingTime = time.time() + self.requestTimeout
self.publisher.sendRaw(self.getTopic('response'), 'ok')
if self.pollTimer:
self.pollTimer.stop()
self.pollTimer = None
self.pollTimer = ioloop.PeriodicCallback(self.pollHandler,
self.pollPeriod * 1000)
self.pollTimer.start()
def pollHandler(self):
try:
self.pollHandler0()
except: # pylint: disable=W0702
logging.warning('%s', traceback.format_exc())
def pollHandler0(self):
logging.debug('pollHandler')
if time.time() > self.stopPollingTime:
logging.info('request timed out, stopping polling')
self.pollTimer.stop()
self.pollTimer = None
for source in self.sources:
newFileInfo = source.checkForNewFileAndRemember(self.timestampSpacing)
if newFileInfo:
self.publishFile(newFileInfo)
def publishFile(self, fileInfo):
path, mtime = fileInfo
if self.imageProcessor and self.imageProcessor.isImage(path):
processedPath = self.imageProcessor.processImage(path)
logging.debug('sending %s', processedPath)
self.publisher.sendJson(self.getTopic('file'),
{'file': getFileDict(processedPath, mtime)})
os.unlink(processedPath)
else:
logging.debug('sending %s', path)
self.publisher.sendJson(self.getTopic('file'),
{'file': getFileDict(path, mtime)})
def main():
import optparse
parser = optparse.OptionParser('usage: %prog <prefix>')
parser.add_option('-p',
'--pretty',
action='store_true',
default=False,
help='Pretty-print JSON objects')
ZmqSubscriber.addOptions(parser, 'zmqGrep')
opts, args = parser.parse_args()
if len(args) != 1:
parser.error('expected exactly 1 arg')
logging.basicConfig(level=logging.DEBUG)
# set up networking
s = ZmqSubscriber(**ZmqSubscriber.getOptionValues(opts))
s.start()
# subscribe to the message we want
topic = args[0]
if opts.pretty:
s.subscribeJson(topic, handleMessagePretty)
else:
s.subscribeRaw(topic, handleMessageSimple)
zmqLoop()
def __init__(self, opts):
self.opts = opts
self.subscriber = ZmqSubscriber(**ZmqSubscriber.getOptionValues(self.opts))
self.publisher = ZmqPublisher(**ZmqPublisher.getOptionValues(self.opts))
class FileReceiver(object):
def __init__(self, opts):
self.request = {
'timeout': opts.timeout,
'pollPeriod': opts.pollPeriod,
}
if opts.timestampSpacing:
self.request['timestampSpacing'] = opts.timestampSpacing
if opts.imageResize:
self.request['imageResize'] = parseImageResize(opts.imageResize)
if opts.imageCrop:
self.request['imageCrop'] = parseImageCrop(opts.imageCrop)
if opts.imageFormat:
self.request['imageFormat'] = opts.imageFormat
self.outputDirectory = opts.output
self.subtopic = opts.subtopic
self.noRequest = opts.noRequest
opts.moduleName = opts.moduleName.format(subtopic=opts.subtopic)
self.publisher = ZmqPublisher(**ZmqPublisher.getOptionValues(opts))
self.subscriber = ZmqSubscriber(**ZmqSubscriber.getOptionValues(opts))
self.requestPeriod = 0.5 * opts.timeout
def getTopic(self, msgType):
return 'geocamUtil.filePublisher.%s.%s' % (self.subtopic, msgType)
def start(self):
self.publisher.start()
self.subscriber.start()
self.subscriber.subscribeJson(self.getTopic('file'),
self.handleFile)
self.subscriber.subscribeRaw(self.getTopic('response'),
self.handleResponse)
if not self.noRequest:
self.sendRequest()
requestTimer = ioloop.PeriodicCallback(self.sendRequest,
self.requestPeriod * 1000)
requestTimer.start()
def sendRequest(self):
logging.debug('sendRequest')
self.publisher.sendJson(self.getTopic('request'),
self.request)
def handleResponse(self, topic, msg):
logging.debug('received response: %s', repr(msg))
# nothing to do
def handleFile(self, topic, msg):
try:
self.handleFile0(topic, msg)
except: # pylint: disable=W0702
logging.warning('%s', traceback.format_exc())
def handleFile0(self, topic, msg):
f = msg['file']
outputPath = os.path.join(self.outputDirectory, f['filename'])
_fmt, data = f['contents'].split(':', 1)
contents = base64.b64decode(data)
file(outputPath, 'w').write(contents)
logging.debug('wrote %s bytes to %s', len(contents), outputPath)
class FileReceiver(object):
def __init__(self, opts):
self.request = {
'timeout': opts.timeout,
'pollPeriod': opts.pollPeriod,
}
if opts.timestampSpacing:
self.request['timestampSpacing'] = opts.timestampSpacing
if opts.imageResize:
self.request['imageResize'] = parseImageResize(opts.imageResize)
if opts.imageCrop:
self.request['imageCrop'] = parseImageCrop(opts.imageCrop)
if opts.imageFormat:
self.request['imageFormat'] = opts.imageFormat
self.outputDirectory = opts.output
self.subtopic = opts.subtopic
self.noRequest = opts.noRequest
opts.moduleName = opts.moduleName.format(subtopic=opts.subtopic)
self.publisher = ZmqPublisher(**ZmqPublisher.getOptionValues(opts))
self.subscriber = ZmqSubscriber(**ZmqSubscriber.getOptionValues(opts))
self.requestPeriod = 0.5 * opts.timeout
def getTopic(self, msgType):
return 'geocamUtil.filePublisher.%s.%s' % (self.subtopic, msgType)
def start(self):
self.publisher.start()
self.subscriber.start()
self.subscriber.subscribeJson(self.getTopic('file'),
self.handleFile)
self.subscriber.subscribeRaw(self.getTopic('response'),
self.handleResponse)
if not self.noRequest:
self.sendRequest()
requestTimer = ioloop.PeriodicCallback(self.sendRequest,
self.requestPeriod * 1000)
requestTimer.start()
def sendRequest(self):
logging.debug('sendRequest')
self.publisher.sendJson(self.getTopic('request'),
self.request)
def handleResponse(self, topic, msg):
logging.debug('received response: %s', repr(msg))
# nothing to do
def handleFile(self, topic, msg):
try:
self.handleFile0(topic, msg)
except: # pylint: disable=W0702
logging.warning('%s', traceback.format_exc())
def handleFile0(self, topic, msg):
f = msg['file']
outputPath = os.path.join(self.outputDirectory, f['filename'])
_fmt, data = f['contents'].split(':', 1)
contents = base64.b64decode(data)
file(outputPath, 'w').write(contents)
logging.debug('wrote %s bytes to %s', len(contents), outputPath)
