def fetch_sensor(platformID: str, name: str,
units: str = None, parser: str = None, tags: list = None,
startime: int = None, endtime: int = None, truncate = True,
db: str = DB, connection: MongoClient = CONN ) -> Sensor:
"""
Fetches a specified Sensor from MongoDB, truncated by startime and endtime
Args:
platformID (str): name of the target collection, typically a platformID
name (str): Sensor name, eg 'm_depth' or 'salinity'
units (str=None): sensor units, eg 'rad' or 'deg' or 'nodim'
parser (str=None): parser-tag, eg mima or xbd or suflog
tags (list=None): sensor's tags. #TODO, currently UNIMPLIMENTED
startime (int=None): epoch query start time
endtime (int=None): epoch query end time
truncate (bool=True): True -> Sensor data range is exactly the requested range
False -> Sensor data returned in 24hr chunks, as saved in database
db (str=<default>): name of the target database
connection (MongoClient=<default>): specifies host and port of target mongodb server
Returns:
a Sensor
"""
collection = connection[db][platformID]
doc = {'name': name}
if units:
doc['units'] = units
if parser:
doc['parser'] = parser
if startime:
startday = MSensor.epoch2epochday(startime)
else:
startday = 0
startime = -inf
if endtime:
enday = endtime
else:
enday = time() #no future data
endtime = inf
doc['epochday'] = {'$gte':startday, '$lt':enday}
cursor = collection.find(doc).sort('epochday',1)
msensors = []
for doc in cursor:
msensors.append(CLASSON.decode(doc))
# sensor.xy.extend(doc['xy'])
sensor = MSensor.to_sensor(msensors)
if truncate:
sensor.xy = list(filter(lambda xy_tup: startime <= xy_tup[0] < endtime, sensor.xy))
return sensor
def store2mongo(collection: str, doc: CLASSON,
db: str = DB, connection: MongoClient = CONN):
"""
Stores a doc into mongodb according to spec. eg insert, replace, subcollection
Args:
collection (str): name of the target collection, typically a platformID
doc (list or CLASSON): an object that inherits from the CLASSON class, or list therof
db (str): name of the target database
connection (MongoClient): specifies host and port of target mongodb server
Returns:
a pymongo.results object (or list therof if doc was a list)
or non if doc type unrecognized
Side-Effects:
stores data in mongodb
"""
if isinstance(doc,list):
results = []
for elem in doc:
print('.', end='')
results.append( store2mongo(collection, elem, db, connection) )
print()
return results
if db not in connection.database_names():
warn('{} is a new database in {}'.format(db,connection))
db = connection[db]
if collection not in db.collection_names():
warn('{} is a new collection in {}, {}'.format(collection, db, connection))
collection = db[collection]
if isinstance(doc, Sensor):
"""
doc = doc.encode()
input_xy = doc.pop('xy')
try:
#TODO make this more efficient
old_xy = collection.find(doc,{'xy': 1, '_id': 0}).next()['xy']
updated_xy = [tuple(elem) for elem in old_xy + input_xy]
updated_xy = sorted(set(updated_xy), key=lambda xy: xy[0])
updated_xy = filter(lambda xy_tup: xy_tup[0]>time()-DATA_AGE_LIMIT, updated_xy)
except Exception as e:
print('update_xy error',e) #TODO report/determine source of this error
updated_xy = input_xy
return collection.update(doc, {'$set': {'xy':updated_xy}}, upsert=True)
"""
r = [] # results
mdocs = MSensor.from_sensor(doc)
for mdoc in mdocs:
mdoc = mdoc.encode()
input_xy = mdoc.pop('xy')
try:
stored_xy = collection.find(mdoc,{'xy': 1, '_id': 0}).next()['xy']
updated_xy = [tuple(elem) for elem in stored_xy + input_xy]
updated_xy = sorted(set(updated_xy), key=lambda xy: xy[0])
except StopIteration:
# sensor doesn't yet exist in database
updated_xy = input_xy
except Exception as e:
print('update_xy error', type(e), e) #yet unseen
updated_xy = stored_xy
r.append(collection.update(mdoc, {'$set': {'xy':updated_xy}}, upsert=True))
return r
elif isinstance(doc,mdr_parser.MDR_File):
if fnmatch(doc.filename, '??.mdr'):
# get most recent uploaded mdr
prev_mdr = collection.find({'_class':'gliderops.dataman.parsers.mdr_parser.MDR_File'}).sort({'_id':1}).limit(1)[0]
# if the gaps don't match, it's new! insert it
#if mdr_parser.diff_gaps(old_mdr['gaps'], doc['gaps']) != ([],[]): #TODO priority: low
if list(map(tuple, doc.gaps )) != list(map(tuple, prev_mdr['gaps'] )):
collection.insert(doc.encode())
else:
pass # it's just an archived copy
elif isinstance(doc,xbd_parser.XBD_File):
ext = doc.filename.split('.')[-1] #file's extention
criteria = {'the8x3_filename':doc.the8x3_filename, # match segment
'filename': {'$regex': '.*'+ext, # match filetype
'$options':'i'}, # ignore case
'size':{'$lte':doc.size}} # only overwrite if db element's size is Less Than incoming doc's size
collection.update(criteria, doc.encode(), upsert=True)
elif isinstance(doc, SourceFile):
return collection.update({'fullpath':doc.fullpath}, doc.encode(), upsert=True)
else:
#if all else fails, simply
warn('{} unrecognized. was not added to in {}'.format(type(doc), collection))
