def fetch_sensor(platformID: str, name: str, units: str = None, parser: str = None, tags: list = None, startime: int = None, endtime: int = None, truncate = True, db: str = DB, connection: MongoClient = CONN ) -> Sensor: """ Fetches a specified Sensor from MongoDB, truncated by startime and endtime Args: platformID (str): name of the target collection, typically a platformID name (str): Sensor name, eg 'm_depth' or 'salinity' units (str=None): sensor units, eg 'rad' or 'deg' or 'nodim' parser (str=None): parser-tag, eg mima or xbd or suflog tags (list=None): sensor's tags. #TODO, currently UNIMPLIMENTED startime (int=None): epoch query start time endtime (int=None): epoch query end time truncate (bool=True): True -> Sensor data range is exactly the requested range False -> Sensor data returned in 24hr chunks, as saved in database db (str=<default>): name of the target database connection (MongoClient=<default>): specifies host and port of target mongodb server Returns: a Sensor """ collection = connection[db][platformID] doc = {'name': name} if units: doc['units'] = units if parser: doc['parser'] = parser if startime: startday = MSensor.epoch2epochday(startime) else: startday = 0 startime = -inf if endtime: enday = endtime else: enday = time() #no future data endtime = inf doc['epochday'] = {'$gte':startday, '$lt':enday} cursor = collection.find(doc).sort('epochday',1) msensors = [] for doc in cursor: msensors.append(CLASSON.decode(doc)) # sensor.xy.extend(doc['xy']) sensor = MSensor.to_sensor(msensors) if truncate: sensor.xy = list(filter(lambda xy_tup: startime <= xy_tup[0] < endtime, sensor.xy)) return sensor
def store2mongo(collection: str, doc: CLASSON, db: str = DB, connection: MongoClient = CONN): """ Stores a doc into mongodb according to spec. eg insert, replace, subcollection Args: collection (str): name of the target collection, typically a platformID doc (list or CLASSON): an object that inherits from the CLASSON class, or list therof db (str): name of the target database connection (MongoClient): specifies host and port of target mongodb server Returns: a pymongo.results object (or list therof if doc was a list) or non if doc type unrecognized Side-Effects: stores data in mongodb """ if isinstance(doc,list): results = [] for elem in doc: print('.', end='') results.append( store2mongo(collection, elem, db, connection) ) print() return results if db not in connection.database_names(): warn('{} is a new database in {}'.format(db,connection)) db = connection[db] if collection not in db.collection_names(): warn('{} is a new collection in {}, {}'.format(collection, db, connection)) collection = db[collection] if isinstance(doc, Sensor): """ doc = doc.encode() input_xy = doc.pop('xy') try: #TODO make this more efficient old_xy = collection.find(doc,{'xy': 1, '_id': 0}).next()['xy'] updated_xy = [tuple(elem) for elem in old_xy + input_xy] updated_xy = sorted(set(updated_xy), key=lambda xy: xy[0]) updated_xy = filter(lambda xy_tup: xy_tup[0]>time()-DATA_AGE_LIMIT, updated_xy) except Exception as e: print('update_xy error',e) #TODO report/determine source of this error updated_xy = input_xy return collection.update(doc, {'$set': {'xy':updated_xy}}, upsert=True) """ r = [] # results mdocs = MSensor.from_sensor(doc) for mdoc in mdocs: mdoc = mdoc.encode() input_xy = mdoc.pop('xy') try: stored_xy = collection.find(mdoc,{'xy': 1, '_id': 0}).next()['xy'] updated_xy = [tuple(elem) for elem in stored_xy + input_xy] updated_xy = sorted(set(updated_xy), key=lambda xy: xy[0]) except StopIteration: # sensor doesn't yet exist in database updated_xy = input_xy except Exception as e: print('update_xy error', type(e), e) #yet unseen updated_xy = stored_xy r.append(collection.update(mdoc, {'$set': {'xy':updated_xy}}, upsert=True)) return r elif isinstance(doc,mdr_parser.MDR_File): if fnmatch(doc.filename, '??.mdr'): # get most recent uploaded mdr prev_mdr = collection.find({'_class':'gliderops.dataman.parsers.mdr_parser.MDR_File'}).sort({'_id':1}).limit(1)[0] # if the gaps don't match, it's new! insert it #if mdr_parser.diff_gaps(old_mdr['gaps'], doc['gaps']) != ([],[]): #TODO priority: low if list(map(tuple, doc.gaps )) != list(map(tuple, prev_mdr['gaps'] )): collection.insert(doc.encode()) else: pass # it's just an archived copy elif isinstance(doc,xbd_parser.XBD_File): ext = doc.filename.split('.')[-1] #file's extention criteria = {'the8x3_filename':doc.the8x3_filename, # match segment 'filename': {'$regex': '.*'+ext, # match filetype '$options':'i'}, # ignore case 'size':{'$lte':doc.size}} # only overwrite if db element's size is Less Than incoming doc's size collection.update(criteria, doc.encode(), upsert=True) elif isinstance(doc, SourceFile): return collection.update({'fullpath':doc.fullpath}, doc.encode(), upsert=True) else: #if all else fails, simply warn('{} unrecognized. was not added to in {}'.format(type(doc), collection))