def from_local_folder(cls, path, config_file): author, ok = QtGui.QInputDialog.getText(None, "Set author", "Please your name : ") if ok: author = author.replace(' ', '_') if author == "": author = 'anonymus' else: author = 'anonymus' QtGui.QMessageBox.warning(None, 'Audio system', 'Please use hearphones or a good audio system to analyze the signals.') obs_dict = pickle.load(config_file) obs = [Observation.from_dict(d) for k, d in obs_dict.items()] return cls(observations=obs, author=author, local=True)
def observations_to_db(db, measurement, location, mIDs, mics, idValues, micValues, signal_info, overwrite=False,**kwargs): """ :param db: :param measurement: :param location: :param mIDs: :param mics: :param idValues: :param micValues: :param signal_info: :param overwrite: :return: """ # config logger logging.basicConfig(format='%(asctime)s %(message)s', filename='obs_to_db.log', level=logging.DEBUG, datefmt='%Y-%m-%d %H:%M:%S') logging.captureWarnings(True) # remove if not in db logging.info('New load Action') logging.info('Number of measurement obs : {}'.format(len(mIDs) * len(mics))) if not overwrite: query = db[OBS_COLLECTION].find({'location': location, 'measurement': measurement, 'mID': {'$in': list(mIDs)}}) logging.info('Number of obs already loaded in db: {}'.format(query.count())) for d in query: try: mIDs.remove(d['mID']) except ValueError: pass logging.info('Remaining obs to load in db: {}'.format(len(mIDs) * len(mics))) # functions def get_Val(var, mID, mic=None): if var in micValues.keys(): return micValues[var].get('values', {}).get(mID, {}).get(mic) if var in idValues.keys(): return idValues[var].get('values', {}).get(mID) else: return None def read_signal_2(mID, signalKey): return read_signal(mID, signalKey, signal_info) print("Begin fill db") # statistics remainingObs = itertools.product(mIDs, mics) totObsToLoad = len(mIDs) * len(mics) loaded = 0 missingS = 0 missingData = 0 # fill db for n, (mID, mic) in tqdm(enumerate(remainingObs)): mic = str(mic) # mic and Id Values variables = [ ('LAeq', 'LAEQ'), ('LAmax', 'LAmax'), ('SEL', 'TEL'), ('trainType', 'trainType'), ('A-Spectrum', 'LAf'), ('mDate', 'mDate'), ('mTime', 'mTime'), ('v1', 'v1'), ('v2', 'v2'), ('t_evaluation_begin', 'Tb'), ('t_evaluation_end', 'Te'), ('t_passby_begin', 'Tp_b'), ('t_passby_end', 'Tp_e'), ('direction', 'direction'), ('track', 'Gleis')] warn = [] doc = {} for k, var in variables: value = get_Val(var, mID, mic) if value is None: warn.append(k) else: doc[k] = value # modify dict if 'A-Spectrum' in doc.keys(): doc['A-Spectrum'] = {'freq': micValues['LAf'].get('colName'), 'values': doc.pop('A-Spectrum')} if 'mDate' in doc.keys() and 'mTime' in doc.keys(): doc['mDateTime'] = excel_date_time(doc.pop('mDate'), doc.pop('mTime')) if 'direction' in doc.keys(): doc['direction'] = int(doc.pop('direction')) if 'v1' in doc.keys() and 'v2' in doc.keys(): doc['speed'] = {'values': [doc.pop('v1'), doc.pop('v2')], 'info': 'firstAxis,lastAxis'} if 'track' in doc.keys(): doc['track'] = int(doc.pop('track')) # meteo data meteoVar = [('humidity_rel', 'humidity'), ('temperature_C', 'Temp'), ('rain_volts', 'rain'), ('wind_ms', 'wind')] for k, var in meteoVar: value = get_Val(var, mID, mic) if value is None: warn.append(k) else: doc.setdefault('meteo',{})[k] = value # log print(n, ': ', mID, ', ', mic, ', Warnings: ', len(warn)) # read signal try: sR, pArray, t = read_signal_2(mID, 'mic{}'.format(mic)) _, prms, tprms = read_signal_2(mID, 'prms{}'.format(mic)) except FileNotFoundError: print('{}-signal-{}-not Found'.format(mID, str(mic))) logging.warning('{}-signal-{}-not Found'.format(mID, str(mic))) sR, t0, pArray = None, None, None missingS += 1 doc['valid'] = False else: doc['prms'] = {'values': prms.tolist(), 'time': tprms.tolist()} t0 = t[0] if len(warn) == 0: loaded += 1 doc['valid'] = True logging.info(mID + "-" + mic + "-ok, Remaining Obs: " + str(totObsToLoad - n)) else: logging.warning(mID + "-" + mic + "-MissingVariables-" + str(warn)) missingData += 1 doc['valid'] = False # add to database Observation.new_to_db(db, measurement, location, mID, int(mic), sR, t0, pArray, doc) logging.info('obs with missing Signals: {}'.format(missingS)) logging.info('obs with missing Data: {}'.format(missingData)) logging.info('obs loaded ok: {}'.format(loaded)) print("See log for details")