def add_qlb_file_record(source, path): """ Attempt to create a QLP file record. Adds to the current SQLAlchemy Session object, but does not commit (will rollback, however, if there is a problem) Returns (record, valid) tuple """ path_id = source.path_id(path) valid_file = True try: qlwell = get_well(path) ## catch custom qlbs that do not containt sw_versions or time stamps if ( qlwell.host_software is not None and len(qlwell.host_software) > 0): software_version = qlwell.host_software.split()[-1] else: software_version = 'Unknown' if (qlwell.host_datetime is not None ): runtime_stamp = datetime.strptime(qlwell.host_datetime, '%Y:%m:%d %H:%M:%S') else: runtime_stamp = '0000-00-00 00:00:01' file_metadata = dict(version=software_version, type='raw', run_id=run_id(path_id), read_status=0, mtime=datetime.fromtimestamp(os.stat(path).st_mtime), runtime=runtime_stamp, dirname=os.path.dirname(path_id), basename=os.path.basename(path_id)) qlbfile = QLBFile(**file_metadata) except Exception, e: # do not add file if the text file is busy # (exists, but does not contain valid data) if hasattr(e, 'errno'): if e.errno in (errno.ETXTBSY, errno.EBUSY): print e valid_file = False qlwell = None qlbfile = None return (qlbfile, qlwell, valid_file) print e file_metadata = {'run_id': run_id(path_id), 'dirname': os.path.dirname(path_id), 'basename': os.path.basename(path_id), 'read_status': -10, 'type': 'unknown', 'version': '', 'mtime': datetime.fromtimestamp(os.stat(path).st_mtime)} qlbfile = QLBFile(**file_metadata) valid_file = False qlwell = None
def fft(self, id=None): from qtools.lib.mplot import cleanup, render as plt_render # make it better? if not deps_loaded: abort(500) if not id: abort(404) well = Session.query(QLBWell).get(id) if not well: abort(404) file = well.file if not file: abort(404) storage = QLStorageSource(config) path = storage.qlbwell_path(well) wellobj = get_well(path) fam_samples = wellobj.samples[:,0][::FFT_DOWNSAMPLE].astype('float') vic_samples = wellobj.samples[:,1][::FFT_DOWNSAMPLE].astype('float') # normalize to 0->1 fam_samples /= np.max(fam_samples) vic_samples /= np.max(vic_samples) Sf = np.fft.fft(fam_samples) numpy.fft.fftpack._fft_cache = {} ff = np.fft.fftfreq(len(Sf), d=FFT_DOWNSAMPLE/wellobj.data_acquisition_params.sample_rate) # only consider positive half of frequency domain, normalize by num of samples Yf = abs(Sf[0:len(ff)/2])/len(fam_samples) # make X-axis in terms of Hz Wf = int((len(ff)/2)/max(ff)) # group by 1 Hz Sv = np.fft.fft(vic_samples) numpy.fft.fftpack._fft_cache = {} fv = np.fft.fftfreq(len(Sv), d=FFT_DOWNSAMPLE/wellobj.data_acquisition_params.sample_rate) Yv = abs(Sv[0:len(fv)/2])/len(vic_samples) Wv = int((len(fv)/2)/max(fv)) # group by 1 Hz # find peaks fam_peak_indices = fft.find_peak_indices(Yf, Wf*2) # group by 2Hz vic_peak_indices = fft.find_peak_indices(Yv, Wv*2) # group by 2Hz ftops = ff.take(fam_peak_indices)[0:10] vtops = fv.take(vic_peak_indices)[0:10] fig = plt.figure() fig.set_figwidth(8) fig.set_figheight(6) plt.subplot(211) plt.title('FAM Channel FFT (0-150Hz)') plt.plot(ff[0:len(ff)/2], Yf) plt.axis([-10, 150, -0.002, 0.04]) plt.text(70, 0.03, "Top Peaks (2 Hz windows):", weight='bold') for i, val in enumerate(ftops): plt.text(70 if i < 5 else 90, 0.027-(0.003*(i % 5)), "%.2f" % val, size=10) # todo render text func? plt.subplot(212) plt.title('VIC Channel FFT (0-150Hz)') plt.plot(fv[0:len(fv)/2], Yv) plt.axis([-10, 150, -0.002, 0.04]) plt.text(70, 0.03, "Top Peaks (2 Hz windows):", weight='bold') for i, val in enumerate(vtops): plt.text(70 if i < 5 else 90, 0.027-(0.003*(i % 5)), "%.2f" % val, size=10) # fft leaves cache-- not good for threaded server imgdata = self.__render(fig) cleanup(fig) del fig, wellobj, fam_samples, vic_samples, Sf, ff, Yf, Wf, Sv, fv, Yv, Wv, ftops, vtops gc.collect() return imgdata