def do_normalize(infile, fast_method, window_size, outfile, compress): # Set up output name name_field = os.path.splitext(infile) if name_field[1] == '.gz': name_field = os.path.splitext(name_field[0]) base_name = name_field[0] # read in data. lf = b.LofasmFile(infile) lf.read_data() # Normalize norm_data, normalize_array = c.normalize(lf.data, fast=fast_method, window=window_size) # Out put data. if outfile == "": outfile = base_name + '_normalized.bbx' if compress: outfile += '.gz' else: if not outfile.endswith('gz'): if compress: outfile += '.gz' outfile = b.LofasmFile(outfile, header=lf.header, mode='write') outfile.add_data(norm_data) outfile.write() outfile.close()
def get_data(self, minimum=True): """Return an array of lofasm data for however many files are loaded to calibrate class. Data from each file is appended to one data array after sorting regardless of contiguity. """ dsampled_power = [] datachunk = [] re = 'Reading data... ' for filename in range(len(self.filelist)): dat = bb.LofasmFile(self.filelist[filename]) dat.read_data() avg_10freq_bins = np.average( dat.data[self.freqbins[filename] - 5:self.freqbins[filename] + 5, :], axis=0) ##Avg 10 bins around frequency if minimum == True: lowest_datafile_power = avg_10freq_bins.min() dsampled_power = np.append(dsampled_power, lowest_datafile_power) else: avg_datafile_power = np.average(avg_10freq_bins) dsampled_power = np.append(dsampled_power, avg_datafile_power) p = (str(filename * 100 / len(self.filelist)) + '%') if filename + 1 not in range(len(self.filelist)): p = 'Done' sys.stdout.write("\r%s%s" % (re, p)) sys.stdout.flush() return dsampled_power
def clean(data): lf = b.LofasmFile(fname) lf.read_data() data = 10 * np.log10( lf.data[args.lower_frequency_bin:args.upper_frequnecy_bin]) norm_data, normalize_array = c.normalize(data, fast=args.fast) o_mask, outlier_average_top, outlier_average_bottom = c.outlier_mask( norm_data, threshold=args.o_thresh) nb_mask, percent_clean_freq_channels = c.narrow_band_mask( norm_data, threshold=args.nb_thresh) wb_mask, percent_clean_time = c.wide_band_mask(norm_data, threshold=args.wb_thresh) print fname + '---------------------------' print 'top outlier average : ' + str(outlier_average_top) print 'bottom outlier average : ' + str(outlier_average_bottom) print 'percentage of clean freq channels : ' + str( percent_clean_freq_channels) print 'percentage of clean time : ' + str(percent_clean_time) print '' print '-------------------------------------------------------' final = norm_data * wb_mask * o_mask * nb_mask final[np.isnan(final)] = 1 final = final * normalize_array lfc = b.LofasmFile('Clean_' + fname.rstrip('.gz'), mode='write') lfc.add_data(final) lfc.set('dim1_start', lf.header['dim1_start']) lfc.set('dim1_span', lf.header['dim1_span']) lfc.set('dim2_start', lower_frequency * 1e6) lfc.set('dim2_span', (upper_frequency - lower_frequency) * 1e6) lfc.set('clean', True) lfc.set('station', lf.header['station']) lfc.set('channel', lf.header['channel']) lfc.write() lfc.close() print 'done' '''
def averageFileSpectra(f): '''Calculate average spectra in file ''' lfx = bbx.LofasmFile(f) lfx.read_data() if lfx.iscplx: result = np.average(np.abs(lfx.data)**2, axis=1) else: result = np.average(lfx.data, axis=1) return result
def read_files_avg(self, files, freq, verbose=True): """Creates data attribute by averaging over a frequency bin range, then taking minimum value for each file. """ self.freqmhz = freq self.filelist = sorted(glob.glob(files)) for f in reversed( self.filelist): #Remove non lofasm files from filelist if not bb.is_lofasm_bbx(f): self.filelist.remove(f) re = 'Reading data...' for i in range(len(self.filelist)): f = bb.LofasmFile(self.filelist[i]) head = f.header startt = head['start_time'] timebins = head['metadata']['dim1_len'] timelength = float(head['dim1_span']) # Convert header start_time to datetime object time_obj = datetime.datetime.strptime(startt[:-8], '%Y-%m-%dT%H:%M:%S') self.times_array = np.append(self.times_array, time_obj) # Find the frequency bin corresponding to the given frequency [Mhz] bw = (float(head['dim2_span']) / 1000000.0) / head['metadata']['dim2_len'] freqbin = int( (self.freqmhz - (float(head['dim2_start']) / 1000000.0)) / bw) self.freq_bin = np.append(self.freq_bin, freqbin) f.read_data() freqbin_range = 10 freqbins = f.data[freqbin - freqbin_range / 2:freqbin + freqbin_range / 2, :] freqavg = np.average(freqbins, axis=0) # min_vals = [np.min(x) for x in np.rot90(freqbins, k=-1)] min_file_val = np.min(freqavg) f.close() # Compute datetime of the selected timebin index = list(freqavg).index(min_file_val) seconds_into_file = (float(index) / timebins) * timelength seconds_into_file = datetime.timedelta(seconds=seconds_into_file) time_from_bin = time_obj + seconds_into_file self.data = np.append(self.data, min_file_val) self.times_array = np.append(self.times_array, time_from_bin) if verbose == True: p = (str(i * 100 / len(self.filelist)) + '%') if i + 1 not in range(len(self.filelist)): p = 'Done \n' sys.stdout.write("\r%s%s" % (re, p)) sys.stdout.flush()
def _getMjdStartTimeFromFileHeader(self, f): lfx = bbx.LofasmFile(f) hdr = lfx.header if not lfx.header['metadata']['dim1_len']: raise ValueError("no data in this file") startt = hdr['start_time'] startt_repr = startt[:-8] if 'T' in startt else startt dfmt = self.startt_fmts[0] if 'T' in startt else self.startt_fmts[1] timeobj = datetime.strptime(startt_repr, dfmt) return Time(timeobj).mjd
def read_times(self, files): self.filelist = sorted(glob.glob(files)) for f in reversed(self.filelist): #Remove non lofasm files from filelist if not bb.is_lofasm_file(f): filelist.remove(f) for i in range(len(self.filelist)): f = bb.LofasmFile(self.filelist[i]) startt = f.header['start_time'] time_obj = datetime.datetime.strptime(startt[:-8],'%Y-%m-%dT%H:%M:%S') self.times_array = np.append(self.times_array, time_obj) f.close()
def freq_average_file(filename, freqs, bw=1.): ''' take a list of frequency bands of width bw centered at freq from each filterbank sample calculate the average over time. the minimum value of this averaged time series will be returned. Parameters ---------- filename : str the data file to process freq : list list of the selected center frequencies in MHz bw : float, optional the bandwidth to include in the frequency averaging Returns ------- freq_avg : numpy.array array of frequency averaged time series minimum values, each corresponding to its respective center frequency at input ''' lfx = bbx.LofasmFile(filename) lfx.read_data() freq_avg_list = [] if isinstance(freqs, list): freqs.sort() for i in range(len(freqs)): lfbin = freq2bin(freqs[i] - bw / 2.) hfbin = freq2bin(freqs[i] + bw / 2.) num_fbins = hfbin - lfbin num_tbins = np.shape(lfx.data)[1] avg_ts = np.average(lfx.data[lfbin:hfbin, :], axis=0) freq_avg_list.append(deepcopy(avg_ts)) rows = len(freq_avg_list) cols = max([len(x) for x in freq_avg]) data = np.zeros((rows, cols), dtype=np.float64) for i in range(rows): data[i, :] = freq_avg_list[i] else: lfbin = freq2bin(freqs - bw / 2.) hfbin = freq2bin(freqs + bw / 2.) avg_ts = np.average(lfx.data[lfbin:hfbin, :], axis=0) data = avg_ts lfx.close() return data
def __init__(self, files, station, freq=20.0, chan='CC'): filelist = glob.glob(files) self.filelist = sorted(filelist) self.station = station self.chan = chan self.freqmhz = freq self.res = len(filelist) time_array = [] station_array = [] freqbin_array = [] for i in range(len(self.filelist)): head = bb.LofasmFile(self.filelist[i]).header file_startt = head['start_time'] file_station = head['station'] file_pol = head['channel'] start_time = datetime.datetime.strptime(file_startt[:-8], '%Y-%m-%dT%H:%M:%S') bw = (float(head['dim2_span'])/1000000.0)/head['metadata']['dim2_len'] freqbin = int((self.freqmhz-(float(head['dim2_start'])/1000000.0))/bw) time_array.append(start_time) station_array.append(file_station) freqbin_array.append(freqbin) self.cali_array = [time_array, station_array] self.freqbins = freqbin_array for i in reversed(range(len(self.filelist))): if self.cali_array[1][i] != str(self.station): del self.cali_array[0][i] del self.cali_array[1][i] lfdic = {1:{'name':'LI', 'lat':[26,33,19.676], 'long':[97,26,31.174], 't_offset':6.496132851851852}, 2:{'name':'LII', 'lat':[34,4,43.497], 'long':[107,37,5.819], 't_offset':7.174552203703703}, 3:{'name':'LIII', 'lat':[38,25,59.0], 'long':[79,50,23.0], 't_offset':5.322648148148148}, 4:{'name':'LIV', 'lat':[34,12,3.0], 'long':[118,10,18.0], 't_offset':7.87811111111111}} self.lfs = lfdic[station]
def add_files(self, files): """Add files to the calibrate class. New files are sorted, duplicates are ignored, header data is read, and the file lists are appended together. Parameters ---------- files : str A path to lofasm `.bbx.gz` files. `*` wildcard can be used for multiple files. """ new_filelist = glob.glob(files) for i in new_filelist: if i in self.filelist: new_filelist.remove(i) self.filelist = (self.filelist + new_filelist) self.filelist = sorted(self.filelist) time_array = [] station_array = [] for i in range(len(self.filelist)): head = bb.LofasmFile(self.filelist[i]).header file_startt = head['start_time'] file_station = head['station'] start_time = datetime.datetime.strptime(file_startt[:-8], '%Y-%m-%dT%H:%M:%S') time_array.append(start_time) station_array.append(file_station) self.cali_array = [time_array, station_array] for i in reversed(range(len(self.filelist))): if self.cali_array[1][i] != str(self.station): del self.cali_array[0][i] del self.cali_array[1][i] del self.filelist[i]
def convert_bbx_file(lofasm_file): from lofasm.bbx import bbx lf = bbx.LofasmFile(lofasm_file) lf.read_data() csvname = get_csv_filename(lofasm_file) if not os.path.isdir(os.path.dirname(csvname)): try: os.makedirs(os.path.dirname(csvname)) except: print "unable to create directory : {}".format( os.path.dirname(csvname)) print "please create the directory manually and try again" sys.stdout.flush() wstart = time() with open(csvname, 'wb') as csvfile: for i in range(np.shape(lf.data)[1]): data = lf.data[:1024, i] writeRow(csvfile, data) wend = time() print "wrote {} in {} s".format(csvname, str(wend - wstart)) sys.stdout.flush()
if filename.endswith(".lofasm.gz"): os.system("loco2bx.py -p CC,DD,CD " + filename) print(os.path.join(filename) +" converted to bbx ("+str(j)+"/"+str(n)+")") j+=1 ''' #go into CC channel and take an average of the values in the array os.chdir(cwd1 + "/bbx/CC") cwd2 = os.getcwd() i = 0 lst = os.listdir(cwd2) lst.sort() for filename2 in lst: if filename2.endswith(".bbx.gz"): lf = bbx.LofasmFile(os.path.join(filename2)) lf.read_data() countavgCC[i] = np.average(lf.data) print(filename2 + " average added to array") print("\naverage = " + str(countavgCC[i])) spect_avg = np.average(lf.data, axis=0) countfreqCC[i] = spect_avg[pdat.freq2bin(50)] print(filename2 + " freq added to array") print("average freq = " + str(countfreqCC[i])) countbinCC[i] = np.average(lf.data, axis=0) print(filename2 + " avg bin added to array") i += 1
def read_files(self, files, freq, verbose=True): """Creates data attribute using the minimum value in frequency and time for each file. """ self.freqmhz = freq self.filelist = sorted(glob.glob(files)) #Remove non lofasm files from filelist for f in reversed(self.filelist): if not bb.is_lofasm_bbx(f): self.filelist.remove(f) re = 'Reading data...' for i in range(len(self.filelist)): f = bb.LofasmFile(self.filelist[i]) head = f.header startt = head['start_time'] timebins = head['metadata']['dim1_len'] timelength = float(head['dim1_span']) # Convert header start_time to datetime object # ================TODO===================================== # Due to a bug some LoFASM data has messed up start times. # What follows is a hotfix so that this library can support # both time formats. In the future, the bug should be fixed # and this library should only have to support 1 time format. # # The following loop tries each format specified in fmts. # If the time is parsed correctly (and does not throw an # exception) then the loop will be broken. # ========================================================== fmts = ["%Y-%m-%dT%H:%M:%S", # correct time format for bbx files "%Y%m%d_%H%M%S"] # additional format found in LoFASM I files startt_repr = startt[:-8] if 'T' in startt else startt for fmt in fmts: try: time_obj = datetime.datetime.strptime(startt_repr, fmt) break except ValueError: pass else: print "Cannot parse start time header field {}".format(startt_repr) # Find the frequency bin corresponding to the given frequency bw = (float(head['dim2_span'])/1000000.0)/head['metadata']['dim2_len'] freqbin = int((self.freqmhz-(float(head['dim2_start'])/1000000.0))/bw) self.freq_bin = np.append(self.freq_bin, freqbin) f.read_data() freqbin_range = 10 #bins around specified 'freq' to read freqbins = f.data[freqbin-freqbin_range/2:freqbin+freqbin_range/2,:] min_vals_per_timebin = [np.min(x) for x in np.rot90(freqbins, k=-1)] min_file_val = np.min(min_vals_per_timebin) f.close() # Compute datetime of the selected timebin index = min_vals_per_timebin.index(min_file_val) seconds_into_file = (float(index)/timebins)*timelength seconds_into_file = datetime.timedelta(seconds=seconds_into_file) time_from_bin = time_obj + seconds_into_file self.data = np.append(self.data, min_file_val) self.times_array = np.append(self.times_array, time_from_bin) if verbose == True: p = (str(i*100/len(self.filelist)) + '%') if i+1 not in range(len(self.filelist)): p = 'Done' sys.stdout.write("\r%s%s" % (re,p)) sys.stdout.flush()
def avgpow(lf, freq): x = lf.data[freq2bin(freq), :] return (np.mean(x)) if __name__ == '__main__': import argparse p = argparse.ArgumentParser() p.add_argument('fnames', nargs='+') p.add_argument('freq', type=float) args = p.parse_args() x = [] for f in args.fnames: lf = bbx.LofasmFile(f) lf.read_data() pows = lf.data[freq2bin(args.freq), :] avg = np.mean(pows) start = lf.header['start_time'] start_cv = Time(start, scale='utc') st = int(lf.header['station']) lfstation = LoFASM_Stations[st] lon = lfstation.lon * 180 / np.pi start_sr = start_cv.sidereal_time('mean', lon) start_hr = start_sr.hour powmodel = galaxyPower.calculatepower(start_hr, st, args.freq, 0) x.append((start_hr, avg, powmodel)) t, avg, gal = zip(*x)
def numpyf(n, filei): countavgCC = np.zeros(n) countfreqCC = np.zeros(n) countbinCC = np.zeros((n, 1024)) countavgDD = np.zeros(n) countfreqDD = np.zeros(n) countbinDD = np.zeros((n, 1024)) countavgCD = np.zeros(n) countfreqCD = np.zeros(n) countbinCD = np.zeros((n, 1024), dtype=complex) cwd1 = os.getcwd() #go into CC channel and take an average of the values in the array os.chdir(cwd1 + "/bbx/CC") cwd2 = os.getcwd() i = 0 lst = os.listdir(cwd2) lst.sort() for filename2 in lst: if filename2.endswith(".bbx.gz"): lf = bbx.LofasmFile(os.path.join(filename2)) lf.read_data() countavgCC[i] = np.average(lf.data) print(filename2 + " average added to array") print("\naverage = " + str(countavgCC[i])) countbinCC[i] = np.average(lf.data, axis=0) print(filename2 + " avg bin added to array") i += 1 lf.close() #go into DD channel and take an average of the values in the array os.chdir(cwd1) os.chdir(cwd1 + "/bbx/DD") cwd3 = os.getcwd() i = 0 lst = os.listdir(cwd3) lst.sort() for filename3 in lst: if filename3.endswith(".bbx.gz"): df = bbx.LofasmFile(os.path.join(filename3)) df.read_data() countavgDD[i] = np.average(df.data) print(filename3 + " average added to array") print("\naverage = " + str(countavgDD[i])) countbinDD[i] = np.average(df.data, axis=0) print(filename3 + " avg bin added to array") i += 1 df.close() #go into CD channel and take an average of the values in the array os.chdir(cwd1) os.chdir(cwd1 + "/bbx/CD") cwd4 = os.getcwd() i = 0 lst = os.listdir(cwd4) lst.sort() for filename4 in lst: if filename4.endswith(".bbx.gz"): df = bbx.LofasmFile(os.path.join(filename4)) df.read_data() countavgCD[i] = np.average(df.data) print(filename4 + " average added to array") print("\naverage = " + str(countavgCD[i])) countbinCD[i] = np.average(df.data, axis=0) print(filename4 + " avg bin added to array") i += 1 df.close() #save/plot the output os.chdir(cwd1) filesv = filei[:-1] np.save(str(filesv) + 'outputavgCC', countavgCC) np.save(str(filesv) + 'outputbinCC', countbinCC) np.save(str(filesv) + 'outputavgDD', countavgDD) np.save(str(filesv) + 'outputbinDD', countbinDD) np.save(str(filesv) + 'outputavgCD', countavgCD) np.save(str(filesv) + 'outputbinCD', countbinCD)
lfx._hdr_fname) else "{} does not exist".format( basename(lfx._hdr_fname)) msg("size of {} on disk (Bytes)".format(basename(lfx._data_fname))) print os.path.getsize(lfx._data_fname) if exists( lfx._data_fname) else "{} does not exist".format( basename(lfx._data_fname)) msg("size of {} on disk (Bytes)".format(basename(lfx.fpath))) print os.path.getsize(lfx.fpath) if exists( lfx.fpath) else "{} does not exist".format(lfx.fpath) fname = 'testfile.bbx' # purge file if it already exists if os.path.exists(fname): os.remove(fname) lfx = bbx.LofasmFile('testfile.bbx', mode='write', gz=True) # set required header fields with fake information lfx.set('station', 'fake') lfx.set('channel', 'fake') lfx.set('dim1_start', '0') lfx.set('dim1_span', '1') lfx.set('dim2_start', '0') lfx.set('dim2_span', '1') lfx.set('data_label', 'filterbank') report(lfx) msg("Appending row of ones to internal buffer:") append_data(lfx, ones_row) report(lfx) msg("Dumping internal buffer to disk") lfx.write() report(lfx)
#! /usr/bin/env python if __name__ == "__main__": import argparse import matplotlib matplotlib.use('agg') # on headless linux import matplotlib.pyplot as plt from lofasm.bbx import bbx import numpy as np p = argparse.ArgumentParser() p.add_argument('filename', type=str, help="path to BBX file") args = p.parse_args() lfx = bbx.LofasmFile(args.filename) lfx.read_data() startt = lfx.header['start_time'] avg_spectra = np.average(lfx.data, axis=1) plt.figure() plt.title('L1 Average Spectra {}'.format(startt)) plt.plot(10 * np.log10(avg_spectra), label='Avg. Spectra') plt.legend() plt.savefig('{}.avg.png'.format(lfx.fname), format='png')
lfx.fpath) else "{} does not exist".format(lfx.fpath) # Set the parser stuff parser = argparse.ArgumentParser() parser.add_argument("-p", "--path", help="Path to input bbx file.") parser.add_argument("-f", "--freq", help="Set frequancy channel to be removed. In MHz.") parser.add_argument("-o", "--outfile", help="Name for file created.") ''' Stream Data. Open the file we want to mod. Prep the file we are to write too. ''' pars_dict = vars(parser.parse_args()) bbxFileHandel = bbx.LofasmFile(pars_dict['path']) bbxfile = pars_dict['path'] print "File to be read open. " bbxFileHeader = bbxFileHandel.header outPath = os.getcwd() outPath = outPath + '/' + pars_dict['outfile'] outFile = bbx.LofasmFile(outPath, bbxFileHeader, mode='write') print "File to write created. " for i in range(int(bbxFileHeader['metadata']['dim1_len'])): bbxFileHandel.read_data(1) # Read one line of data spectra = bbxFileHandel.data[:, :] #.astype(np.float32) spectra[:, pdat.freq2bin(float(pars_dict['freq']))] = 0 # report(outFile) append_data(outFile, spectra)
if __name__ == '__main__': import argparse p = argparse.ArgumentParser() p.add_argument('fnames', nargs='+') #reading the given arguments into p.add_argument('freq', type=float) #into variables p.add_argument('hor', type=int) args = p.parse_args() args.fnames.sort() #sort the files x = [] start_hr_arr = np.zeros(len(args.fnames)) fil = args.fnames[0] lf = bbx.LofasmFile(fil) st = int(lf.header['station']) # lsts, pows = galaxyPower.loadfile(st, args.freq, args.hor) for ii, f in enumerate(args.fnames): #for loop to read through the data lf = bbx.LofasmFile(f) #and append to an empty list lf.read_data() #the time in lst, the average power lofasm_pows = lf.data[:, freq2bin(args.freq)] #and the given galaxy power avg = np.mean(lofasm_pows) #at that time #LOFASM 3 [ # start = float(lf.header['start_mjd']) # start_postmjd = Time(start, format = 'mjd') # st = int(lf.header['station'])
help='wide band mask threshold.') # Parse the commonline parameters args = parser.parse_args() start_time = time.time() infile = args.infilename outfile = args.outfilename compress = args.compress med_window_size = int(args.median_window_size) min_window_size = int(args.running_minimum_window_size) norm = args.normalize outlier_thrhd = float(args.o_thresh) narrowband_thrhd = float(args.nb_thresh) wideband_thrhd = float(args.wb_thresh) lf = b.LofasmFile(infile) lf.read_data() norm_data, normalize_array = n.robust_normalize( lf.data, median_window=med_window_size, running_min_window=min_window_size) #medianed_data = n.running_median(lf.data, median_window=med_window_size) #norm_data, normalize_array = c.normalize(medianed_data, fast=True, window=med_window_size) o_mask, outlier_average_top, outlier_average_bottom = c.outlier_mask( norm_data, threshold=outlier_thrhd) nb_mask, percent_clean_freq_channels = c.narrow_band_mask( norm_data, threshold=narrowband_thrhd) wb_mask, percent_clean_time = c.wide_band_mask(norm_data, threshold=wideband_thrhd) print infile + '---------------------------'
if __name__ == "__main__": import argparse p = argparse.ArgumentParser() p.add_argument('datasetdir', type=str, help="path to dataset directory") p.add_argument( '--savedata', action='store_true', help='Save a copy of the resulting data array as a numpy file') args = p.parse_args() flist = glob(os.path.join(args.datasetdir, "*.bbx.gz")) flist.sort() lfx = bbx.LofasmFile(flist[0]) pol = lfx.header['channel'] lfx.close() data = np.zeros((1024, len(flist)), dtype=np.float64) for i in range(len(flist)): re = "Processing {}/{}".format(i + 1, len(flist)) sys.stdout.write("\r" + re) sys.stdout.flush() avgSpectra = averageFileSpectra(flist[i]) data[:, i] = avgSpectra if args.savedata: with open('waterfall_out_data_freqvtime_{}.numpy'.format(pol), 'w') as f: data.tofile(f)
parser.add_argument( '-fn', action='store', dest='file_name', default='*.bbx.gz', help='do you want to clean a specific file or type of file? try ') args = parser.parse_args() lower_frequency = args.lower_frequency_bin * (100.0 / 1024.0) upper_frequency = args.upper_frequnecy_bin * (100.0 / 1024.0) for fname in glob.glob(args.file_name): lf = b.LofasmFile(fname) lf.read_data() data = 10 * np.log10( lf.data[args.lower_frequency_bin:args.upper_frequnecy_bin]) norm_data, normalize_array = c.normalize(data, fast=args.fast) o_mask, outlier_average_top, outlier_average_bottom = c.outlier_mask( norm_data, threshold=args.o_thresh) nb_mask, percent_clean_freq_channels = c.narrow_band_mask( norm_data, threshold=args.nb_thresh) wb_mask, percent_clean_time = c.wide_band_mask(norm_data, threshold=args.wb_thresh) print fname + '---------------------------' print 'top outlier average : ' + str(outlier_average_top) print 'bottom outlier average : ' + str(outlier_average_bottom)
import os import numpy as np from lofasm.bbx import bbx cwd = os.getcwd() for filei in os.listdir(cwd): if filei.endswith(".bbx.gz"): lf = bbx.LofasmFile(os.path.join(filei)) lf.read_data() np.save(filei, lf.data) lf.close()
def __init__( self, files, output, station, freq=20.0, ): cal = calibrate(files, station, freq=freq) filelist = sorted(glob.glob(files)) dat1 = bb.LofasmFile(filelist[0]) dat1.read_data() avgfull = np.average(dat1.data[cal.freqbins[0] - 5:cal.freqbins[0] + 5, :], axis=0) #First element of list of data arrays list_of_powers = [avgfull] #List of data arrays (2d) dat = np.average( list_of_powers[0]) #First element of dat array for calibration re = 'Reading data... ' for filei in range(len(filelist) - 1): filei += 1 bbfile = bb.LofasmFile(filelist[filei]) bbfile.read_data() ### Preparing array containing full power of each file avgfull_power = np.average( bbfile.data[cal.freqbins[filei] - 5:cal.freqbins[filei] + 5, :], axis=0) ##Avg 10 bins around frequency list_of_powers.append(avgfull_power) ### Preparing data array for calibration: Each datapoint is avg power of file avg_datafile_power = np.average(avgfull_power) dat = np.append(dat, avg_datafile_power) p = ((str(filei * 100 / len(filelist)) + '%') + ' - [' + str(filei + 1) + ' out of ' + str(len(filelist)) + ' files]') if filei not in range(len(filelist) - 1): p = 'Done \n' sys.stdout.write("\r%s%s" % (re, p)) sys.stdout.flush() # print "Generating models... " done by calibration_pmts f'n calibration_pmts = cal.calibration_parameters(data=dat) sys.stdout.write('\rCalibrating and writing files... ') if output[-1] != '/': output += '/' for i in range(len(filelist)): filename = os.path.basename(filelist[i]) calname = (output + 'Calibrated_' + filename) calibrated = (list_of_powers[i] - calibration_pmts[1]) / calibration_pmts[0] lfc = bb.LofasmFile(output + 'Calibrated_' + filename, mode='write') lfc.add_data(calibrated) lfc.write() lfc.close() sys.stdout.write('\rDone - ' + str(len(filelist)) + ' calibrated files written.') sys.stdout.flush()