def do_normalize(infile, fast_method, window_size, outfile, compress):
# Set up output name
name_field = os.path.splitext(infile)
if name_field[1] == '.gz':
name_field = os.path.splitext(name_field[0])
base_name = name_field[0]
# read in data.
lf = b.LofasmFile(infile)
lf.read_data()
# Normalize
norm_data, normalize_array = c.normalize(lf.data,
fast=fast_method,
window=window_size)
# Out put data.
if outfile == "":
outfile = base_name + '_normalized.bbx'
if compress:
outfile += '.gz'
else:
if not outfile.endswith('gz'):
if compress:
outfile += '.gz'
outfile = b.LofasmFile(outfile, header=lf.header, mode='write')
outfile.add_data(norm_data)
outfile.write()
outfile.close()
def get_data(self, minimum=True):
"""Return an array of lofasm data for however many files are loaded to
calibrate class.
Data from each file is appended to one data array after sorting
regardless of contiguity.
"""
dsampled_power = []
datachunk = []
re = 'Reading data... '
for filename in range(len(self.filelist)):
dat = bb.LofasmFile(self.filelist[filename])
dat.read_data()
avg_10freq_bins = np.average(
dat.data[self.freqbins[filename] - 5:self.freqbins[filename] +
5, :],
axis=0) ##Avg 10 bins around frequency
if minimum == True:
lowest_datafile_power = avg_10freq_bins.min()
dsampled_power = np.append(dsampled_power,
lowest_datafile_power)
else:
avg_datafile_power = np.average(avg_10freq_bins)
dsampled_power = np.append(dsampled_power, avg_datafile_power)
p = (str(filename * 100 / len(self.filelist)) + '%')
if filename + 1 not in range(len(self.filelist)):
p = 'Done'
sys.stdout.write("\r%s%s" % (re, p))
sys.stdout.flush()
return dsampled_power
def clean(data):
lf = b.LofasmFile(fname)
lf.read_data()
data = 10 * np.log10(
lf.data[args.lower_frequency_bin:args.upper_frequnecy_bin])
norm_data, normalize_array = c.normalize(data, fast=args.fast)
o_mask, outlier_average_top, outlier_average_bottom = c.outlier_mask(
norm_data, threshold=args.o_thresh)
nb_mask, percent_clean_freq_channels = c.narrow_band_mask(
norm_data, threshold=args.nb_thresh)
wb_mask, percent_clean_time = c.wide_band_mask(norm_data,
threshold=args.wb_thresh)
print fname + '---------------------------'
print 'top outlier average : ' + str(outlier_average_top)
print 'bottom outlier average : ' + str(outlier_average_bottom)
print 'percentage of clean freq channels : ' + str(
percent_clean_freq_channels)
print 'percentage of clean time : ' + str(percent_clean_time)
print ''
print '-------------------------------------------------------'
final = norm_data * wb_mask * o_mask * nb_mask
final[np.isnan(final)] = 1
final = final * normalize_array
lfc = b.LofasmFile('Clean_' + fname.rstrip('.gz'), mode='write')
lfc.add_data(final)
lfc.set('dim1_start', lf.header['dim1_start'])
lfc.set('dim1_span', lf.header['dim1_span'])
lfc.set('dim2_start', lower_frequency * 1e6)
lfc.set('dim2_span', (upper_frequency - lower_frequency) * 1e6)
lfc.set('clean', True)
lfc.set('station', lf.header['station'])
lfc.set('channel', lf.header['channel'])
lfc.write()
lfc.close()
print 'done'
'''
def averageFileSpectra(f):
'''Calculate average spectra in file
'''
lfx = bbx.LofasmFile(f)
lfx.read_data()
if lfx.iscplx:
result = np.average(np.abs(lfx.data)**2, axis=1)
else:
result = np.average(lfx.data, axis=1)
return result
def read_files_avg(self, files, freq, verbose=True):
"""Creates data attribute by averaging over a frequency bin range,
then taking minimum value for each file.
"""
self.freqmhz = freq
self.filelist = sorted(glob.glob(files))
for f in reversed(
self.filelist): #Remove non lofasm files from filelist
if not bb.is_lofasm_bbx(f):
self.filelist.remove(f)
re = 'Reading data...'
for i in range(len(self.filelist)):
f = bb.LofasmFile(self.filelist[i])
head = f.header
startt = head['start_time']
timebins = head['metadata']['dim1_len']
timelength = float(head['dim1_span'])
# Convert header start_time to datetime object
time_obj = datetime.datetime.strptime(startt[:-8],
'%Y-%m-%dT%H:%M:%S')
self.times_array = np.append(self.times_array, time_obj)
# Find the frequency bin corresponding to the given frequency [Mhz]
bw = (float(head['dim2_span']) /
1000000.0) / head['metadata']['dim2_len']
freqbin = int(
(self.freqmhz - (float(head['dim2_start']) / 1000000.0)) / bw)
self.freq_bin = np.append(self.freq_bin, freqbin)
f.read_data()
freqbin_range = 10
freqbins = f.data[freqbin - freqbin_range / 2:freqbin +
freqbin_range / 2, :]
freqavg = np.average(freqbins, axis=0)
# min_vals = [np.min(x) for x in np.rot90(freqbins, k=-1)]
min_file_val = np.min(freqavg)
f.close()
# Compute datetime of the selected timebin
index = list(freqavg).index(min_file_val)
seconds_into_file = (float(index) / timebins) * timelength
seconds_into_file = datetime.timedelta(seconds=seconds_into_file)
time_from_bin = time_obj + seconds_into_file
self.data = np.append(self.data, min_file_val)
self.times_array = np.append(self.times_array, time_from_bin)
if verbose == True:
p = (str(i * 100 / len(self.filelist)) + '%')
if i + 1 not in range(len(self.filelist)):
p = 'Done \n'
sys.stdout.write("\r%s%s" % (re, p))
sys.stdout.flush()
def _getMjdStartTimeFromFileHeader(self, f):
lfx = bbx.LofasmFile(f)
hdr = lfx.header
if not lfx.header['metadata']['dim1_len']:
raise ValueError("no data in this file")
startt = hdr['start_time']
startt_repr = startt[:-8] if 'T' in startt else startt
dfmt = self.startt_fmts[0] if 'T' in startt else self.startt_fmts[1]
timeobj = datetime.strptime(startt_repr, dfmt)
return Time(timeobj).mjd
def read_times(self, files):
self.filelist = sorted(glob.glob(files))
for f in reversed(self.filelist): #Remove non lofasm files from filelist
if not bb.is_lofasm_file(f):
filelist.remove(f)
for i in range(len(self.filelist)):
f = bb.LofasmFile(self.filelist[i])
startt = f.header['start_time']
time_obj = datetime.datetime.strptime(startt[:-8],'%Y-%m-%dT%H:%M:%S')
self.times_array = np.append(self.times_array, time_obj)
f.close()
def freq_average_file(filename, freqs, bw=1.):
'''
take a list of frequency bands of width bw centered at freq from
each filterbank sample calculate the average over time.
the minimum value of this averaged time series will be returned.
Parameters
----------
filename : str
the data file to process
freq : list
list of the selected center frequencies in MHz
bw : float, optional
the bandwidth to include in the frequency averaging
Returns
-------
freq_avg : numpy.array
array of frequency averaged time series minimum values,
each corresponding to its respective center
frequency at input
'''
lfx = bbx.LofasmFile(filename)
lfx.read_data()
freq_avg_list = []
if isinstance(freqs, list):
freqs.sort()
for i in range(len(freqs)):
lfbin = freq2bin(freqs[i] - bw / 2.)
hfbin = freq2bin(freqs[i] + bw / 2.)
num_fbins = hfbin - lfbin
num_tbins = np.shape(lfx.data)[1]
avg_ts = np.average(lfx.data[lfbin:hfbin, :], axis=0)
freq_avg_list.append(deepcopy(avg_ts))
rows = len(freq_avg_list)
cols = max([len(x) for x in freq_avg])
data = np.zeros((rows, cols), dtype=np.float64)
for i in range(rows):
data[i, :] = freq_avg_list[i]
else:
lfbin = freq2bin(freqs - bw / 2.)
hfbin = freq2bin(freqs + bw / 2.)
avg_ts = np.average(lfx.data[lfbin:hfbin, :], axis=0)
data = avg_ts
lfx.close()
return data
def __init__(self, files, station, freq=20.0, chan='CC'):
filelist = glob.glob(files)
self.filelist = sorted(filelist)
self.station = station
self.chan = chan
self.freqmhz = freq
self.res = len(filelist)
time_array = []
station_array = []
freqbin_array = []
for i in range(len(self.filelist)):
head = bb.LofasmFile(self.filelist[i]).header
file_startt = head['start_time']
file_station = head['station']
file_pol = head['channel']
start_time = datetime.datetime.strptime(file_startt[:-8],
'%Y-%m-%dT%H:%M:%S')
bw = (float(head['dim2_span'])/1000000.0)/head['metadata']['dim2_len']
freqbin = int((self.freqmhz-(float(head['dim2_start'])/1000000.0))/bw)
time_array.append(start_time)
station_array.append(file_station)
freqbin_array.append(freqbin)
self.cali_array = [time_array, station_array]
self.freqbins = freqbin_array
for i in reversed(range(len(self.filelist))):
if self.cali_array[1][i] != str(self.station):
del self.cali_array[0][i]
del self.cali_array[1][i]
lfdic = {1:{'name':'LI', 'lat':[26,33,19.676], 'long':[97,26,31.174], 't_offset':6.496132851851852},
2:{'name':'LII', 'lat':[34,4,43.497], 'long':[107,37,5.819], 't_offset':7.174552203703703},
3:{'name':'LIII', 'lat':[38,25,59.0], 'long':[79,50,23.0], 't_offset':5.322648148148148},
4:{'name':'LIV', 'lat':[34,12,3.0], 'long':[118,10,18.0], 't_offset':7.87811111111111}}
self.lfs = lfdic[station]
def add_files(self, files):
"""Add files to the calibrate class.
New files are sorted, duplicates are ignored, header data is read, and
the file lists are appended together.
Parameters
----------
files : str
A path to lofasm `.bbx.gz` files. `*` wildcard can be used for multiple
files.
"""
new_filelist = glob.glob(files)
for i in new_filelist:
if i in self.filelist:
new_filelist.remove(i)
self.filelist = (self.filelist + new_filelist)
self.filelist = sorted(self.filelist)
time_array = []
station_array = []
for i in range(len(self.filelist)):
head = bb.LofasmFile(self.filelist[i]).header
file_startt = head['start_time']
file_station = head['station']
start_time = datetime.datetime.strptime(file_startt[:-8],
'%Y-%m-%dT%H:%M:%S')
time_array.append(start_time)
station_array.append(file_station)
self.cali_array = [time_array, station_array]
for i in reversed(range(len(self.filelist))):
if self.cali_array[1][i] != str(self.station):
del self.cali_array[0][i]
del self.cali_array[1][i]
del self.filelist[i]
def convert_bbx_file(lofasm_file):
from lofasm.bbx import bbx
lf = bbx.LofasmFile(lofasm_file)
lf.read_data()
csvname = get_csv_filename(lofasm_file)
if not os.path.isdir(os.path.dirname(csvname)):
try:
os.makedirs(os.path.dirname(csvname))
except:
print "unable to create directory : {}".format(
os.path.dirname(csvname))
print "please create the directory manually and try again"
sys.stdout.flush()
wstart = time()
with open(csvname, 'wb') as csvfile:
for i in range(np.shape(lf.data)[1]):
data = lf.data[:1024, i]
writeRow(csvfile, data)
wend = time()
print "wrote {} in {} s".format(csvname, str(wend - wstart))
sys.stdout.flush()
if filename.endswith(".lofasm.gz"):
os.system("loco2bx.py -p CC,DD,CD " + filename)
print(os.path.join(filename) +" converted to bbx ("+str(j)+"/"+str(n)+")")
j+=1
'''
#go into CC channel and take an average of the values in the array
os.chdir(cwd1 + "/bbx/CC")
cwd2 = os.getcwd()
i = 0
lst = os.listdir(cwd2)
lst.sort()
for filename2 in lst:
if filename2.endswith(".bbx.gz"):
lf = bbx.LofasmFile(os.path.join(filename2))
lf.read_data()
countavgCC[i] = np.average(lf.data)
print(filename2 + " average added to array")
print("\naverage = " + str(countavgCC[i]))
spect_avg = np.average(lf.data, axis=0)
countfreqCC[i] = spect_avg[pdat.freq2bin(50)]
print(filename2 + " freq added to array")
print("average freq = " + str(countfreqCC[i]))
countbinCC[i] = np.average(lf.data, axis=0)
print(filename2 + " avg bin added to array")
i += 1
def read_files(self, files, freq, verbose=True):
"""Creates data attribute using the minimum value in frequency and time
for each file.
"""
self.freqmhz = freq
self.filelist = sorted(glob.glob(files))
#Remove non lofasm files from filelist
for f in reversed(self.filelist):
if not bb.is_lofasm_bbx(f):
self.filelist.remove(f)
re = 'Reading data...'
for i in range(len(self.filelist)):
f = bb.LofasmFile(self.filelist[i])
head = f.header
startt = head['start_time']
timebins = head['metadata']['dim1_len']
timelength = float(head['dim1_span'])
# Convert header start_time to datetime object
# ================TODO=====================================
# Due to a bug some LoFASM data has messed up start times.
# What follows is a hotfix so that this library can support
# both time formats. In the future, the bug should be fixed
# and this library should only have to support 1 time format.
#
# The following loop tries each format specified in fmts.
# If the time is parsed correctly (and does not throw an
# exception) then the loop will be broken.
# ==========================================================
fmts = ["%Y-%m-%dT%H:%M:%S", # correct time format for bbx files
"%Y%m%d_%H%M%S"] # additional format found in LoFASM I files
startt_repr = startt[:-8] if 'T' in startt else startt
for fmt in fmts:
try:
time_obj = datetime.datetime.strptime(startt_repr, fmt)
break
except ValueError:
pass
else:
print "Cannot parse start time header field {}".format(startt_repr)
# Find the frequency bin corresponding to the given frequency
bw = (float(head['dim2_span'])/1000000.0)/head['metadata']['dim2_len']
freqbin = int((self.freqmhz-(float(head['dim2_start'])/1000000.0))/bw)
self.freq_bin = np.append(self.freq_bin, freqbin)
f.read_data()
freqbin_range = 10 #bins around specified 'freq' to read
freqbins = f.data[freqbin-freqbin_range/2:freqbin+freqbin_range/2,:]
min_vals_per_timebin = [np.min(x) for x in np.rot90(freqbins, k=-1)]
min_file_val = np.min(min_vals_per_timebin)
f.close()
# Compute datetime of the selected timebin
index = min_vals_per_timebin.index(min_file_val)
seconds_into_file = (float(index)/timebins)*timelength
seconds_into_file = datetime.timedelta(seconds=seconds_into_file)
time_from_bin = time_obj + seconds_into_file
self.data = np.append(self.data, min_file_val)
self.times_array = np.append(self.times_array, time_from_bin)
if verbose == True:
p = (str(i*100/len(self.filelist)) + '%')
if i+1 not in range(len(self.filelist)):
p = 'Done'
sys.stdout.write("\r%s%s" % (re,p))
sys.stdout.flush()
def avgpow(lf, freq):
x = lf.data[freq2bin(freq), :]
return (np.mean(x))
if __name__ == '__main__':
import argparse
p = argparse.ArgumentParser()
p.add_argument('fnames', nargs='+')
p.add_argument('freq', type=float)
args = p.parse_args()
x = []
for f in args.fnames:
lf = bbx.LofasmFile(f)
lf.read_data()
pows = lf.data[freq2bin(args.freq), :]
avg = np.mean(pows)
start = lf.header['start_time']
start_cv = Time(start, scale='utc')
st = int(lf.header['station'])
lfstation = LoFASM_Stations[st]
lon = lfstation.lon * 180 / np.pi
start_sr = start_cv.sidereal_time('mean', lon)
start_hr = start_sr.hour
powmodel = galaxyPower.calculatepower(start_hr, st, args.freq, 0)
x.append((start_hr, avg, powmodel))
t, avg, gal = zip(*x)
def numpyf(n, filei):
countavgCC = np.zeros(n)
countfreqCC = np.zeros(n)
countbinCC = np.zeros((n, 1024))
countavgDD = np.zeros(n)
countfreqDD = np.zeros(n)
countbinDD = np.zeros((n, 1024))
countavgCD = np.zeros(n)
countfreqCD = np.zeros(n)
countbinCD = np.zeros((n, 1024), dtype=complex)
cwd1 = os.getcwd()
#go into CC channel and take an average of the values in the array
os.chdir(cwd1 + "/bbx/CC")
cwd2 = os.getcwd()
i = 0
lst = os.listdir(cwd2)
lst.sort()
for filename2 in lst:
if filename2.endswith(".bbx.gz"):
lf = bbx.LofasmFile(os.path.join(filename2))
lf.read_data()
countavgCC[i] = np.average(lf.data)
print(filename2 + " average added to array")
print("\naverage = " + str(countavgCC[i]))
countbinCC[i] = np.average(lf.data, axis=0)
print(filename2 + " avg bin added to array")
i += 1
lf.close()
#go into DD channel and take an average of the values in the array
os.chdir(cwd1)
os.chdir(cwd1 + "/bbx/DD")
cwd3 = os.getcwd()
i = 0
lst = os.listdir(cwd3)
lst.sort()
for filename3 in lst:
if filename3.endswith(".bbx.gz"):
df = bbx.LofasmFile(os.path.join(filename3))
df.read_data()
countavgDD[i] = np.average(df.data)
print(filename3 + " average added to array")
print("\naverage = " + str(countavgDD[i]))
countbinDD[i] = np.average(df.data, axis=0)
print(filename3 + " avg bin added to array")
i += 1
df.close()
#go into CD channel and take an average of the values in the array
os.chdir(cwd1)
os.chdir(cwd1 + "/bbx/CD")
cwd4 = os.getcwd()
i = 0
lst = os.listdir(cwd4)
lst.sort()
for filename4 in lst:
if filename4.endswith(".bbx.gz"):
df = bbx.LofasmFile(os.path.join(filename4))
df.read_data()
countavgCD[i] = np.average(df.data)
print(filename4 + " average added to array")
print("\naverage = " + str(countavgCD[i]))
countbinCD[i] = np.average(df.data, axis=0)
print(filename4 + " avg bin added to array")
i += 1
df.close()
#save/plot the output
os.chdir(cwd1)
filesv = filei[:-1]
np.save(str(filesv) + 'outputavgCC', countavgCC)
np.save(str(filesv) + 'outputbinCC', countbinCC)
np.save(str(filesv) + 'outputavgDD', countavgDD)
np.save(str(filesv) + 'outputbinDD', countbinDD)
np.save(str(filesv) + 'outputavgCD', countavgCD)
np.save(str(filesv) + 'outputbinCD', countbinCD)
lfx._hdr_fname) else "{} does not exist".format(
basename(lfx._hdr_fname))
msg("size of {} on disk (Bytes)".format(basename(lfx._data_fname)))
print os.path.getsize(lfx._data_fname) if exists(
lfx._data_fname) else "{} does not exist".format(
basename(lfx._data_fname))
msg("size of {} on disk (Bytes)".format(basename(lfx.fpath)))
print os.path.getsize(lfx.fpath) if exists(
lfx.fpath) else "{} does not exist".format(lfx.fpath)
fname = 'testfile.bbx'
# purge file if it already exists
if os.path.exists(fname):
os.remove(fname)
lfx = bbx.LofasmFile('testfile.bbx', mode='write', gz=True)
# set required header fields with fake information
lfx.set('station', 'fake')
lfx.set('channel', 'fake')
lfx.set('dim1_start', '0')
lfx.set('dim1_span', '1')
lfx.set('dim2_start', '0')
lfx.set('dim2_span', '1')
lfx.set('data_label', 'filterbank')
report(lfx)
msg("Appending row of ones to internal buffer:")
append_data(lfx, ones_row)
report(lfx)
msg("Dumping internal buffer to disk")
lfx.write()
report(lfx)
#! /usr/bin/env python
if __name__ == "__main__":
import argparse
import matplotlib
matplotlib.use('agg') # on headless linux
import matplotlib.pyplot as plt
from lofasm.bbx import bbx
import numpy as np
p = argparse.ArgumentParser()
p.add_argument('filename', type=str, help="path to BBX file")
args = p.parse_args()
lfx = bbx.LofasmFile(args.filename)
lfx.read_data()
startt = lfx.header['start_time']
avg_spectra = np.average(lfx.data, axis=1)
plt.figure()
plt.title('L1 Average Spectra {}'.format(startt))
plt.plot(10 * np.log10(avg_spectra), label='Avg. Spectra')
plt.legend()
plt.savefig('{}.avg.png'.format(lfx.fname), format='png')
lfx.fpath) else "{} does not exist".format(lfx.fpath)
# Set the parser stuff
parser = argparse.ArgumentParser()
parser.add_argument("-p", "--path", help="Path to input bbx file.")
parser.add_argument("-f",
"--freq",
help="Set frequancy channel to be removed. In MHz.")
parser.add_argument("-o", "--outfile", help="Name for file created.")
'''
Stream Data. Open the file we want to mod.
Prep the file we are to write too.
'''
pars_dict = vars(parser.parse_args())
bbxFileHandel = bbx.LofasmFile(pars_dict['path'])
bbxfile = pars_dict['path']
print "File to be read open. "
bbxFileHeader = bbxFileHandel.header
outPath = os.getcwd()
outPath = outPath + '/' + pars_dict['outfile']
outFile = bbx.LofasmFile(outPath, bbxFileHeader, mode='write')
print "File to write created. "
for i in range(int(bbxFileHeader['metadata']['dim1_len'])):
bbxFileHandel.read_data(1) # Read one line of data
spectra = bbxFileHandel.data[:, :] #.astype(np.float32)
spectra[:, pdat.freq2bin(float(pars_dict['freq']))] = 0
# report(outFile)
append_data(outFile, spectra)
if __name__ == '__main__':
import argparse
p = argparse.ArgumentParser()
p.add_argument('fnames', nargs='+') #reading the given arguments into
p.add_argument('freq', type=float) #into variables
p.add_argument('hor', type=int)
args = p.parse_args()
args.fnames.sort() #sort the files
x = []
start_hr_arr = np.zeros(len(args.fnames))
fil = args.fnames[0]
lf = bbx.LofasmFile(fil)
st = int(lf.header['station'])
# lsts, pows = galaxyPower.loadfile(st, args.freq, args.hor)
for ii, f in enumerate(args.fnames): #for loop to read through the data
lf = bbx.LofasmFile(f) #and append to an empty list
lf.read_data() #the time in lst, the average power
lofasm_pows = lf.data[:,
freq2bin(args.freq)] #and the given galaxy power
avg = np.mean(lofasm_pows) #at that time
#LOFASM 3 [
# start = float(lf.header['start_mjd'])
# start_postmjd = Time(start, format = 'mjd')
# st = int(lf.header['station'])
help='wide band mask threshold.')
# Parse the commonline parameters
args = parser.parse_args()
start_time = time.time()
infile = args.infilename
outfile = args.outfilename
compress = args.compress
med_window_size = int(args.median_window_size)
min_window_size = int(args.running_minimum_window_size)
norm = args.normalize
outlier_thrhd = float(args.o_thresh)
narrowband_thrhd = float(args.nb_thresh)
wideband_thrhd = float(args.wb_thresh)
lf = b.LofasmFile(infile)
lf.read_data()
norm_data, normalize_array = n.robust_normalize(
lf.data,
median_window=med_window_size,
running_min_window=min_window_size)
#medianed_data = n.running_median(lf.data, median_window=med_window_size)
#norm_data, normalize_array = c.normalize(medianed_data, fast=True, window=med_window_size)
o_mask, outlier_average_top, outlier_average_bottom = c.outlier_mask(
norm_data, threshold=outlier_thrhd)
nb_mask, percent_clean_freq_channels = c.narrow_band_mask(
norm_data, threshold=narrowband_thrhd)
wb_mask, percent_clean_time = c.wide_band_mask(norm_data,
threshold=wideband_thrhd)
print infile + '---------------------------'
if __name__ == "__main__":
import argparse
p = argparse.ArgumentParser()
p.add_argument('datasetdir', type=str, help="path to dataset directory")
p.add_argument(
'--savedata',
action='store_true',
help='Save a copy of the resulting data array as a numpy file')
args = p.parse_args()
flist = glob(os.path.join(args.datasetdir, "*.bbx.gz"))
flist.sort()
lfx = bbx.LofasmFile(flist[0])
pol = lfx.header['channel']
lfx.close()
data = np.zeros((1024, len(flist)), dtype=np.float64)
for i in range(len(flist)):
re = "Processing {}/{}".format(i + 1, len(flist))
sys.stdout.write("\r" + re)
sys.stdout.flush()
avgSpectra = averageFileSpectra(flist[i])
data[:, i] = avgSpectra
if args.savedata:
with open('waterfall_out_data_freqvtime_{}.numpy'.format(pol),
'w') as f:
data.tofile(f)
parser.add_argument(
'-fn',
action='store',
dest='file_name',
default='*.bbx.gz',
help='do you want to clean a specific file or type of file? try ')
args = parser.parse_args()
lower_frequency = args.lower_frequency_bin * (100.0 / 1024.0)
upper_frequency = args.upper_frequnecy_bin * (100.0 / 1024.0)
for fname in glob.glob(args.file_name):
lf = b.LofasmFile(fname)
lf.read_data()
data = 10 * np.log10(
lf.data[args.lower_frequency_bin:args.upper_frequnecy_bin])
norm_data, normalize_array = c.normalize(data, fast=args.fast)
o_mask, outlier_average_top, outlier_average_bottom = c.outlier_mask(
norm_data, threshold=args.o_thresh)
nb_mask, percent_clean_freq_channels = c.narrow_band_mask(
norm_data, threshold=args.nb_thresh)
wb_mask, percent_clean_time = c.wide_band_mask(norm_data,
threshold=args.wb_thresh)
print fname + '---------------------------'
print 'top outlier average : ' + str(outlier_average_top)
print 'bottom outlier average : ' + str(outlier_average_bottom)
import os
import numpy as np
from lofasm.bbx import bbx
cwd = os.getcwd()
for filei in os.listdir(cwd):
if filei.endswith(".bbx.gz"):
lf = bbx.LofasmFile(os.path.join(filei))
lf.read_data()
np.save(filei, lf.data)
lf.close()
def __init__(
self,
files,
output,
station,
freq=20.0,
):
cal = calibrate(files, station, freq=freq)
filelist = sorted(glob.glob(files))
dat1 = bb.LofasmFile(filelist[0])
dat1.read_data()
avgfull = np.average(dat1.data[cal.freqbins[0] - 5:cal.freqbins[0] +
5, :],
axis=0) #First element of list of data arrays
list_of_powers = [avgfull] #List of data arrays (2d)
dat = np.average(
list_of_powers[0]) #First element of dat array for calibration
re = 'Reading data... '
for filei in range(len(filelist) - 1):
filei += 1
bbfile = bb.LofasmFile(filelist[filei])
bbfile.read_data()
### Preparing array containing full power of each file
avgfull_power = np.average(
bbfile.data[cal.freqbins[filei] - 5:cal.freqbins[filei] +
5, :],
axis=0) ##Avg 10 bins around frequency
list_of_powers.append(avgfull_power)
### Preparing data array for calibration: Each datapoint is avg power of file
avg_datafile_power = np.average(avgfull_power)
dat = np.append(dat, avg_datafile_power)
p = ((str(filei * 100 / len(filelist)) + '%') + ' - [' +
str(filei + 1) + ' out of ' + str(len(filelist)) + ' files]')
if filei not in range(len(filelist) - 1):
p = 'Done \n'
sys.stdout.write("\r%s%s" % (re, p))
sys.stdout.flush()
# print "Generating models... " done by calibration_pmts f'n
calibration_pmts = cal.calibration_parameters(data=dat)
sys.stdout.write('\rCalibrating and writing files... ')
if output[-1] != '/':
output += '/'
for i in range(len(filelist)):
filename = os.path.basename(filelist[i])
calname = (output + 'Calibrated_' + filename)
calibrated = (list_of_powers[i] -
calibration_pmts[1]) / calibration_pmts[0]
lfc = bb.LofasmFile(output + 'Calibrated_' + filename,
mode='write')
lfc.add_data(calibrated)
lfc.write()
lfc.close()
sys.stdout.write('\rDone - ' + str(len(filelist)) +
' calibrated files written.')
sys.stdout.flush()
