def DataUpdate():
Data = pd.read_csv("dataset13-v3.csv", sep=';', engine='python')
tstart = str(int(Data["Year"][len(Data) - 1])) + '/' + str(
int(Data["Month"][len(Data) - 1])) + '/' + str(
int(Data["Day"][len(Data) - 1])) + ' 23:59:59.000'
tend = dt.datetime.now()
print(tend)
if tend.day != int(tstart[8:10]):
time_range = TimeRange((tstart, tend))
n_days = int(round(time_range.days.value))
days_array = np.array(
list(str(i.start).split('T')[0] for i in time_range.split(n_days)))
year, month, day = tstart[:10].split('/')
for m in time_range.split(n_days):
time_range1 = TimeRange((m.start, m.end))
print(str(m.start)[:10] + ' / ' + str(tend), end='')
print('\r', end='')
try:
key1_1, value1_1, key1_2, value1_2 = Sunflare_data_finder(
time_range1, n_days, days_array, client)
key2_1, value2_1, key2_2, value2_2 = Coronal_Holes_data_finder(
time_range1, n_days, days_array, client)
key3_1, value3_1, key3_2, value3_2, key3_3, value3_3 = Sunspot_Finder(
time_range1, n_days, client)
key5_1, value5_1, key5_2, value5_2, key5_3, value5_3 = CME_Finder(
time_range1, n_days, client)
except:
key1_1, value1_1, key1_2, value1_2 = Sunflare_data_finder(
time_range1, n_days, days_array, client)
key2_1, value2_1, key2_2, value2_2 = Coronal_Holes_data_finder(
time_range1, n_days, days_array, client)
key3_1, value3_1, key3_2, value3_2, key3_3, value3_3 = Sunspot_Finder(
time_range1, n_days, client)
key5_1, value5_1, key5_2, value5_2, key5_3, value5_3 = CME_Finder(
time_range1, n_days, client)
key4_1, value4_1, key4_2, value4_2, key4_3, value4_3, key4_4, value4_4 = Geostorm_Finder(
m.end, days_array)
data = Dict_Generator([key4_1, key1_1, key1_2, key2_1, key2_2, key3_1, key3_2, key3_3, key4_2, key4_3, key4_4, key5_1, key5_2, key5_3],
[value4_1, value1_1, value1_2, value2_1,\
value2_2, value3_1, value3_2, value3_3,\
value4_2,\
value4_3,\
value4_4,\
value5_1,\
value5_2,\
value5_3])
Data = Data.append(data, ignore_index=True, sort=False)
Data.to_csv('dataset13-v3.csv', sep=';', index=False)
pass
readTimestamp = 0
dataBlock_all = filReader.readBlock(readTimestamp, samplesPerBlock)
stokesI = spp.Filterbank.FilterbankBlock(dataBlock_all,
dataBlock_all.header)
stokesI = stokesI.normalise()
time_len = stokesI.shape[1]
time_res = 5.12e-6 * 512
trange = TimeRange(tstart, time_len * time_res * u.second)
sbs = np.arange(51, 461)
obs_mode = 3
no_sbs = len(sbs)
ylims, xlims = get_data_lims(sbs, obs_mode, no_sbs, trange)
nsplit = 10
df_chunk = data_chunker(stokesI.data, nsplit)
dts = trange.split(nsplit)
plot_names = 'all_data_plots/Uranus_StokesI_normalised_'
for i, j in enumerate(df_chunk):
print(plot_names + str(i))
ys = j.sum(axis=1)[::-1]
xs = j.sum(axis=0)
_, xlims = get_data_lims(sbs, obs_mode, no_sbs, dts[i])
plot_data(j.T, xs, ys, xlims, ylims, xlabel, ylabel,
plot_names + str(i), plot_title)
plt.close()
print('...next')
rawdata.data = rawdata.data[:, :
no_sbs] #need to do this because of the way subbands were set up for uranus observations! (only use 78 subbands!)
#off-beam
rawoffdata = LofarRaw(fname=off_fname,
sbs=sbs,
obs_mode=obs_mode,
frange=frange)
rawoffdata.data = rawoffdata.data[:, :no_sbs]
#stokes V
sV_data = LofarRaw(fname=sV, sbs=sbs, obs_mode=obs_mode, frange=frange)
sV_data.data = sV_data.data[:, :no_sbs]
df_chunk = ued.data_chunker(rawdata.data, nsplit)
off_chunk = ued.data_chunker(rawoffdata.data, nsplit)
sV_chunks = ued.data_chunker(sV_data.data, nsplit)
tchunks = trange.split(nsplit)
strings = ["StokesI", "StokesI_OFF", "StokesV"]
#strings = ["StokesV"]
for i, df in enumerate([df_chunk, off_chunk, sV_chunks]):
#for i,df in enumerate([sV_chunks]):
total_f_sum = []
print("Analysing {}".format(strings[i]))
for n, df_split in enumerate(df):
print("Analysing Chunk #{}".format(n + 1))
ylims, xlims = ued.get_data_lims(sbs, obs_mode, no_sbs, tchunks[n])
print("Removing RFI")
df_norfi = ued.resample_dataset(
df_split, f=1220
) #resampled to 100 ms resolution to mask rfi shorter than this
rfi_mask = edf.basic_filter(df_norfi, 4.)
