if df_filename[-4:] == '.adr': freqList = freqList_GURT AmplitudeReIm = AmplitudeReIm_GURT [df_filename, df_filesize, df_system_name, df_obs_place, df_description, CLCfrq, df_creation_timeUTC, ReceiverMode, Mode, sumDifMode, NAvr, TimeRes, fmin, fmax, df, frequency, FFTsize, SLine, Width, BlockSize] = FileHeaderReaderADR(path_to_data + dat_files_list[file_no], 0, 0) if df_filename[-4:] == '.jds': # If data obrained from DSPZ receiver freqList = freqList_UTR2 AmplitudeReIm = AmplitudeReIm_GURT [df_filename, df_filesize, df_system_name, df_obs_place, df_description, CLCfrq, df_creation_timeUTC, SpInFile, ReceiverMode, Mode, Navr, TimeRes, fmin, fmax, df, frequency, FreqPointsNum, dataBlockSize] = FileHeaderReaderJDS(path_to_data + dat_files_list[file_no], 0, 0) if AutoSourceSwitch == 1: if df_filename[-4:] == '.jds': # If data obrained from DSPZ receiver ''' if '3c461' in df_description.lower() or 'cas' in df_description.lower() or '461' in df_description.lower(): source = '3C461' elif '3c405' in df_description.lower() or 'cyg' in df_description.lower() or '405' in df_description.lower(): source = '3C405' else: print(' Source not detected !!!') source = str(input(' * Enter source name like 3C405 or 3C461: ')) ''' if file_no == 0: source = '3C405' elif file_no == 1:
def normalize_dat_file(directory, filename, no_of_spectra_in_bunch, median_filter_window, show_aver_spectra): """ function calculates the average spectrum in DAT file and normalizes all spectra in file to average spectra Input parameters: directory - name of directory with initial dat file filename - name of initial dat file no_of_spectra_in_bunch - number of spectra in bunch to read median_filter_window - window of median filter to process the average spectrum show_aver_spectra - boolean variable which indicates if the picture of average spectrum to be shown and the script paused till the picture window is closed Output parameters: output_file_name - name of result normalized .dat file """ print( '\n Preparations and calculation of the average spectrum to normalize... \n' ) output_file_name = directory + 'Norm_' + filename filename = directory + filename # Opening DAT datafile file = open(filename, 'rb') # *** Data file header read *** df_filesize = os.stat(filename).st_size # Size of file df_filename = file.read(32).decode('utf-8').rstrip( '\x00') # Initial data file name file.close() if df_filename[-4:] == '.adr': [ df_filename, df_filesize, df_system_name, df_obs_place, df_description, CLCfrq, df_creation_timeUTC, ReceiverMode, Mode, sumDifMode, NAvr, TimeRes, fmin, fmax, df, frequency, FFTsize, SLine, Width, BlockSize ] = FileHeaderReaderADR(filename, 0, 0) if df_filename[-4:] == '.jds': # If data obtained from DSPZ receiver [ df_filename, df_filesize, df_system_name, df_obs_place, df_description, CLCfrq, df_creation_timeUTC, SpInFile, ReceiverMode, Mode, Navr, TimeRes, fmin, fmax, df, frequency, FreqPointsNum, dataBlockSize ] = FileHeaderReaderJDS(filename, 0, 0) # Calculation of the dimensions of arrays to read nx = len(frequency) # the first dimension of the array ny = int(( (df_filesize - 1024) / (nx * 8))) # the second dimension of the array: file size - 1024 bytes # Number of data blocks to read from file num_of_blocks = int(ny // no_of_spectra_in_bunch) # Read data from file by blocks and average it file = open(filename, 'rb') file.seek(1024) average_array = np.empty((nx, 0), float) for block in range(num_of_blocks): if block == (num_of_blocks - 1): spectra_num_in_bunch = ny - (num_of_blocks - 1) * no_of_spectra_in_bunch else: spectra_num_in_bunch = no_of_spectra_in_bunch data = np.fromfile(file, dtype=np.float64, count=nx * spectra_num_in_bunch) data = np.reshape(data, [nx, spectra_num_in_bunch], order='F') tmp = np.empty((nx, 1), float) # tmp[:, 0] = data.mean(axis=1)[:] tmp[:, 0] = data.min(axis=1)[:] average_array = np.append(average_array, tmp, axis=1) # # Average average spectra of all data blocks average_profile = average_array.mean(axis=1) init_average_profile = average_profile.copy() # # Make a figure of average spectrum (profile) # fig = plt.figure(figsize=(9, 5)) # ax1 = fig.add_subplot(111) # ax1.plot(10 * np.log10(average_profile), linestyle='-', linewidth='1.00', label='Average spectra') # ax1.legend(loc='upper right', fontsize=6) # ax1.grid(b=True, which='both', color='silver', linestyle='-') # ax1.set_xlabel('Frequency points, num.', fontsize=6, fontweight='bold') # ax1.set_ylabel('Intensity, dB', fontsize=6, fontweight='bold') # pylab.savefig('Averaged_spectra_'+filename[:-4]+'_before_filtering.png', bbox_inches='tight', dpi=160) # plt.close('all') # Apply median filter to average profile average_profile = median_filter(average_profile, median_filter_window) med_average_profile = average_profile.copy() average_profile = average_filter(average_profile, median_filter_window + 20) # Make a figure of filtered average spectrum (profile) fig = plt.figure(figsize=(12, 8)) ax1 = fig.add_subplot(111) ax1.plot(10 * np.log10(init_average_profile), linestyle='-', linewidth='1.50', label='Initial spectra', color='C0', alpha=0.6) ax1.plot(10 * np.log10(med_average_profile), linestyle='-', linewidth='1.25', label='Median spectra', color='C1', alpha=0.8) ax1.plot(10 * np.log10(average_profile), linestyle='-', linewidth='1.00', label='Median averaged spectra', color='C3') ax1.legend(loc='upper right', fontsize=6) ax1.grid(b=True, which='both', color='silver', linestyle='-') ax1.set_xlabel('Frequency points, num.', fontsize=6, fontweight='bold') ax1.set_ylabel('Intensity, dB', fontsize=6, fontweight='bold') pylab.savefig('Averaged_spectra_' + filename[:-4] + '_after_filtering.png', bbox_inches='tight', dpi=160) if show_aver_spectra: print('\n Close the figure window to continue processing!!!\n') plt.show() plt.close('all') del init_average_profile, med_average_profile # Normalization print(' Spectra normalization... \n') file.seek(0) file_header = file.read(1024) normalized_file = open(output_file_name, 'wb') normalized_file.write(file_header) del file_header bar = IncrementalBar(' Normalizing of the DAT file: ', max=num_of_blocks, suffix='%(percent)d%%') bar.start() for block in range(num_of_blocks): if block == (num_of_blocks - 1): spectra_num_in_bunch = ny - (num_of_blocks - 1) * no_of_spectra_in_bunch else: spectra_num_in_bunch = no_of_spectra_in_bunch data = np.fromfile(file, dtype=np.float64, count=nx * spectra_num_in_bunch) data = np.reshape(data, [nx, spectra_num_in_bunch], order='F') for j in range(spectra_num_in_bunch): data[:, j] = data[:, j] / average_profile[:] temp = data.transpose().copy(order='C') normalized_file.write(np.float64(temp)) bar.next() file.close() normalized_file.close() bar.finish() # *** Creating a new timeline TXT file for results *** new_tl_file_name = output_file_name.split('_Data_', 1)[0] + '_Timeline.txt' new_tl_file = open( new_tl_file_name, 'w') # Open and close to delete the file with the same name new_tl_file.close() # *** Reading timeline file *** old_tl_file_name = filename.split('_Data_', 1)[0] + '_Timeline.txt' old_tl_file = open(old_tl_file_name, 'r') new_tl_file = open(new_tl_file_name, 'w') # Read time from timeline file time_scale_bunch = old_tl_file.readlines() # Saving time data to new file for j in range(len(time_scale_bunch)): new_tl_file.write((time_scale_bunch[j][:]) + '') old_tl_file.close() new_tl_file.close() return output_file_name
def check_long_spectra_files_from_wf(directory, file_re, file_im): ''' Checks spectra from the long data files ''' # Preparing long data files fname = directory + fileList[0] [ df_filename, df_filesize, df_system_name, df_obs_place, df_description, CLCfrq, df_creation_timeUTC, Channel, ReceiverMode, Mode, Navr, TimeRes, fmin, fmax, df, frequency, FreqPointsNum, data_block_size ] = FileHeaderReaderJDS(fname, 0, 1) no_of_blocks_in_file = (df_filesize - 1024) / data_block_size print(' Number of blocks in file: ', no_of_blocks_in_file) no_of_blocks_in_batch = int(no_of_blocks_in_file / (2 * no_of_batches_in_file)) print(' Number of blocks in batch: ', no_of_blocks_in_batch) # *** Calculation of the dimensions of arrays to read *** nx = len(frequency) # the first dimension of the array ny = int(( (df_filesize - 1024) / (nx * 8))) # the second dimension of the array: file size - 1024 bytes istart = 0 istop = len(timeline) print(' Number of frequency channels: ', nx, '\n') print(' Number of spectra: ', ny, '\n') print(' Recomended spectra number for averaging is: ', int(ny / 1024)) averageConst = int(ny / 1024) if int(averageConst) < 1: averageConst = 1 # *** Data reading and averaging *** print('\n\n\n *** Data reading and averaging *** \n\n') file1 = open(file_re, 'rb') file2 = open(file_im, 'rb') file1.seek( 1024 + istart * 8 * nx, os.SEEK_SET ) # Jumping to 1024+number of spectra to skip byte from file beginning file2.seek( 1024 + istart * 8 * nx, os.SEEK_SET ) # Jumping to 1024+number of spectra to skip byte from file beginning array = np.empty((nx, 0), float) numOfBlocks = int(ny / averageConst) for block in range(numOfBlocks): data1 = np.fromfile(file1, dtype=np.float64, count=nx * averageConst) data2 = np.fromfile(file2, dtype=np.float64, count=nx * averageConst) data = np.power(np.abs(data1[:, :] + 1j * data2[:, :]), 2) del data1, data2 data = np.reshape(data, [nx, averageConst], order='F') dataApp = np.empty((nx, 1), float) with np.errstate(invalid='ignore'): dataApp[:, 0] = 10 * np.log10(data.mean(axis=1)[:]) array = np.append(array, dataApp, axis=1) array[np.isnan(array)] = -120 del dataApp, data file1.close() if (typesOfData[j] == 'A+B' or typesOfData[j] == 'A-B'): file2.close() return
del hour, minutes, seconds, phase_of_second return timeline_block_str if __name__ == '__main__': fname = 'DATA/E010621_090610.jds' print('\n\n Parameters of the file: ') # *** Data file header read *** [ df_filename, df_filesize, df_system_name, df_obs_place, df_description, CLCfrq, df_creation_timeUTC, Channel, ReceiverMode, Mode, Navr, TimeRes, fmin, fmax, df, frequency, FreqPointsNum, data_block_size ] = FileHeaderReaderJDS(fname, 0, 1) # ******************************************************************************* # R E A D I N G D A T A * # ******************************************************************************* no_of_spectra_to_average = 64 print('\n *** Reading data from file *** \n') with open(fname, 'rb') as file: file.seek(1024) # Jumping to 1024 byte from file beginning # for av_sp in range(1): # Reading and reshaping all data with readers wf_data = np.fromfile(file, dtype='i2',
df_description, CLCfrq, df_creation_timeUTC, ReceiverMode, Mode, sumDifMode, NAvr, TimeRes, fmin, fmax, df, frequency, FFTsize, SLine, Width, BlockSize ] = FileHeaderReaderADR(path_to_data + dat_files_list[file_no], 0, 0) if df_filename[-4:] == '.jds': # If data obtained from DSPZ receiver freqList = freqList_UTR2 AmplitudeReIm = AmplitudeReIm_GURT [ df_filename, df_filesize, df_system_name, df_obs_place, df_description, CLCfrq, df_creation_timeUTC, SpInFile, ReceiverMode, Mode, Navr, TimeRes, fmin, fmax, df, frequency, FreqPointsNum, dataBlockSize ] = FileHeaderReaderJDS(path_to_data + dat_files_list[file_no], 0, 0) if AutoSourceSwitch == 1: if df_filename[ -4:] == '.jds': # If data obrained from DSPZ receiver ''' if '3c461' in df_description.lower() or 'cas' in df_description.lower() or '461' in df_description.lower(): source = '3C461' elif '3c405' in df_description.lower() or 'cyg' in df_description.lower() or '405' in df_description.lower(): source = '3C405' else: print(' Source not detected !!!') source = str(input(' * Enter source name like 3C405 or 3C461: ')) ''' if file_no == 0: source = '3C405'
def DAT_file_reader(common_path, DAT_file_name, typesOfData, DAT_result_path, averOrMin, StartStopSwitch, SpecFreqRange, VminMan, VmaxMan, VminNormMan, VmaxNormMan, RFImeanConst, customDPI, colormap, ChannelSaveTXT, ChannelSavePNG, ListOrAllFreq, AmplitudeReIm, freqStart, freqStop, dateTimeStart, dateTimeStop, freqStartTXT, freqStopTXT, freqList, print_or_not): startTime = time.time() currentTime = time.strftime("%H:%M:%S") currentDate = time.strftime("%d.%m.%Y") # Files to be analyzed: filename = common_path + DAT_file_name + '_Data_chA.dat' timeLineFileName = common_path + DAT_file_name + '_Timeline.txt' for j in range(len(typesOfData)): # Main loop by types of data to analyze # Current name of DAT file to be analyzed dependent on data type: temp = list(filename) temp[-7:-4] = typesOfData[j] filename = "".join(temp) temp = list(DAT_file_name + '_Data_chA.dat') temp[-7:-4] = typesOfData[j] only_file_name = "".join(temp) if (typesOfData[j] == 'A+B' or typesOfData[j] == 'A-B'): temp = list(filename) temp[-7:-4] = 'chA' filename01 = "".join(temp) temp[-7:-4] = 'chB' filename02 = "".join(temp) filename = filename01 # Print the type of data to be analyzed if print_or_not == 1: print('\n\n Processing data type: ', typesOfData[j], '\n') currentTime = time.strftime("%H:%M:%S") print(' Processing file: ', only_file_name, ' started at: ', currentTime) if print_or_not == 1: print('\n') #************************************************************* # WHAT TO PLOT AND CORRESPONDING PARAMETERS * #************************************************************* YaxName = 'Intensity, dB' Label = 'Intensity' nameAdd = '' fileNameAdd = '' fileNameAddNorm = '' fileNameAddSpectr = '' Vmin = VminMan # Switch once more to initial manual settings after changes in previous loop Vmax = VmaxMan VminNorm = VminNormMan VmaxNorm = VmaxNormMan if typesOfData[j] == 'chA': nameAdd = ' channel A' fileNameAdd = '' fileNameAddNorm = '001_' fileNameAddSpectr = '008_' if typesOfData[j] == 'chB': nameAdd = ' channel B' fileNameAdd = '' fileNameAddNorm = '001_' fileNameAddSpectr = '008_' if typesOfData[j] == 'C_m': nameAdd = ' correlation module' Vmin = -160 VmaxNorm = 2 * VmaxNormMan fileNameAdd = '' fileNameAddNorm = '004_' fileNameAddSpectr = '011_' if typesOfData[j] == 'C_p': nameAdd = ' correlation phase' YaxName = 'Phase, rad' Label = 'Phase' Vmin = -3.5 Vmax = 3.5 fileNameAdd = '005_' fileNameAddSpectr = '012_' if typesOfData[j] == 'CRe': nameAdd = ' correlation RE part' YaxName = 'Amplitude' fileNameAdd = '006_' fileNameAddSpectr = '013_' if typesOfData[j] == 'CIm': nameAdd = ' correlation IM part' YaxName = 'Amplitude' fileNameAdd = '007_' fileNameAddSpectr = '014_' if typesOfData[j] == 'A+B': nameAdd = ' sum A + B' fileNameAddNorm = '003_' fileNameAddSpectr = '009_' if typesOfData[j] == 'A-B': nameAdd = ' difference |A - B|' Vmin = Vmin - 20 Vmax = Vmax - 20 fileNameAdd = '' fileNameAddNorm = '002_' fileNameAddSpectr = '010_' #********************************************************************************* # *** Creating a folder where all pictures and results will be stored (if it doen't exist) *** newpath = common_path + 'DAT_Results_' + DAT_result_path if not os.path.exists(newpath): os.makedirs(newpath) # *** Opening DAT datafile *** file = open(filename, 'rb') # *** Data file header read *** df_filesize = (os.stat(filename).st_size) # Size of file df_filename = file.read(32).decode('utf-8').rstrip( '\x00') # Initial data file name file.close() if df_filename[-4:] == '.adr': [ df_filename, df_filesize, df_system_name, df_obs_place, df_description, CLCfrq, df_creation_timeUTC, ReceiverMode, Mode, sumDifMode, NAvr, TimeRes, fmin, fmax, df, frequency, FFTsize, SLine, Width, BlockSize ] = FileHeaderReaderADR(filename, 0, 0) FreqPointsNum = len(frequency) if df_filename[-4:] == '.jds': # If data obrained from DSPZ receiver [ df_filename, df_filesize, df_system_name, df_obs_place, df_description, CLCfrq, df_creation_timeUTC, SpInFile, ReceiverMode, Mode, Navr, TimeRes, fmin, fmax, df, frequency, FreqPointsNum, dataBlockSize ] = FileHeaderReaderJDS(filename, 0, 0) sumDifMode = '' #************************************************************************************ # R E A D I N G D A T A * #************************************************************************************ # *** Reading timeline file *** TLfile = open(timeLineFileName, 'r') timeline = [] for line in TLfile: timeline.append(str(line)) TLfile.close() if StartStopSwitch == 1: # If we read only specified time limits of files # *** Converting text to ".datetime" format *** dt_timeline = [] for i in range( len(timeline)): # converting text to ".datetime" format # Check is the uS field is empty. If so it means it is equal to '000000' uSecond = timeline[i][20:26] if len(uSecond) < 2: uSecond = '000000' dt_timeline.append( datetime(int(timeline[i][0:4]), int(timeline[i][5:7]), int(timeline[i][8:10]), int(timeline[i][11:13]), int(timeline[i][14:16]), int(timeline[i][17:19]), int(uSecond))) dt_dateTimeStart = datetime(int(dateTimeStart[0:4]), int(dateTimeStart[5:7]), int(dateTimeStart[8:10]), int(dateTimeStart[11:13]), int(dateTimeStart[14:16]), int(dateTimeStart[17:19]), 0) dt_dateTimeStop = datetime(int(dateTimeStop[0:4]), int(dateTimeStop[5:7]), int(dateTimeStop[8:10]), int(dateTimeStop[11:13]), int(dateTimeStop[14:16]), int(dateTimeStop[17:19]), 0) # *** Showing the time limits of file and time limits of chosen part if print_or_not == 1: print( '\n\n Start End \n' ) if print_or_not == 1: print(' File time limits: ', dt_timeline[0], ' ', dt_timeline[len(timeline) - 1], '\n') if print_or_not == 1: print(' Chosen time limits: ', dt_dateTimeStart, ' ', dt_dateTimeStop, '\n') # Verifying that chosen time limits are inside file and are correct if (dt_timeline[len(timeline) - 1] >= dt_dateTimeStart > dt_timeline[0]) and ( dt_timeline[len(timeline) - 1] > dt_dateTimeStop >= dt_timeline[0]) and (dt_dateTimeStop > dt_dateTimeStart): if print_or_not == 1: print(' Time is chosen correctly! \n\n') else: print(' ERROR! Time is chosen out of file limits!!! \n\n') sys.exit(' Program stopped') # Finding the closest spectra to the chosen time limits A = [] B = [] for i in range(len(timeline)): dt_diff_start = dt_timeline[i] - dt_dateTimeStart dt_diff_stop = dt_timeline[i] - dt_dateTimeStop A.append( abs( divmod(dt_diff_start.total_seconds(), 60)[0] * 60 + divmod(dt_diff_start.total_seconds(), 60)[1])) B.append( abs( divmod(dt_diff_stop.total_seconds(), 60)[0] * 60 + divmod(dt_diff_stop.total_seconds(), 60)[1])) istart = A.index(min(A)) istop = B.index(min(B)) if print_or_not == 1: print('\n Start specter number is: ', istart) if print_or_not == 1: print('\n Stop specter number is: ', istop) if print_or_not == 1: print('\n Total number of spectra to read: ', istop - istart) # *** Calculation of the dimensions of arrays to read *** nx = len(frequency) # the first dimension of the array if StartStopSwitch == 1: # If we read only specified time limits of files ny = int( istop - istart ) # the second dimension of the array: number of spectra to read else: ny = int( ((df_filesize - 1024) / (nx * 8)) ) # the second dimension of the array: file size - 1024 bytes istart = 0 istop = len(timeline) if print_or_not == 1: print(' ') if print_or_not == 1: print(' Number of frequency channels: ', nx, '\n') if print_or_not == 1: print(' Number of spectra: ', ny, '\n') if print_or_not == 1: print(' Recomended spectra number for averaging is: ', int(ny / 1024)) # averageConst = raw_input('\n Enter number of spectra to be averaged: ') #if (len(averageConst) < 1 or int(averageConst) < 1): # averageConst = 1 #else: # averageConst = int(averageConst) averageConst = int(ny / 1024) if int(averageConst) < 1: averageConst = 1 # *** Data reading and averaging *** if print_or_not == 1: print('\n\n\n *** Data reading and averaging *** \n\n') file1 = open(filename, 'rb') if (typesOfData[j] == 'A+B' or typesOfData[j] == 'A-B'): file2 = open(filename02, 'rb') file1.seek( 1024 + istart * 8 * nx, os.SEEK_SET ) # Jumping to 1024+number of spectra to skip byte from file beginning if (typesOfData[j] == 'A+B' or typesOfData[j] == 'A-B'): file2.seek( 1024 + istart * 8 * nx, os.SEEK_SET ) # Jumping to 1024+number of spectra to skip byte from file beginning array = np.empty((nx, 0), float) numOfBlocks = int(ny / averageConst) for block in range(numOfBlocks): data1 = np.fromfile(file1, dtype=np.float64, count=nx * averageConst) if (typesOfData[j] == 'A+B' or typesOfData[j] == 'A-B'): data2 = np.fromfile(file2, dtype=np.float64, count=nx * averageConst) if (typesOfData[j] == 'A+B' or typesOfData[j] == 'A-B'): if typesOfData[j] == 'A+B': data = data1 + data2 if typesOfData[j] == 'A-B': data = data1 - data2 else: data = data1 del data1 if (typesOfData[j] == 'A+B' or typesOfData[j] == 'A-B'): del data2 data = np.reshape(data, [nx, averageConst], order='F') dataApp = np.empty((nx, 1), float) if (typesOfData[j] == 'chA' or typesOfData[j] == 'chB' or typesOfData[j] == 'A+B'): # If analyzing intensity - average and log data if averOrMin == 0: with np.errstate(invalid='ignore'): dataApp[:, 0] = 10 * np.log10(data.mean(axis=1)[:]) elif averOrMin == 1: with np.errstate(invalid='ignore'): dataApp[:, 0] = 10 * np.log10(np.amin(data, axis=1)[:]) else: print('\n\n Error!!! Wrong value of parameters!') array = np.append(array, dataApp, axis=1) array[np.isnan(array)] = -120 if (typesOfData[j] == 'A-B'): # If analyzing intensity - average and log absolute values of data with np.errstate(invalid='ignore'): dataApp[:, 0] = 10 * np.log10(np.abs(data.mean(axis=1)[:])) array = np.append(array, dataApp, axis=1) array[np.isnan(array)] = -120 if (typesOfData[j] == 'C_p' or typesOfData[j] == 'CRe' or typesOfData[j] == 'CIm' ): # If analyzing phase/Re/Im - no logarythming needed # If analyzing phase of Re/Im we do not log data, only averaging dataApp[:, 0] = (data.mean(axis=1)[:]) array = np.append(array, dataApp, axis=1) array[np.isnan(array)] = 0 if typesOfData[j] == 'C_m': dataApp[:, 0] = (data.mean(axis=1)[:]) array = np.append(array, dataApp, axis=1) #array[np.isinf(array)] = -120 del dataApp, data file1.close() if (typesOfData[j] == 'A+B' or typesOfData[j] == 'A-B'): file2.close() if print_or_not == 1: print('\n Array shape is now ', array.shape) # *** Cutting timeline to time limits *** dateTimeNew = timeline[istart:istop:averageConst] del dateTimeNew[numOfBlocks:] if print_or_not == 1: print('\n TimeLine length is now: ', len(dateTimeNew)) #******************************************************************************* # F I G U R E S * #******************************************************************************* if print_or_not == 1: print('\n\n\n *** Building images *** \n\n') # Exact string timescales to show on plots TimeScaleFig = np.empty_like(dateTimeNew) for i in range(len(dateTimeNew)): TimeScaleFig[i] = str(dateTimeNew[i][0:11] + '\n' + dateTimeNew[i][11:23]) # Limits of figures for common case or for Re/Im parts to show the interferometric picture if typesOfData[j] == 'CRe' or typesOfData[j] == 'CIm': Vmin = 0 - AmplitudeReIm Vmax = 0 + AmplitudeReIm # *** Immediate spectrum *** Suptitle = ('Immediate spectrum ' + str(df_filename[0:18]) + ' ' + nameAdd) Title = ('Initial parameters: dt = ' + str(round(TimeRes, 3)) + ' Sec, df = ' + str(round(df / 1000, 3)) + ' kHz ' + sumDifMode + 'Processing: Averaging ' + str(averageConst) + ' spectra (' + str(round(averageConst * TimeRes, 3)) + ' sec.)') TwoOrOneValuePlot( 1, frequency, array[:, [1]], [], 'Spectrum', ' ', frequency[0], frequency[FreqPointsNum - 1], Vmin, Vmax, Vmin, Vmax, 'Frequency, MHz', YaxName, ' ', Suptitle, Title, newpath + '/' + fileNameAddSpectr + df_filename[0:14] + '_' + typesOfData[j] + ' Immediate Spectrum.png', currentDate, currentTime, Software_version) # *** Decide to use only list of frequencies or all frequencies in range if ListOrAllFreq == 0: freqList = np.array(freqList) if ListOrAllFreq == 1: freqList = np.array(frequency) # *** Finding frequency most close to specified by user *** for fc in range(len(freqList)): if (freqList[fc] > freqStartTXT) and (freqList[fc] < freqStopTXT): newFreq = np.array(frequency) newFreq = np.absolute(newFreq - freqList[fc]) index = np.argmin(newFreq) + 1 tempArr1 = np.arange(0, len(dateTimeNew), 1) if ChannelSavePNG == 1 or typesOfData[ j] == 'CRe' or typesOfData[j] == 'CIm': if typesOfData[j] == 'CRe' or typesOfData[j] == 'CIm': Vmin = 0 - AmplitudeReIm Vmax = 0 + AmplitudeReIm # *** Plotting intensity changes at particular frequency *** timeline = [] for i in range(len(dateTimeNew)): timeline.append( str(dateTimeNew[i][0:11] + '\n' + dateTimeNew[i][11:23])) Suptitle = 'Intensity variation ' + str( df_filename[0:18]) + ' ' + nameAdd Title = ('Initial parameters: dt = ' + str(round(TimeRes, 3)) + ' Sec, df = ' + str(round(df / 1000, 3)) + ' kHz, Frequency = ' + str(round(frequency[index], 3)) + ' MHz ' + sumDifMode + ' Processing: Averaging ' + str(averageConst) + ' spectra (' + str(round(averageConst * TimeRes, 3)) + ' sec.)') FileName = (newpath + '/' + df_filename[0:14] + '_' + typesOfData[j] + df_filename[-4:] + ' variation at ' + str(round(frequency[index], 3)) + ' MHz.png') OneValueWithTimePlot( timeline, array[[index], :].transpose(), Label, 0, len(dateTimeNew), Vmin, Vmax, 0, 0, 'UTC Date and time, YYYY-MM-DD HH:MM:SS.ms', YaxName, Suptitle, Title, FileName, currentDate, currentTime, Software_version) # *** Saving value changes at particular frequency to TXT file *** if ChannelSaveTXT == 1: SingleChannelData = open( newpath + '/' + df_filename[0:14] + '_' + filename[-7:-4:] + df_filename[-4:] + ' variation at ' + str(round(frequency[index], 3)) + ' MHz.txt', "w") for i in range(len(dateTimeNew)): SingleChannelData.write( str(dateTimeNew[i]).rstrip() + ' ' + str(array.transpose()[i, index]) + ' \n') SingleChannelData.close() # *** Cutting the array inside frequency range specified by user *** if SpecFreqRange == 1 and ( frequency[0] <= freqStart <= frequency[FreqPointsNum - 1] ) and (frequency[0] <= freqStop <= frequency[FreqPointsNum - 1]) and (freqStart < freqStop): print('\n You have chosen the frequency range', freqStart, '-', freqStop, 'MHz') A = [] B = [] for i in range(len(frequency)): A.append(abs(frequency[i] - freqStart)) B.append(abs(frequency[i] - freqStop)) ifmin = A.index(min(A)) ifmax = B.index(min(B)) array = array[ifmin:ifmax, :] print('\n New data array shape is: ', array.shape) freqLine = frequency[ifmin:ifmax] else: freqLine = frequency # Limits of figures for common case or for Re/Im parts to show the interferometric picture Vmin = np.min(array) Vmax = np.max(array) if typesOfData[j] == 'CRe' or typesOfData[j] == 'CIm': Vmin = 0 - AmplitudeReIm Vmax = 0 + AmplitudeReIm # *** Dynamic spectrum of initial signal*** Suptitle = ('Dynamic spectrum starting from file ' + str(df_filename[0:18]) + ' ' + nameAdd + '\n Initial parameters: dt = ' + str(round(TimeRes, 3)) + ' Sec, df = ' + str(round(df / 1000, 3)) + ' kHz, ' + sumDifMode + ' Processing: Averaging ' + str(averageConst) + ' spectra (' + str(round(averageConst * TimeRes, 3)) + ' sec.)\n' + ' Receiver: ' + str(df_system_name) + ', Place: ' + str(df_obs_place) + ', Description: ' + str(df_description)) fig_file_name = (newpath + '/' + fileNameAdd + df_filename[0:14] + '_' + typesOfData[j] + ' Dynamic spectrum.png') OneDynSpectraPlot(array, Vmin, Vmax, Suptitle, 'Intensity, dB', len(dateTimeNew), TimeScaleFig, freqLine, len(freqLine), colormap, 'UTC Date and time, YYYY-MM-DD HH:MM:SS.msec', fig_file_name, currentDate, currentTime, Software_version, customDPI) if (typesOfData[j] != 'C_p' and typesOfData[j] != 'CRe' and typesOfData[j] != 'CIm'): # *** Normalization and cleaning of dynamic spectra *** Normalization_dB(array.transpose(), len(freqLine), len(dateTimeNew)) simple_channel_clean(array.transpose(), RFImeanConst) # *** Dynamic spectra of cleaned and normalized signal *** Suptitle = ( 'Dynamic spectrum cleaned and normalized starting from file ' + str(df_filename[0:18]) + ' ' + nameAdd + '\n Initial parameters: dt = ' + str(round(TimeRes, 3)) + ' Sec, df = ' + str(round(df / 1000, 3)) + ' kHz, ' + sumDifMode + ' Processing: Averaging ' + str(averageConst) + ' spectra (' + str(round(averageConst * TimeRes, 3)) + ' sec.)\n' + ' Receiver: ' + str(df_system_name) + ', Place: ' + str(df_obs_place) + ', Description: ' + str(df_description)) fig_file_name = (newpath + '/' + fileNameAddNorm + df_filename[0:14] + '_' + typesOfData[j] + ' Dynamic spectrum cleanned and normalized' + '.png') OneDynSpectraPlot(array, VminNorm, VmaxNorm, Suptitle, 'Intensity, dB', len(dateTimeNew), TimeScaleFig, freqLine, len(freqLine), colormap, 'UTC Date and time, YYYY-MM-DD HH:MM:SS.msec', fig_file_name, currentDate, currentTime, Software_version, customDPI) ''' # *** TEMPLATE FOR JOURNLS Dynamic spectra of cleaned and normalized signal *** plt.figure(2, figsize=(16.0, 7.0)) ImA = plt.imshow(np.flipud(array), aspect='auto', extent=[0,len(dateTimeNew),freqLine[0],freqLine[len(freqLine)-1]], vmin=VminNorm, vmax=VmaxNorm, cmap=colormap) # plt.ylabel('Frequency, MHz', fontsize=12, fontweight='bold') #plt.suptitle('Dynamic spectrum cleaned and normalized starting from file '+str(df_filename[0:18])+' '+nameAdd+ # '\n Initial parameters: dt = '+str(round(TimeRes,3))+ # ' Sec, df = '+str(round(df/1000,3))+' kHz, '+sumDifMode+ # ' Processing: Averaging '+str(averageConst)+' spectra ('+str(round(averageConst*TimeRes,3))+' sec.)\n'+ # ' Receiver: '+str(df_system_name)+ # ', Place: '+str(df_obs_place) + # ', Description: '+str(df_description), # fontsize=10, fontweight='bold', x = 0.46, y = 0.96) plt.yticks(fontsize=12, fontweight='bold') rc('font', weight='bold') cbar = plt.colorbar(ImA, pad=0.005) cbar.set_label('Intensity, dB', fontsize=12, fontweight='bold') cbar.ax.tick_params(labelsize=12) ax1 = plt.figure(2).add_subplot(1,1,1) a = ax1.get_xticks().tolist() for i in range(len(a)-1): #a-1 k = int(a[i]) #a[i] = str(dateTimeNew[k][0:11]+'\n'+dateTimeNew[k][11:23]) a[i] = str(dateTimeNew[k][11:19]) ax1.set_xticklabels(a) plt.xticks(fontsize=12, fontweight='bold') plt.xlabel('UTC time, HH:MM:SS', fontsize=12, fontweight='bold') #plt.text(0.72, 0.04,'Processed '+currentDate+ ' at '+currentTime, fontsize=6, transform=plt.gcf().transFigure) pylab.savefig('DAT_Results/' + fileNameAddNorm + df_filename[0:14]+'_'+typesOfData[j]+' Dynamic spectrum cleanned and normalized'+'.png', bbox_inches='tight', dpi = customDPI) #pylab.savefig('DAT_Results/' +fileNameAddNorm+ df_filename[0:14]+'_'+typesOfData[j]+ ' Dynamic spectrum cleanned and normalized'+'.eps', bbox_inches='tight', dpi = customDPI) #filename[-7:-4:] plt.close('all') ''' ok = 1 return ok
def convert_one_jds_wf_to_wf32(source_file, result_directory, no_of_bunches_per_file): """ function converts jds waveform data to wf32 waveform data for further processing (coherent dedispersion) and saves txt files with time data Input parameters: source_directory - directory where initial jds waveform data are stored result_directory - directory where new wf32 files will be stored no_of_bunches_per_file - number of data bunches per file to peocess (depends on RAM volume on the PC) Output parameters: result_wf32_files - list of results files """ # *** Data file header read *** [ df_filename, df_filesize, df_system_name, df_obs_place, df_description, clock_freq, df_creation_timeUTC, channel, receiver_mode, Mode, Navr, time_res, fmin, fmax, df, frequency, freq_points_num, data_block_size ] = FileHeaderReaderJDS(source_file, 0, 0) if Mode > 0: sys.exit( ' ERROR!!! Data recorded in wrong mode! Waveform mode needed.\n\n Program stopped!' ) result_wf32_files = [] fname = source_file # Create long data files and copy first data file header to them with open(fname, 'rb') as file: # *** Data file header read *** file_header = file.read(1024) # *** Creating a name for long timeline TXT file *** tl_file_name = df_filename + '_Timeline.wtxt' tl_file = open(result_directory + tl_file_name, 'w') # Open and close to delete the file with the same name tl_file.close() # *** Creating a binary file with data for long data storage *** file_data_A_name = result_directory + df_filename + '_Data_chA.wf32' result_wf32_files.append(file_data_A_name) file_data_A = open(file_data_A_name, 'wb') file_data_A.write(file_header) file_data_A.close() if channel == 2: file_data_B_name = result_directory + df_filename + '_Data_chB.wf32' result_wf32_files.append(file_data_B_name) file_data_B = open(file_data_B_name, 'wb') file_data_B.write(file_header) file_data_B.close() del file_header # Calculation of number of blocks and number of spectra in the file if channel == 0 or channel == 1: # Single channel mode no_of_spectra_in_bunch = int( (df_filesize - 1024) / (no_of_bunches_per_file * 2 * data_block_size)) else: # Two channels mode no_of_spectra_in_bunch = int( (df_filesize - 1024) / (no_of_bunches_per_file * 4 * data_block_size)) no_of_blocks_in_file = (df_filesize - 1024) / data_block_size print(' Number of blocks in file: ', no_of_blocks_in_file) print(' Number of bunches to read in file: ', no_of_bunches_per_file, '\n') # ******************************************************************************* # R E A D I N G D A T A * # ******************************************************************************* with open(fname, 'rb') as file: file.seek(1024) # Jumping to 1024 byte from file beginning # !!! Fake timing. Real timing to be done!!! TimeFigureScaleFig = np.linspace(0, no_of_bunches_per_file, no_of_bunches_per_file + 1) for i in range(no_of_bunches_per_file): TimeFigureScaleFig[i] = str(TimeFigureScaleFig[i]) time_scale_bunch = [] bar = IncrementalBar(' File reading: ', max=no_of_bunches_per_file, suffix='%(percent)d%%') for bunch in range(no_of_bunches_per_file): bar.next() # Reading and reshaping all data with time data if channel == 0 or channel == 1: # Single channel mode wf_data = np.fromfile(file, dtype='i2', count=no_of_spectra_in_bunch * data_block_size) wf_data = np.reshape(wf_data, [data_block_size, no_of_spectra_in_bunch], order='F') if channel == 2: # Two channels mode wf_data = np.fromfile(file, dtype='i2', count=2 * no_of_spectra_in_bunch * data_block_size) wf_data = np.reshape( wf_data, [data_block_size, 2 * no_of_spectra_in_bunch], order='F') # Timing timeline_block_str = jds_waveform_time(wf_data, clock_freq, data_block_size) if channel == 2: # Two channels mode timeline_block_str = timeline_block_str[0:int( len(timeline_block_str) / 2)] # Cut the timeline of second channel for i in range(len(timeline_block_str)): time_scale_bunch.append(df_creation_timeUTC[0:10] + ' ' + timeline_block_str[i]) # [0:12] # Deleting the time blocks from waveform data real_data_block_size = data_block_size - 4 wf_data = wf_data[0:real_data_block_size, :] # Separation data into channels if channel == 0 or channel == 1: # Single channel mode wf_data_chA = np.reshape( wf_data, [real_data_block_size * no_of_spectra_in_bunch, 1], order='F') del wf_data # Deleting unnecessary array name just in case if channel == 2: # Two channels mode # Separating the data into two channels wf_data = np.reshape( wf_data, [2 * real_data_block_size * no_of_spectra_in_bunch, 1], order='F') wf_data_chA = wf_data[0:(2 * real_data_block_size * no_of_spectra_in_bunch):2] # A wf_data_chB = wf_data[1:(2 * real_data_block_size * no_of_spectra_in_bunch):2] # B del wf_data # Saving WF data to dat file file_data_A = open(file_data_A_name, 'ab') file_data_A.write( np.float32(wf_data_chA).transpose().copy(order='C')) file_data_A.close() if channel == 2: file_data_B = open(file_data_B_name, 'ab') file_data_B.write( np.float32(wf_data_chB).transpose().copy(order='C')) file_data_B.close() # Saving time data to ling timeline file with open(tl_file_name, 'a') as tl_file: for i in range(no_of_spectra_in_bunch): tl_file.write((str(time_scale_bunch[i][:])) + ' \n') # str bar.finish() file.close() # Close the data file del file_data_A if channel == 2: del file_data_B return result_wf32_files
file = open(data_filename, 'rb') # reading FHEADER df_filesize = (os.stat(data_filename).st_size) # Size of file df_filename = file.read(32).decode('utf-8').rstrip('\x00') # Initial data file name file.close() receiver_type = df_filename[-4:] # Reading file header to obtain main parameters of the file if receiver_type == '.adr': [TimeRes, fmin, fmax, df, frequency_list, FFTsize] = FileHeaderReaderADR(data_filename, 0) if receiver_type == '.jds': [df_filename, df_filesize, df_system_name, df_obs_place, df_description, CLCfrq, df_creation_timeUTC, SpInFile, ReceiverMode, Mode, Navr, TimeRes, fmin, fmax, df, frequency_list, FFTsize, dataBlockSize] = FileHeaderReaderJDS(data_filename, 0, 1) #************************************************************************************ # R E A D I N G D A T A * #************************************************************************************ if receiver_type == '.jds': num_frequencies = len(frequency_list)-4 shift_vector = DM_full_shift_calc(len(frequency_list), fmin, fmax, df / pow(10,6), TimeRes, DM, receiver_type) #plot1D(shift_vector, newpath+'/01 - Shift parameter.png', 'Shift parameter', 'Shift parameter', 'Shift parameter', 'Frequency channel number', customDPI) max_shift = np.abs(shift_vector[0]) print (' Maximal shift is: ', max_shift, ' pixels \n')
def jds_wf_simple_reader(directory, no_of_spectra_to_average, skip_data_blocks, VminNorm, VmaxNorm, colormap, custom_dpi, save_long_file_aver, dyn_spectr_save_init, dyn_spectr_save_norm): current_time = time.strftime("%H:%M:%S") current_date = time.strftime("%d.%m.%Y") # *** Creating a folder where all pictures and results will be stored (if it doesn't exist) *** result_folder = 'RESULTS_JDS_waveform_' + directory.split('/')[-2] if not os.path.exists(result_folder): os.makedirs(result_folder) service_folder = result_folder + '/Service' if not os.path.exists(service_folder): os.makedirs(service_folder) if dyn_spectr_save_init == 1: initial_spectra_folder = result_folder + '/Initial spectra' if not os.path.exists(initial_spectra_folder): os.makedirs(initial_spectra_folder) # *** Search JDS files in the directory *** file_list = find_files_only_in_current_folder(directory, '.jds', 1) print('') if len( file_list ) > 1: # Check if files have same parameters if there are more then one file in list # Check if all files (except the last) have same size same_or_not = check_if_all_files_of_same_size(directory, file_list, 1) # Check if all files in this folder have the same parameters in headers equal_or_not = check_if_JDS_files_of_equal_parameters( directory, file_list) if same_or_not and equal_or_not: print( '\n\n\n :-) All files seem to be of the same parameters! :-) \n\n\n' ) else: print( '\n\n\n ************************************************************************************* ' ) print( ' * *' ) print( ' * Seems files in folders are different check the errors and restart the script! *' ) print( ' * * ' '\n ************************************************************************************* \n\n\n' ) decision = int( input( '* Enter "1" to start processing, or "0" to stop the script: ' )) if decision != 1: sys.exit( '\n\n\n *** Program stopped! *** \n\n\n') # To print in console the header of first file print('\n First file header parameters: \n') # *** Data file header read *** [ df_filename, df_filesize, df_system_name, df_obs_place, df_description, CLCfrq, df_creation_timeUTC, Channel, ReceiverMode, Mode, Navr, TimeRes, fmin, fmax, df, frequency, freq_points_num, data_block_size ] = FileHeaderReaderJDS(directory + file_list[0], 0, 1) # Main loop by files start for file_no in range(len(file_list)): # loop by files # *** Opening datafile *** fname = directory + file_list[file_no] # ********************************************************************************* # *** Data file header read *** [ df_filename, df_filesize, df_system_name, df_obs_place, df_description, CLCfrq, df_creation_timeUTC, Channel, ReceiverMode, Mode, Navr, TimeRes, fmin, fmax, df, frequency, freq_points_num, data_block_size ] = FileHeaderReaderJDS(fname, 0, 0) # Create long data files and copy first data file header to them if file_no == 0 and save_long_file_aver == 1: with open(fname, 'rb') as file: # *** Data file header read *** file_header = file.read(1024) # *** Creating a name for long timeline TXT file *** tl_file_name = df_filename + '_Timeline.txt' tl_file = open( tl_file_name, 'w') # Open and close to delete the file with the same name tl_file.close() # *** Creating a binary file with data for long data storage *** file_data_a_name = df_filename + '_Data_chA.dat' file_data_a = open(file_data_a_name, 'wb') file_data_a.write(file_header) file_data_a.seek(574) # FFT size place in header file_data_a.write(np.int32(data_block_size).tobytes()) file_data_a.seek(624) # Lb place in header file_data_a.write(np.int32(0).tobytes()) file_data_a.seek(628) # Hb place in header file_data_a.write(np.int32(data_block_size / 2).tobytes()) file_data_a.seek(632) # Wb place in header file_data_a.write(np.int32(data_block_size / 2).tobytes()) file_data_a.seek(636) # Navr place in header file_data_a.write( bytes([np.int32(Navr * no_of_spectra_to_average)])) file_data_a.close() if Channel == 2: file_data_b_name = df_filename + '_Data_chB.dat' file_data_b = open(file_data_b_name, 'wb') file_data_b.write(file_header) file_data_b.seek(574) # FFT size place in header file_data_b.write(np.int32(data_block_size).tobytes()) file_data_b.seek(624) # Lb place in header file_data_b.write(np.int32(0).tobytes()) file_data_b.seek(628) # Hb place in header file_data_b.write(np.int32(data_block_size / 2).tobytes()) file_data_b.seek(632) # Wb place in header file_data_b.write(np.int32(data_block_size / 2).tobytes()) file_data_b.seek(636) # Navr place in header file_data_b.write( bytes([np.int32(Navr * no_of_spectra_to_average)])) file_data_b.close() del file_header # !!! Make automatic calculations of time and frequency resolutions for waveform mode!!! # Manually set frequencies for one channel mode if (Channel == 0 and int(CLCfrq / 1000000) == 66) or (Channel == 1 and int(CLCfrq / 1000000) == 66): freq_points_num = 8192 frequency = np.linspace(0.0, 33.0, freq_points_num) # Manually set frequencies for two channels mode if Channel == 2 or (Channel == 0 and int(CLCfrq / 1000000) == 33) or ( Channel == 1 and int(CLCfrq / 1000000) == 33): freq_points_num = 8192 frequency = np.linspace(16.5, 33.0, freq_points_num) # For new receiver (temporary): if Channel == 2 and int(CLCfrq / 1000000) == 80: freq_points_num = 8192 frequency = np.linspace(0.0, 40.0, freq_points_num) # Calculation of number of blocks and number of spectra in the file if Channel == 0 or Channel == 1: # Single channel mode no_of_av_spectra_per_file = (df_filesize - 1024) / ( 2 * data_block_size * no_of_spectra_to_average) else: # Two channels mode no_of_av_spectra_per_file = (df_filesize - 1024) / ( 4 * data_block_size * no_of_spectra_to_average) no_of_blocks_in_file = (df_filesize - 1024) / data_block_size no_of_av_spectra_per_file = int(no_of_av_spectra_per_file) fine_clock_frq = (int(CLCfrq / 1000000.0) * 1000000.0) # Real time resolution of averaged spectra real_av_spectra_dt = (1 / fine_clock_frq) * ( data_block_size - 4) * no_of_spectra_to_average if file_no == 0: print(' Number of blocks in file: ', no_of_blocks_in_file) print(' Number of spectra to average: ', no_of_spectra_to_average) print(' Number of averaged spectra in file: ', no_of_av_spectra_per_file) print(' Time resolution of averaged spectrum: ', round(real_av_spectra_dt * 1000, 3), ' ms.') print('\n *** Reading data from file *** \n') # ******************************************************************************* # R E A D I N G D A T A * # ******************************************************************************* with open(fname, 'rb') as file: file.seek( 1024 + data_block_size * 4 * skip_data_blocks) # Jumping to 1024 byte from file beginning # *** DATA READING process *** # Preparing arrays for dynamic spectra dyn_spectra_ch_a = np.zeros( (int(data_block_size / 2), no_of_av_spectra_per_file), float) if Channel == 2: # Two channels mode dyn_spectra_ch_b = np.zeros( (int(data_block_size / 2), no_of_av_spectra_per_file), float) # !!! Fake timing. Real timing to be done!!! # TimeFigureScaleFig = np.linspace(0, no_of_av_spectra_per_file, no_of_av_spectra_per_file+1) # for i in range(no_of_av_spectra_per_file): # TimeFigureScaleFig[i] = str(TimeFigureScaleFig[i]) time_scale_fig = [] time_scale_full = [] bar = IncrementalBar(' File ' + str(file_no + 1) + ' of ' + str(len(file_list)) + ' reading: ', max=no_of_av_spectra_per_file, suffix='%(percent)d%%') for av_sp in range(no_of_av_spectra_per_file): bar.next() # Reading and reshaping all data with readers if Channel == 0 or Channel == 1: # Single channel mode wf_data = np.fromfile(file, dtype='i2', count=no_of_spectra_to_average * data_block_size) wf_data = np.reshape( wf_data, [data_block_size, no_of_spectra_to_average], order='F') if Channel == 2: # Two channels mode wf_data = np.fromfile(file, dtype='i2', count=2 * no_of_spectra_to_average * data_block_size) wf_data = np.reshape( wf_data, [data_block_size, 2 * no_of_spectra_to_average], order='F') # Timing timeline_block_str = jds_waveform_time(wf_data, CLCfrq, data_block_size) time_scale_fig.append(timeline_block_str[-1][0:12]) time_scale_full.append(df_creation_timeUTC[0:10] + ' ' + timeline_block_str[-1][0:12]) # Nulling the time blocks in waveform data wf_data[data_block_size - 4:data_block_size, :] = 0 # Scaling of the data - seems to be wrong in absolute value wf_data = wf_data / 32768.0 if Channel == 0 or Channel == 1: # Single channel mode wf_data_ch_a = wf_data # All the data is channel A data del wf_data # Deleting unnecessary array to free the memory if Channel == 2: # Two channels mode # Resizing to obtain the matrix for separation of channels wf_data_new = np.zeros( (2 * data_block_size, no_of_spectra_to_average)) for i in range(2 * no_of_spectra_to_average): if i % 2 == 0: wf_data_new[0:data_block_size, int(i / 2)] = wf_data[:, i] # Even else: wf_data_new[data_block_size:2 * data_block_size, int(i / 2)] = wf_data[:, i] # Odd del wf_data # Deleting unnecessary array to free the memory # Separating the data into two channels wf_data_ch_a = np.zeros( (data_block_size, no_of_spectra_to_average)) # Preparing empty array wf_data_ch_b = np.zeros( (data_block_size, no_of_spectra_to_average)) # Preparing empty array wf_data_ch_a[:, :] = wf_data_new[0:( 2 * data_block_size):2, :] # Separation to channel A wf_data_ch_b[:, :] = wf_data_new[1:( 2 * data_block_size):2, :] # Separation to channel B del wf_data_new # preparing matrices for spectra spectra_ch_a = np.zeros_like(wf_data_ch_a) if Channel == 2: spectra_ch_b = np.zeros_like(wf_data_ch_b) # Calculation of spectra for i in range(no_of_spectra_to_average): spectra_ch_a[:, i] = np.power( np.abs(np.fft.fft(wf_data_ch_a[:, i])), 2) if Channel == 2: # Two channels mode spectra_ch_b[:, i] = np.power( np.abs(np.fft.fft(wf_data_ch_b[:, i])), 2) # Storing only first (left) mirror part of spectra spectra_ch_a = spectra_ch_a[:int(data_block_size / 2), :] if Channel == 2: spectra_ch_b = spectra_ch_b[:int(data_block_size / 2), :] # At 33 MHz the specter is usually upside down, to correct it we use flip up/down if int(CLCfrq / 1000000) == 33: spectra_ch_a = np.flipud(spectra_ch_a) if Channel == 2: spectra_ch_b = np.flipud(spectra_ch_b) # Plotting first waveform block and first immediate spectrum in a file if av_sp == 0: # First data block in a file i = 0 # First immediate spectrum in a block # Prepare parameters for plot data_1 = wf_data_ch_a[:, i] if Channel == 0 or Channel == 1: # Single channel mode no_of_sets = 1 data_2 = [] if Channel == 2: no_of_sets = 2 data_2 = wf_data_ch_b[:, i] suptitle = ('Waveform data, first block in file ' + str(df_filename)) Title = (ReceiverMode + ', Fclock = ' + str(round(CLCfrq / 1000000, 1)) + ' MHz, Description: ' + str(df_description)) TwoOrOneValuePlot( no_of_sets, np.linspace(no_of_sets, data_block_size, data_block_size), data_1, data_2, 'Channel A', 'Channel B', 1, data_block_size, -0.6, 0.6, -0.6, 0.6, 'ADC clock counts', 'Amplitude, V', 'Amplitude, V', suptitle, Title, service_folder + '/' + df_filename[0:14] + ' Waveform first data block.png', current_date, current_time, software_version) # Prepare parameters for plot data_1 = 10 * np.log10(spectra_ch_a[:, i]) if Channel == 0 or Channel == 1: # Single channel mode no_of_sets = 1 data_2 = [] if Channel == 2: no_of_sets = 2 data_2 = 10 * np.log10(spectra_ch_b[:, i]) suptitle = ('Immediate spectrum, first in file ' + str(df_filename)) Title = (ReceiverMode + ', Fclock = ' + str(round(CLCfrq / 1000000, 1)) + ' MHz, Description: ' + str(df_description)) TwoOrOneValuePlot( no_of_sets, frequency, data_1, data_2, 'Channel A', 'Channel B', frequency[0], frequency[-1], -80, 60, -80, 60, 'Frequency, MHz', 'Intensity, dB', 'Intensity, dB', suptitle, Title, service_folder + '/' + df_filename[0:14] + ' Immediate spectrum first in file.png', current_date, current_time, software_version) # Deleting the unnecessary matrices del wf_data_ch_a if Channel == 2: del wf_data_ch_b # Calculation the averaged spectrum aver_spectra_ch_a = spectra_ch_a.mean(axis=1)[:] if Channel == 2: aver_spectra_ch_b = spectra_ch_b.mean(axis=1)[:] # Plotting only first averaged spectrum if av_sp == 0: # Prepare parameters for plot data_1 = 10 * np.log10(aver_spectra_ch_a) if Channel == 0 or Channel == 1: # Single channel mode no_of_sets = 1 data_2 = [] if Channel == 2: no_of_sets = 2 data_2 = 10 * np.log10(aver_spectra_ch_b) suptitle = ('Average spectrum, first data block in file ' + str(df_filename)) Title = (ReceiverMode + ', Fclock = ' + str(round(CLCfrq / 1000000, 1)) + ' MHz, Avergaed spectra: ' + str(no_of_spectra_to_average) + ', Description: ' + str(df_description)) TwoOrOneValuePlot( no_of_sets, frequency, data_1, data_2, 'Channel A', 'Channel B', frequency[0], frequency[-1], -80, 60, -80, 60, 'Frequency, MHz', 'Intensity, dB', 'Intensity, dB', suptitle, Title, service_folder + '/' + df_filename[0:14] + ' Average spectrum first data block in file.png', current_date, current_time, software_version) # Adding calculated averaged spectrum to dynamic spectra array dyn_spectra_ch_a[:, av_sp] = aver_spectra_ch_a[:] if Channel == 2: dyn_spectra_ch_b[:, av_sp] = aver_spectra_ch_b[:] bar.finish() # file.close() # Close the data file # Saving averaged spectra to long data files if save_long_file_aver == 1: temp = dyn_spectra_ch_a.transpose().copy(order='C') file_data_a = open(file_data_a_name, 'ab') file_data_a.write(temp) file_data_a.close() if Channel == 2: temp = dyn_spectra_ch_b.transpose().copy(order='C') file_data_b = open(file_data_b_name, 'ab') file_data_b.write(temp) file_data_b.close() # Saving time data to ling timeline file with open(tl_file_name, 'a') as tl_file: for i in range(no_of_av_spectra_per_file): tl_file.write((time_scale_full[i][:]) + ' \n') # str del time_scale_full # Log data (make dB scale) with np.errstate(invalid='ignore', divide='ignore'): dyn_spectra_ch_a = 10 * np.log10(dyn_spectra_ch_a) if Channel == 2: dyn_spectra_ch_b = 10 * np.log10(dyn_spectra_ch_b) # If the data contains minus infinity values change them to particular values dyn_spectra_ch_a[np.isinf(dyn_spectra_ch_a)] = 40 if Channel == 2: dyn_spectra_ch_b[np.isinf(dyn_spectra_ch_b)] = 40 # ******************************************************************************* # P L O T T I N G D Y N A M I C S P E C T R A * # ******************************************************************************* # if dyn_spectr_save_init == 1 or dyn_spectr_save_norm == 1: # print('\n *** Making figures of dynamic spectra *** \n') if dyn_spectr_save_init == 1: # Plot of initial dynamic spectra v_min_a = np.min(dyn_spectra_ch_a) v_max_a = np.max(dyn_spectra_ch_a) v_min_b = v_min_a v_max_b = v_max_a if Channel == 2: v_min_b = np.min(dyn_spectra_ch_b) v_max_b = np.max(dyn_spectra_ch_b) if Channel == 0 or Channel == 1: # Single channel mode dyn_spectra_ch_b = dyn_spectra_ch_a suptitle = ('Dynamic spectrum (initial) ' + str(df_filename) + ' - Fig. ' + str(1) + ' of ' + str(1) + '\n Initial parameters: dt = ' + str(round(TimeRes * 1000., 3)) + ' ms, df = ' + str(round(df / 1000., 3)) + ' kHz, Receiver: ' + str(df_system_name) + ', Place: ' + str(df_obs_place) + '\n' + ReceiverMode + ', Fclock = ' + str(round(CLCfrq / 1000000, 1)) + ' MHz, Avergaed spectra: ' + str(no_of_spectra_to_average) + ' (' + str(round(no_of_spectra_to_average * TimeRes, 3)) + ' sec.), Description: ' + str(df_description)) fig_file_name = (initial_spectra_folder + '/' + df_filename[0:14] + ' Initial dynamic spectrum fig.' + str(0 + 1) + '.png') if Channel == 0 or Channel == 1: # Single channel mode OneDynSpectraPlot(dyn_spectra_ch_a, v_min_a, v_max_a, suptitle, 'Intensity, dB', no_of_av_spectra_per_file, time_scale_fig, frequency, freq_points_num, colormap, 'UTC Time, HH:MM:SS.msec', fig_file_name, current_date, current_time, software_version, custom_dpi) if Channel == 2: TwoDynSpectraPlot(dyn_spectra_ch_a, dyn_spectra_ch_b, v_min_a, v_max_a, v_min_b, v_max_b, suptitle, 'Intensity, dB', 'Intensity, dB', no_of_av_spectra_per_file, time_scale_fig, time_scale_fig, frequency, freq_points_num, colormap, 'Channel A', 'Channel B', fig_file_name, current_date, current_time, software_version, custom_dpi) if dyn_spectr_save_norm == 1: # Normalization and cleaning of data Normalization_dB(dyn_spectra_ch_a.transpose(), freq_points_num, no_of_av_spectra_per_file) if Channel == 2: Normalization_dB(dyn_spectra_ch_b.transpose(), freq_points_num, no_of_av_spectra_per_file) simple_channel_clean(dyn_spectra_ch_a, 8) if Channel == 2: simple_channel_clean(dyn_spectra_ch_b, 8) # Plot of normalized and cleaned dynamic spectra suptitle = ('Normalized and cleaned dynamic spectrum (initial) ' + str(df_filename) + ' - Fig. ' + str(0 + 1) + ' of ' + str(1) + '\n Initial parameters: dt = ' + str(round(TimeRes * 1000, 3)) + ' ms, df = ' + str(round(df / 1000., 3)) + ' kHz, Receiver: ' + str(df_system_name) + ', Place: ' + str(df_obs_place) + '\n' + ReceiverMode + ', Fclock = ' + str(round(CLCfrq / 1000000, 1)) + ' MHz, Avergaed spectra: ' + str(no_of_spectra_to_average) + ' (' + str(round(no_of_spectra_to_average * TimeRes, 3)) + ' sec.), Description: ' + str(df_description)) fig_file_name = (result_folder + '/' + df_filename[0:14] + ' Normalized and cleaned dynamic spectrum fig.' + str(0 + 1) + '.png') if Channel == 0 or Channel == 1: # Single channel mode OneDynSpectraPlot(dyn_spectra_ch_a, VminNorm, VmaxNorm, suptitle, 'Intensity, dB', no_of_av_spectra_per_file, time_scale_fig, frequency, freq_points_num, colormap, 'UTC Time, HH:MM:SS.msec', fig_file_name, current_date, current_time, software_version, custom_dpi) if Channel == 2: TwoDynSpectraPlot(dyn_spectra_ch_a, dyn_spectra_ch_b, VminNorm, VmaxNorm, VminNorm, VmaxNorm, suptitle, 'Intensity, dB', 'Intensity, dB', no_of_av_spectra_per_file, time_scale_fig, time_scale_fig, frequency, freq_points_num, colormap, 'Channel A', 'Channel B', fig_file_name, current_date, current_time, software_version, custom_dpi) del time_scale_fig, file_data_a if Channel == 2: del file_data_b results_files_list = [] results_files_list.append(file_data_a_name) if Channel == 2: results_files_list.append(file_data_b_name) return results_files_list
def cut_needed_pulsar_period_from_dat_to_dat(common_path, filename, pulsar_name, period_number, profile_pic_min, profile_pic_max, spectrum_pic_min, spectrum_pic_max, periods_per_fig, customDPI, colormap): """ Function to find and cut the selected pulsar period (by its number) from the DAT files """ software_version = '2021.08.07' current_time = time.strftime("%H:%M:%S") current_date = time.strftime("%d.%m.%Y") # Creating a folder where all pictures and results will be stored (if it doesn't exist) result_path = "RESULTS_pulsar_extracted_pulse_" + filename if not os.path.exists(result_path): os.makedirs(result_path) # Taking pulsar period from catalogue pulsar_ra, pulsar_dec, DM, p_bar = catalogue_pulsar(pulsar_name) # DAT file to be analyzed: filepath = common_path + filename # Timeline file to be analyzed: timeline_filepath = common_path + filename.split( '_Data_')[0] + '_Timeline.txt' # Opening DAT datafile file = open(filepath, 'rb') # Data file header read df_filesize = os.stat(filepath).st_size # Size of file df_filepath = file.read(32).decode('utf-8').rstrip( '\x00') # Initial data file name file.close() if df_filepath[-4:] == '.adr': [ df_filepath, df_filesize, df_system_name, df_obs_place, df_description, CLCfrq, df_creation_timeUTC, ReceiverMode, Mode, sumDifMode, NAvr, time_resolution, fmin, fmax, df, frequency, FFTsize, SLine, Width, BlockSize ] = FileHeaderReaderADR(filepath, 0, 0) freq_points_num = len(frequency) if df_filepath[-4:] == '.jds': # If data obtained from DSPZ receiver [ df_filepath, df_filesize, df_system_name, df_obs_place, df_description, CLCfrq, df_creation_timeUTC, SpInFile, ReceiverMode, Mode, Navr, time_resolution, fmin, fmax, df, frequency, freq_points_num, dataBlockSize ] = FileHeaderReaderJDS(filepath, 0, 0) # ************************************************************************************ # R E A D I N G D A T A * # ************************************************************************************ # Time line file reading timeline, dt_timeline = time_line_file_reader(timeline_filepath) # Calculation of the dimensions of arrays to read taking into account the pulsar period spectra_in_file = int( (df_filesize - 1024) / (8 * freq_points_num)) # int(df_filesize - 1024)/(2*4*freq_points_num) spectra_to_read = int( np.round((periods_per_fig * p_bar / time_resolution), 0)) spectra_per_period = int(np.round((p_bar / time_resolution), 0)) num_of_blocks = int(np.floor(spectra_in_file / spectra_to_read)) print('\n Pulsar name: ', pulsar_name, '') print(' Pulsar period: ', p_bar, 's.') print(' Time resolution: ', time_resolution, 's.') print(' Number of spectra to read in', periods_per_fig, 'periods: ', spectra_to_read, ' ') print(' Number of spectra in file: ', spectra_in_file, ' ') print(' Number of', periods_per_fig, 'periods blocks in file: ', num_of_blocks, '\n') # Data reading and making figures print('\n Data reading and making figure...') data_file = open(filepath, 'rb') # Jumping to 1024+number of spectra to skip bytes from file beginning data_file.seek( 1024 + (period_number - 1) * spectra_per_period * len(frequency) * 8, os.SEEK_SET) # Reading and preparing block of data (3 periods) data = np.fromfile(data_file, dtype=np.float64, count=spectra_to_read * len(frequency)) data_file.close() data = np.reshape(data, [len(frequency), spectra_to_read], order='F') # Read data file header from initial file with open(filepath, 'rb') as file: file_header = file.read(1024) # Create binary file with the header and pulsar one or two periods data dat_file_name = 'Single_pulse_' + filename file_data = open(result_path + '/' + dat_file_name, 'wb') file_data.write(file_header) del file_header # Prepare data to save to the file temp = data.transpose().copy(order='C') file_data.write(np.float64(temp)) del temp file_data.close() # Time line fig_time_scale = timeline[(period_number - 1) * spectra_per_period:(period_number - 1 + spectra_to_read) * spectra_per_period] # Prepared code to save timeline to a file, but as the timeline is wrong comented them temporarily # # Creating and filling a new timeline TXT file for results # new_tl_file_name = dat_file_name.split('_Data_', 1)[0] + '_Timeline.txt' # new_tl_file = open(result_path + '/' + new_tl_file_name, 'w') # # Saving time data to new file # for j in range(len(fig_time_scale)): # new_tl_file.write((fig_time_scale[j][:]) + '') # new_tl_file.close() # Logging data for figure data = 10 * np.log10(data) # Normalizing data data = data - np.mean(data) # Making result picture fig = plt.figure(figsize=(9.2, 4.5)) rc('font', size=5, weight='bold') ax2 = fig.add_subplot(111) ax2.set_title('File: ' + filename + ' Description: ' + df_description + ' Resolution: ' + str(np.round(df / 1000, 3)) + ' kHz and ' + str(np.round(time_resolution * 1000, 3)) + ' ms.', fontsize=5, fontweight='bold') ax2.imshow( np.flipud(data), aspect='auto', cmap=colormap, vmin=spectrum_pic_min, vmax=spectrum_pic_max, extent=[0, data.shape[1], frequency[0] + 16.5, frequency[-1] + 16.5]) # len(profile) ax2.set_xlabel('Time UTC (at the lowest frequency), HH:MM:SS.ms', fontsize=6, fontweight='bold') ax2.set_ylabel('Frequency, MHz', fontsize=6, fontweight='bold') text = ax2.get_xticks().tolist() for i in range(len(text) - 1): k = int(text[i]) text[i] = fig_time_scale[k][11:23] ax2.set_xticklabels(text, fontsize=5, fontweight='bold') fig.subplots_adjust(hspace=0.05, top=0.91) fig.suptitle('Extracted single pulse of ' + pulsar_name + ' (DM: ' + str(DM) + r' $\mathrm{pc \cdot cm^{-3}}$' + ', Period: ' + str(p_bar) + ' s.)', fontsize=7, fontweight='bold') fig.text(0.80, 0.04, 'Processed ' + current_date + ' at ' + current_time, fontsize=3, transform=plt.gcf().transFigure) fig.text(0.09, 0.04, 'Software version: ' + software_version + ', [email protected], IRA NASU', fontsize=3, transform=plt.gcf().transFigure) pylab.savefig(result_path + '/' + 'Single_pulse_' + filename[:-4] + '.png', bbox_inches='tight', dpi=customDPI) plt.close('all') return result_path, 'Single_pulse_' + filename, filename + 'Single_pulse_' + filename[: -4] + '.png'
decision = int( input( '* Enter "1" to start processing, or "0" to stop the script: ') ) if decision != 1: sys.exit('\n\n\n *** Program stopped! *** \n\n\n') # To print in console the header of first file print('\n First file header parameters: \n') # *** Data file header read *** [ df_filename, df_filesize, df_system_name, df_obs_place, df_description, CLCfrq, df_creation_timeUTC, Channel, ReceiverMode, Mode, Navr, TimeRes, fmin, fmax, df, frequency, FreqPointsNum, data_block_size ] = FileHeaderReaderJDS(source_directory + fileList[0], 0, 1) # CLCfrq = 80 # Main loop by files start for fileNo in range(len(fileList)): # loop by files # *** Opening datafile *** fname = source_directory + fileList[fileNo] # ********************************************************************************* # *** Data file header read *** [ df_filename, df_filesize, df_system_name, df_obs_place, df_description, CLCfrq, df_creation_timeUTC, Channel, ReceiverMode, Mode, Navr,
def jds_wf_simple_reader(directory, no_of_spectra_to_average, skip_data_blocks, VminNorm, VmaxNorm, colormap, custom_dpi, save_long_file_aver, dyn_spectr_save_init, dyn_spectr_save_norm): """ Does not seem to work better or faster, takes a lot of RAM (32 GB) but works Is not used in any other scripts and is more a demonstration The same functions in non-fast file works approximately the same time but consumes less memory The only advantage of this function is reading the whole file at once """ current_time = time.strftime("%H:%M:%S") current_date = time.strftime("%d.%m.%Y") # *** Creating a folder where all pictures and results will be stored (if it doesn't exist) *** # result_folder = 'RESULTS_JDS_waveform_' + directory.split('/')[-2] result_folder = 'RESULTS_JDS_waveform' if not os.path.exists(result_folder): os.makedirs(result_folder) if dyn_spectr_save_init == 1: initial_spectra_folder = result_folder + '/Initial spectra' if not os.path.exists(initial_spectra_folder): os.makedirs(initial_spectra_folder) # *** Search JDS files in the directory *** file_list = find_files_only_in_current_folder(directory, '.jds', 1) print('') if len( file_list ) > 1: # Check if files have same parameters if there are more then one file in list # Check if all files (except the last) have same size same_or_not = check_if_all_files_of_same_size(directory, file_list, 1) # Check if all files in this folder have the same parameters in headers equal_or_not = check_if_JDS_files_of_equal_parameters( directory, file_list) if same_or_not and equal_or_not: print( '\n\n\n :-) All files seem to be of the same parameters! :-) \n\n\n' ) else: print( '\n\n\n ************************************************************************************* ' ) print( ' * *' ) print( ' * Seems files in folders are different check the errors and restart the script! *' ) print( ' * * ' '\n ************************************************************************************* \n\n\n' ) decision = int( input( '* Enter "1" to start processing, or "0" to stop the script: ' )) if decision != 1: sys.exit( '\n\n\n *** Program stopped! *** \n\n\n') # To print in console the header of first file print('\n First file header parameters: \n') # *** Data file header read *** [ df_filename, df_filesize, df_system_name, df_obs_place, df_description, CLCfrq, df_creation_timeUTC, Channel, ReceiverMode, Mode, Navr, TimeRes, fmin, fmax, df, frequency, freq_points_num, data_block_size ] = FileHeaderReaderJDS(directory + file_list[0], 0, 1) # Main loop by files start for fileNo in range(len(file_list)): # loop by files # *** Opening datafile *** fname = directory + file_list[fileNo] # ********************************************************************************* # *** Data file header read *** [ df_filename, df_filesize, df_system_name, df_obs_place, df_description, CLCfrq, df_creation_timeUTC, Channel, ReceiverMode, Mode, Navr, TimeRes, fmin, fmax, df, frequency, freq_points_num, data_block_size ] = FileHeaderReaderJDS(fname, 0, 0) # Create long data files and copy first data file header to them if fileNo == 0 and save_long_file_aver == 1: with open(fname, 'rb') as file: # *** Data file header read *** file_header = file.read(1024) # *** Creating a name for long timeline TXT file *** TLfile_name = df_filename + '_Timeline.txt' TLfile = open( TLfile_name, 'w') # Open and close to delete the file with the same name TLfile.close() # *** Creating a binary file with data for long data storage *** file_data_A_name = df_filename + '_Data_chA.dat' file_data_A = open(file_data_A_name, 'wb') file_data_A.write(file_header) file_data_A.seek(574) # FFT size place in header file_data_A.write(np.int32(data_block_size).tobytes()) file_data_A.seek(624) # Lb place in header file_data_A.write(np.int32(0).tobytes()) file_data_A.seek(628) # Hb place in header file_data_A.write(np.int32(data_block_size / 2).tobytes()) file_data_A.seek(632) # Wb place in header file_data_A.write(np.int32(data_block_size / 2).tobytes()) file_data_A.seek(636) # Navr place in header file_data_A.write( bytes([np.int32(Navr * no_of_spectra_to_average)])) file_data_A.close() if Channel == 2: file_data_B_name = df_filename + '_Data_chB.dat' file_data_B = open(file_data_B_name, 'wb') file_data_B.write(file_header) file_data_B.seek(574) # FFT size place in header file_data_B.write(np.int32(data_block_size).tobytes()) file_data_B.seek(624) # Lb place in header file_data_B.write(np.int32(0).tobytes()) file_data_B.seek(628) # Hb place in header file_data_B.write(np.int32(data_block_size / 2).tobytes()) file_data_B.seek(632) # Wb place in header file_data_B.write(np.int32(data_block_size / 2).tobytes()) file_data_B.seek(636) # Navr place in header file_data_B.write( bytes([np.int32(Navr * no_of_spectra_to_average)])) file_data_B.close() del file_header # !!! Make automatic calculations of time and frequency resolutions for waveform mode!!! # Manually set frequencies for one channel mode if (Channel == 0 and int(CLCfrq / 1000000) == 66) or (Channel == 1 and int(CLCfrq / 1000000) == 66): freq_points_num = 8192 frequency = np.linspace(0.0, 33.0, freq_points_num) # Manually set frequencies for two channels mode if Channel == 2 or (Channel == 0 and int(CLCfrq / 1000000) == 33) or ( Channel == 1 and int(CLCfrq / 1000000) == 33): freq_points_num = 8192 frequency = np.linspace(16.5, 33.0, freq_points_num) # For new receiver (temporary): if Channel == 2 and int(CLCfrq / 1000000) == 80: freq_points_num = 8192 frequency = np.linspace(0.0, 40.0, freq_points_num) # Calculation of number of blocks and number of spectra in the file if Channel == 0 or Channel == 1: # Single channel mode no_of_av_spectra_per_file = (df_filesize - 1024) / ( 2 * data_block_size * no_of_spectra_to_average) else: # Two channels mode no_of_av_spectra_per_file = (df_filesize - 1024) / ( 4 * data_block_size * no_of_spectra_to_average) no_of_blocks_in_file = (df_filesize - 1024) / data_block_size no_of_av_spectra_per_file = int(no_of_av_spectra_per_file) fine_CLCfrq = (int(CLCfrq / 1000000.0) * 1000000.0) # Real time resolution of averaged spectra real_av_spectra_dt = (1 / fine_CLCfrq) * (data_block_size - 4) * no_of_spectra_to_average if fileNo == 0: print(' Number of blocks in file: ', no_of_blocks_in_file) print(' Number of spectra to average: ', no_of_spectra_to_average) print(' Number of averaged spectra in file: ', no_of_av_spectra_per_file) print(' Time resolution of averaged spectrum: ', round(real_av_spectra_dt * 1000, 3), ' ms.') print('\n *** Reading data from file *** \n') # ******************************************************************************* # R E A D I N G D A T A * # ******************************************************************************* with open(fname, 'rb') as file: file.seek( 1024 + data_block_size * 4 * skip_data_blocks) # Jumping to 1024 byte from file beginning # *** DATA READING process *** # !!! Fake timing. Real timing to be done!!! TimeFigureScaleFig = np.linspace(0, no_of_av_spectra_per_file, no_of_av_spectra_per_file + 1) for i in range(no_of_av_spectra_per_file): TimeFigureScaleFig[i] = str(TimeFigureScaleFig[i]) TimeScaleFig = [] TimeScaleFull = [] # Calculation of number of blocks and number of spectra in the file if Channel == 0 or Channel == 1: # Single channel mode no_of_spectra_in_file = int( (df_filesize - 1024) / (1 * 2 * data_block_size)) else: # Two channels mode no_of_spectra_in_file = int( (df_filesize - 1024) / (1 * 4 * data_block_size)) no_of_av_spectra_per_file = 1 # Reading and reshaping all data with time data if Channel == 0 or Channel == 1: # Single channel mode wf_data = np.fromfile(file, dtype='i2', count=no_of_spectra_in_file * data_block_size) wf_data = np.reshape(wf_data, [data_block_size, no_of_spectra_in_file], order='F') if Channel == 2: # Two channels mode wf_data = np.fromfile(file, dtype='i2', count=2 * no_of_spectra_in_file * data_block_size) wf_data = np.reshape( wf_data, [data_block_size, 2 * no_of_spectra_in_file], order='F') print('Waveform read, shape: ', wf_data.shape) # Timing timeline_block_str = jds_waveform_time(wf_data, CLCfrq, data_block_size) timeline_block_str = timeline_block_str[0::16] for j in range(len(timeline_block_str)): TimeScaleFig.append(timeline_block_str[j][0:12]) TimeScaleFull.append(df_creation_timeUTC[0:10] + ' ' + timeline_block_str[j][0:12]) # Nulling the time blocks in waveform data wf_data[data_block_size - 4:data_block_size, :] = 0 # Scaling of the data - seems to be wrong in absolute value wf_data = wf_data / 32768.0 if Channel == 0 or Channel == 1: # Single channel mode wf_data_chA = wf_data # All the data is channel A data del wf_data # Deleting unnecessary array to free the memory if Channel == 2: # Two channels mode # Resizing to obtain the matrix for separation of channels wf_data_new = np.zeros( (2 * data_block_size, no_of_spectra_in_file)) for i in range(2 * no_of_spectra_in_file): if i % 2 == 0: wf_data_new[0:data_block_size, int(i / 2)] = wf_data[:, i] # Even else: wf_data_new[data_block_size:2 * data_block_size, int(i / 2)] = wf_data[:, i] # Odd del wf_data # Deleting unnecessary array to free the memory # Separating the data into two channels wf_data_chA = np.zeros( (data_block_size, no_of_spectra_in_file)) # Preparing empty array wf_data_chB = np.zeros( (data_block_size, no_of_spectra_in_file)) # Preparing empty array wf_data_chA[:, :] = wf_data_new[0:( 2 * data_block_size):2, :] # Separation to channel A wf_data_chB[:, :] = wf_data_new[1:( 2 * data_block_size):2, :] # Separation to channel B del wf_data_new print('Before transpose, shape: ', wf_data_chA.shape) # preparing matrices for spectra wf_data_chA = np.transpose(wf_data_chA) spectra_ch_a = np.zeros_like(wf_data_chA) if Channel == 2: wf_data_chB = np.transpose(wf_data_chB) spectra_ch_b = np.zeros_like(wf_data_chB) print('After transpose, shape: ', wf_data_chA.shape) # Calculation of spectra spectra_ch_a[:] = np.power(np.abs(np.fft.fft(wf_data_chA[:])), 2) if Channel == 2: # Two channels mode spectra_ch_b[:] = np.power(np.abs(np.fft.fft(wf_data_chB[:])), 2) print('After fft, spectrum shape: ', spectra_ch_a.shape) # Storing only first (left) mirror part of spectra spectra_ch_a = spectra_ch_a[:, :int(data_block_size / 2)] if Channel == 2: spectra_ch_b = spectra_ch_b[:, :int(data_block_size / 2)] print('After fft cut, spectrum shape: ', spectra_ch_a.shape) # At 33 MHz the specter is usually upside down, to correct it we use flip up/down if int(CLCfrq / 1000000) == 33: spectra_ch_a = np.fliplr(spectra_ch_a) if Channel == 2: spectra_ch_b = np.fliplr(spectra_ch_b) # Deleting the unnecessary matrices del wf_data_chA if Channel == 2: del wf_data_chB # Dimensions of [data_block_size / 2, no_of_spectra_in_file] # Calculation the averaged spectrum print('Shape before averaging: ', spectra_ch_a.shape) spectra_ch_a = np.reshape(spectra_ch_a, [ int(no_of_spectra_in_file / no_of_spectra_to_average), no_of_spectra_to_average, int(data_block_size / 2) ], order='F') spectra_ch_a = spectra_ch_a.mean(axis=1)[:] print('Shape after averaging: ', spectra_ch_a.shape) if Channel == 2: spectra_ch_b = np.reshape(spectra_ch_b, [ int(no_of_spectra_in_file / no_of_spectra_to_average), no_of_spectra_to_average, int(data_block_size / 2) ], order='F') spectra_ch_b = spectra_ch_b.mean(axis=1)[:] file.close() # Close the data file # Saving averaged spectra to a long data files if save_long_file_aver == 1: temp = spectra_ch_a.copy(order='C') file_data_A = open(file_data_A_name, 'ab') file_data_A.write(temp) file_data_A.close() if Channel == 2: temp = spectra_ch_b.copy(order='C') file_data_B = open(file_data_B_name, 'ab') file_data_B.write(temp) file_data_B.close() # Saving time data to long timeline file with open(TLfile_name, 'a') as TLfile: for i in range(no_of_av_spectra_per_file): TLfile.write((TimeScaleFull[i][:]) + ' \n') # str # Log data (make dB scale) with np.errstate(invalid='ignore', divide='ignore'): spectra_ch_a = 10 * np.log10(spectra_ch_a) if Channel == 2: spectra_ch_b = 10 * np.log10(spectra_ch_b) # If the data contains minus infinity values change them to particular values spectra_ch_a[np.isinf(spectra_ch_a)] = 40 if Channel == 2: spectra_ch_b[np.isinf(spectra_ch_b)] = 40 # ******************************************************************************* # P L O T T I N G D Y N A M I C S P E C T R A * # ******************************************************************************* spectra_ch_a = np.transpose(spectra_ch_a) if Channel == 2: spectra_ch_b = np.transpose(spectra_ch_b) no_of_av_spectra_per_file = spectra_ch_a.shape[1] if dyn_spectr_save_init == 1: # Plot of initial dynamic spectra VminA = np.min(spectra_ch_a) VmaxA = np.max(spectra_ch_a) VminB = VminA VmaxB = VmaxA if Channel == 2: VminB = np.min(spectra_ch_b) VmaxB = np.max(spectra_ch_b) if Channel == 0 or Channel == 1: # Single channel mode spectra_ch_b = spectra_ch_a suptitle = ('Dynamic spectrum (initial) ' + str(df_filename) + ' - Fig. ' + str(1) + ' of ' + str(1) + '\n Initial parameters: dt = ' + str(round(TimeRes * 1000., 3)) + ' ms, df = ' + str(round(df / 1000., 3)) + ' kHz, Receiver: ' + str(df_system_name) + ', Place: ' + str(df_obs_place) + '\n' + ReceiverMode + ', Fclock = ' + str(round(CLCfrq / 1000000, 1)) + ' MHz, Avergaed spectra: ' + str(no_of_spectra_to_average) + ' (' + str(round(no_of_spectra_to_average * TimeRes, 3)) + ' sec.), Description: ' + str(df_description)) fig_file_name = (initial_spectra_folder + '/' + df_filename[0:14] + ' Initial dynamic spectrum fig.' + str(0 + 1) + '.png') if Channel == 0 or Channel == 1: # Single channel mode OneDynSpectraPlot(spectra_ch_a, VminA, VmaxA, suptitle, 'Intensity, dB', no_of_av_spectra_per_file, TimeScaleFig, frequency, freq_points_num, colormap, 'UTC Time, HH:MM:SS.msec', fig_file_name, current_date, current_time, software_version, custom_dpi) if Channel == 2: TwoDynSpectraPlot(spectra_ch_a, spectra_ch_b, VminA, VmaxA, VminB, VmaxB, suptitle, 'Intensity, dB', 'Intensity, dB', no_of_av_spectra_per_file, TimeScaleFig, TimeScaleFig, frequency, freq_points_num, colormap, 'Channel A', 'Channel B', fig_file_name, current_date, current_time, software_version, custom_dpi) if dyn_spectr_save_norm == 1: # Normalization and cleaning of data Normalization_dB(spectra_ch_a.transpose(), freq_points_num, no_of_av_spectra_per_file) if Channel == 2: Normalization_dB(spectra_ch_b.transpose(), freq_points_num, no_of_av_spectra_per_file) simple_channel_clean(spectra_ch_a, 8) if Channel == 2: simple_channel_clean(spectra_ch_b, 8) # Plot of normalized and cleaned dynamic spectra suptitle = ('Normalized and cleaned dynamic spectrum (initial) ' + str(df_filename) + ' - Fig. ' + str(0 + 1) + ' of ' + str(1) + '\n Initial parameters: dt = ' + str(round(TimeRes * 1000, 3)) + ' ms, df = ' + str(round(df / 1000., 3)) + ' kHz, Receiver: ' + str(df_system_name) + ', Place: ' + str(df_obs_place) + '\n' + ReceiverMode + ', Fclock = ' + str(round(CLCfrq / 1000000, 1)) + ' MHz, Avergaed spectra: ' + str(no_of_spectra_to_average) + ' (' + str(round(no_of_spectra_to_average * TimeRes, 3)) + ' sec.), Description: ' + str(df_description)) fig_file_name = (result_folder + '/' + df_filename[0:14] + ' Normalized and cleaned dynamic spectrum fig.' + str(0 + 1) + '.png') if Channel == 0 or Channel == 1: # Single channel mode OneDynSpectraPlot(spectra_ch_a, VminNorm, VmaxNorm, suptitle, 'Intensity, dB', no_of_av_spectra_per_file, TimeScaleFig, frequency, freq_points_num, colormap, 'UTC Time, HH:MM:SS.msec', fig_file_name, current_date, current_time, software_version, custom_dpi) if Channel == 2: TwoDynSpectraPlot(spectra_ch_a, spectra_ch_b, VminNorm, VmaxNorm, VminNorm, VmaxNorm, suptitle, 'Intensity, dB', 'Intensity, dB', no_of_av_spectra_per_file, TimeScaleFig, TimeScaleFig, frequency, freq_points_num, colormap, 'Channel A', 'Channel B', fig_file_name, current_date, current_time, software_version, custom_dpi) results_files_list = [] results_files_list.append(file_data_A_name) if Channel == 2: results_files_list.append(file_data_B_name) return results_files_list
def wf32_two_channel_phase_calibration(fname, no_of_points_for_fft_dedisp, no_of_spectra_in_bunch, phase_calibr_txt_file): """ function reads waveform data in wf32 format, makes FFT, cuts the symmetrical half of the spectra and multiplies complex data by phase calibration data read from txt file. Then a symmetrcal part of spectra are made and joined to the shifted one, inverse FFT as applied and data are stored in waveform wf32 format Input parameters: fname - name of file with initial wf32 data no_of_points_for_fft_dedisp - number of waveform data points to use for FFT phase_calibr_txt_file - txt file with phase calibration data Output parameters: file_data_name - name of file with calibrated data """ # Rename the data file to make the new data file of the same name as initial one non_calibrated_fname = fname[:-5] + '_without_phase_calibration' + '.wf32' calibrated_fname = fname print('\n Phase calibration of one channel \n') print(' Old filename of initial file: ', calibrated_fname) print(' New filename of initial file: ', non_calibrated_fname) os.rename(calibrated_fname, non_calibrated_fname) # *** Data file header read *** [ df_filename, df_filesize, df_system_name, df_obs_place, df_description, clock_freq, df_creation_timeUTC, Channel, ReceiverMode, Mode, Navr, time_resolution, fmin, fmax, df, frequency_list, freq_points_num, data_block_size ] = FileHeaderReaderJDS(non_calibrated_fname, 0, 0) # Read phase calibration txt file phase_calibr_file = open(phase_calibr_txt_file, 'r') phase_vs_freq = [] for line in phase_calibr_file: phase_vs_freq.append(np.float(line)) phase_calibr_file.close() fig = plt.figure(figsize=(9, 5)) ax1 = fig.add_subplot(111) ax1.plot(phase_vs_freq, linestyle='-', linewidth='1.00', label='Phase to add') ax1.legend(loc='upper right', fontsize=6) ax1.grid(b=True, which='both', color='silver', linestyle='-') ax1.set_ylabel('Phase, a.u.', fontsize=6, fontweight='bold') pylab.savefig('00_Phase to add.png', bbox_inches='tight', dpi=160) plt.close('all') # Converting phase to complex numbers cmplx_phase = np.zeros((len(phase_vs_freq)), dtype=np.complex) for i in range(len(phase_vs_freq)): cmplx_phase[i] = np.cos( phase_vs_freq[i]) + 1j * np.sin(phase_vs_freq[i]) # Create long data files and copy first data file header to them non_calibr_file_data = open(non_calibrated_fname, 'rb') file_header = non_calibr_file_data.read(1024) # *** Creating a binary file with data for long data storage *** calibr_file_data = open(calibrated_fname, 'wb') calibr_file_data.write(file_header) calibr_file_data.close() del file_header # Calculation of number of blocks and number of spectra in the file no_of_spectra_per_file = int( (df_filesize - 1024) / (no_of_points_for_fft_dedisp * 4)) no_of_bunches_per_file = math.ceil(no_of_spectra_per_file / no_of_spectra_in_bunch) print(' Number of spectra in bunch: ', no_of_spectra_in_bunch) print(' Number of batches per file: ', no_of_bunches_per_file, '') print(' Number of spectra per file: ', no_of_spectra_per_file, '\n') non_calibr_file_data.seek(1024) # Jumping to 1024 byte from file beginning bar = IncrementalBar(' Phase calibration of the file: ', max=no_of_bunches_per_file - 1, suffix='%(percent)d%%') for bunch in range(no_of_bunches_per_file): if bunch < no_of_bunches_per_file - 1: pass else: no_of_spectra_in_bunch = no_of_spectra_per_file - bunch * no_of_spectra_in_bunch # print(' Last bunch ', bunch, ', spectra in bunch: ', no_of_spectra_in_bunch) bar.next() # Reading and reshaping all data with time data wf_data = np.fromfile(non_calibr_file_data, dtype='f4', count=no_of_spectra_in_bunch * no_of_points_for_fft_dedisp) wf_data = np.reshape( wf_data, [no_of_points_for_fft_dedisp, no_of_spectra_in_bunch], order='F') # preparing matrices for spectra spectra = np.zeros( (no_of_points_for_fft_dedisp, no_of_spectra_in_bunch), dtype='complex64') # Calculation of spectra for i in range(no_of_spectra_in_bunch): spectra[:, i] = np.fft.fft(wf_data[:, i]) del wf_data # Add phase to the data (multiply by complex number) for i in range(no_of_spectra_in_bunch): spectra[:, i] = spectra[:, i] * cmplx_phase[:] # Preparing array for new waveform wf_data = np.zeros( (no_of_points_for_fft_dedisp, no_of_spectra_in_bunch)) # Making IFFT for i in range(no_of_spectra_in_bunch): wf_data[:, i] = np.real(np.fft.ifft(spectra[:, i])) del spectra # Reshaping the waveform to single dimension (real) wf_data = np.reshape( wf_data, [no_of_points_for_fft_dedisp * no_of_spectra_in_bunch, 1], order='F') # Saving waveform data to wf32 file calibr_file_data = open(calibrated_fname, 'ab') calibr_file_data.write(np.float32(wf_data).transpose().copy(order='C')) calibr_file_data.close() bar.finish() return
'\x00') # Initial data file name file.close() receiver_type = df_filename[-4:] # Reading file header to obtain main parameters of the file if receiver_type == '.adr': [TimeRes, fmin, fmax, df, frequency_list, FFTsize] = FileHeaderReaderADR(data_filename, 0) if receiver_type == '.jds': [ df_filename, df_filesize, df_system_name, df_obs_place, df_description, CLCfrq, df_creation_timeUTC, SpInFile, ReceiverMode, Mode, Navr, TimeRes, fmin, fmax, df, frequency_list, FFTsize, dataBlockSize ] = FileHeaderReaderJDS(data_filename, 0, 1) #************************************************************************************ # R E A D I N G D A T A * #************************************************************************************ num_frequencies = len(frequency_list) # Calculating number of samples per period and number of blocks samples_per_period = int(np.ceil(pulsar_period / TimeRes)) num_of_blocks = int(np.floor(SpInFile / samples_per_period)) print(' Number of samples per period: ', samples_per_period, ' \n') print(' Number of blocks in file: ', num_of_blocks, ' \n') if receiver_type == '.jds': num_frequencies_initial = len(frequency_list) - 4
def cut_needed_time_points_from_dat_to_txt(path, filename): import matplotlib matplotlib.use('TkAgg') """ Function to cut the part of pulsar period data with the pulse from dat file to the txt file User is asked to specify the start and stop point of the selected pulse period. Returns the start and stop point numbers, the data and image are saved to HDD """ software_version = '2021.08.08' current_time = time.strftime("%H:%M:%S") current_date = time.strftime("%d.%m.%Y") data_file = open(path + '/' + filename, 'rb') df_filepath = data_file.read(32).decode('utf-8').rstrip('\x00') # Initial data file name data_file.close() if df_filepath[-4:] == '.jds': # If data obtained from DSPZ receiver [df_filepath, df_filesize, df_system_name, df_obs_place, df_description, CLCfrq, df_creation_timeUTC, SpInFile, ReceiverMode, Mode, Navr, time_resolution, fmin, fmax, df, frequency, freq_points_num, dataBlockSize] = FileHeaderReaderJDS(path + '/' + filename, 0, 0) spectra_to_read = int((df_filesize - 1024) / (8 * freq_points_num)) data_file = open(path + '/' + filename, 'rb') data_file.seek(1024, os.SEEK_SET) array = np.fromfile(data_file, dtype=np.float64, count=spectra_to_read * freq_points_num) array = np.reshape(array, [freq_points_num, spectra_to_read], order='F') data_file.close() print(' Shape of the array from dat file: ', array.shape) # Logging data array = 10 * np.log10(array) # Normalizing data array = array - np.mean(array) # Show the pulse profile to select the start and end points of the cut fig = plt.figure(figsize=(16.0, 9.0)) ax1 = fig.add_subplot(111) ax1.plot(np.mean(array, axis=0), linewidth='0.50') ax1.set_xlim(xmin=0, xmax=array.shape[1]) ax1.tick_params(axis='both', which='major', labelsize=10) ax1.set_xlabel('Time points to select', fontsize=8, fontweight='bold') ax1.set_ylabel('Amplitude, a.u.', fontsize=8, fontweight='bold') plt.show() # Enter the points from the keyboard start_point = int(input('\n Begin from point: ')) end_point = int(input(' End at point: ')) result_array = array[:, start_point: end_point] del array print('\n Shape of result array: ', result_array.shape) # Save cut data to the new txt file single_pulse_txt = open(path + '/Extracted_s' + filename[1:-4] + '.txt', "w") for freq in range(result_array.shape[0] - 1): single_pulse_txt.write(' '.join(' {:+12.7E}'.format(result_array[freq, i]) \ for i in range(result_array.shape[1])) + ' \n') single_pulse_txt.close() # Making result figure with dynamic spectra and profiles in full band and split in subbands fig = plt.figure(figsize=(16.0, 8.0)) gs = GridSpec(3, 5, figure=fig) rc('font', size=8, weight='bold') ax1 = fig.add_subplot(gs[0:2, 0]) ax1.set_title('Full band, points: ' + str(start_point) + ' - ' + str(end_point), fontsize=8, fontweight='bold') ax1.imshow(np.flipud(result_array), aspect='auto', cmap=colormap, vmin=spectrum_pic_min, vmax=spectrum_pic_max, extent=[0, result_array.shape[1], 16.5, 33.0]) ax1.xaxis.set_ticklabels([]) ax1.set_ylabel('Frequency, MHz', fontsize=10, fontweight='bold') ax2 = fig.add_subplot(gs[2, 0]) ax2.plot(np.mean(result_array, axis=0), linewidth='0.50', color='C1') ax2.set_xlim(xmin=0, xmax=result_array.shape[1]) ax2.set_xlabel('Time points', fontsize=10, fontweight='bold') ax2.set_ylabel('Amplitude, a.u.', fontsize=10, fontweight='bold') ax3 = fig.add_subplot(gs[0:2, 1]) ax3.set_title('16.5 - 20.5 MHz', fontsize=8, fontweight='bold') ax3.imshow(np.flipud(result_array[0:1986]), aspect='auto', cmap=colormap, vmin=spectrum_pic_min, vmax=spectrum_pic_max, extent=[0, result_array.shape[1], 16.5, 20.5]) ax3.xaxis.set_ticklabels([]) ax4 = fig.add_subplot(gs[2, 1]) ax4.plot(np.mean(result_array[0:1986], axis=0), linewidth='0.50', color='C4') ax4.set_xlim(xmin=0, xmax=result_array.shape[1]) ax4.set_yticklabels([]) ax4.set_yticks([]) ax4.set_xlabel('Time points', fontsize=10, fontweight='bold') ax5 = fig.add_subplot(gs[0:2, 2]) ax5.set_title('20.5 - 24.5 MHz', fontsize=8, fontweight='bold') ax5.imshow(np.flipud(result_array[1986:3972]), aspect='auto', cmap=colormap, vmin=spectrum_pic_min, vmax=spectrum_pic_max, extent=[0, result_array.shape[1], 20.5, 24.5]) ax5.xaxis.set_ticklabels([]) ax6 = fig.add_subplot(gs[2, 2]) ax6.plot(np.mean(result_array[1986:3972], axis=0), linewidth='0.50', color='C4') ax6.set_xlim(xmin=0, xmax=result_array.shape[1]) ax6.set_yticklabels([]) ax6.set_yticks([]) ax6.set_xlabel('Time points', fontsize=10, fontweight='bold') ax7 = fig.add_subplot(gs[0:2, 3]) ax7.set_title('24.5 - 28.5 MHz', fontsize=8, fontweight='bold') ax7.imshow(np.flipud(result_array[3972:5958]), aspect='auto', cmap=colormap, vmin=spectrum_pic_min, vmax=spectrum_pic_max, extent=[0, result_array.shape[1], 24.5, 28.5]) ax7.xaxis.set_ticklabels([]) ax8 = fig.add_subplot(gs[2, 3]) ax8.plot(np.mean(result_array[3972:5958], axis=0), linewidth='0.50', color='C4') ax8.set_xlim(xmin=0, xmax=result_array.shape[1]) ax8.set_yticklabels([]) ax8.set_yticks([]) ax8.set_xlabel('Time points', fontsize=10, fontweight='bold') ax9 = fig.add_subplot(gs[0:2, 4]) ax9.set_title('28.5 - 32.5 MHz', fontsize=8, fontweight='bold') ax9.imshow(np.flipud(result_array[5958:7944]), aspect='auto', cmap=colormap, vmin=spectrum_pic_min, vmax=spectrum_pic_max, extent=[0, result_array.shape[1], 28.5, 32.5]) ax9.xaxis.set_ticklabels([]) ax10 = fig.add_subplot(gs[2, 4]) ax10.plot(np.mean(result_array[5958:7944], axis=0), linewidth='0.50', color='C4') ax10.set_xlim(xmin=0, xmax=result_array.shape[1]) ax10.set_yticklabels([]) ax10.set_yticks([]) ax10.set_xlabel('Time points', fontsize=10, fontweight='bold') fig.subplots_adjust(hspace=0.00, wspace=0.25, top=0.93) fig.suptitle('Result pulse cut from ' + df_filepath + ' (' + df_description + '), DM: ' + find_between(filename, 'DM_', '_') + r' $pc * cm^{-3}$', fontsize=10, fontweight='bold') # fig.text(0.80, 0.05, 'Processed ' + current_date + ' at ' + current_time, fontsize=5, transform=plt.gcf().transFigure) fig.text(0.09, 0.05, 'Software version: '+software_version+', [email protected], IRA NASU', fontsize=5, transform=plt.gcf().transFigure) pylab.savefig(path + '/Extracted_s' + filename[1:-4] + '.png', bbox_inches='tight', dpi=customDPI) plt.close('all') # from PIL import Image # img = Image.open(path + '/Extracted_s' + filename[1:-4] + '.png') # img.show() # import sys # import subprocess # # def openImage(path): # imageViewerFromCommandLine = {'linux': 'xdg-open', # 'win32': 'explorer', # 'darwin': 'open'}[sys.platform] # subprocess.Popen([imageViewerFromCommandLine, path]) # # openImage(path + '/Extracted_s' + filename[1:-4] + '.png') # # Array preprocessing # # array = np.flipud(result_array[1024:3972]) # freq_aver_const = 2 # time_aver_const = 1 # # # Cutting the array ends which are not divisible to average constants # array = array[:(array.shape[0] // freq_aver_const) * freq_aver_const, # :(array.shape[1] // time_aver_const) * time_aver_const] # # # Time averaging # if time_aver_const > 1: # array = array.reshape(array.shape[0], -1, time_aver_const) # array = np.mean(array, axis=2) # # # Frequency averaging # if freq_aver_const > 1: # array = np.transpose(array) # array = np.mean(array.reshape(array.shape[0], -1, freq_aver_const), axis=2) # array = np.transpose(array) # # # Plot of the pulse dynamic spectrum # rc('font', size=8, weight='bold') # fig = plt.figure(figsize=(5.0, 8.0)) # ax1 = fig.add_subplot() # # ax1.set_title('Full band, points: ', fontsize=8, fontweight='bold') # ax1.imshow(array, aspect='auto', cmap='Greys', vmin=-1.0, vmax=10.0, # extent=[0, array.shape[1], 0, array.shape[0]]) # ax1.set_ylabel('Frequency, MHz', fontsize=10, fontweight='bold') # # fig.subplots_adjust(hspace=0.00, wspace=0.25, top=0.93) # # fig.suptitle('Result pulse cut for further processing', fontsize=10, fontweight='bold') # # fig.text(0.80, 0.05, 'Processed ' + current_date + ' at ' + current_time, # # fontsize=5, transform=plt.gcf().transFigure) # # fig.text(0.09, 0.05, 'Software version: '+software_version+', [email protected], IRA NASU', # # fontsize=5, transform=plt.gcf().transFigure) # pylab.savefig(path + '/Extracted_s' + filename[1:-4] + '_dynamic_spectrum.png', bbox_inches='tight', dpi=customDPI) # plt.close('all') return start_point, end_point
def pulsar_period_DM_compensated_pics(common_path, filename, pulsar_name, normalize_response, profile_pic_min, profile_pic_max, spectrum_pic_min, spectrum_pic_max, periods_per_fig, customDPI, colormap, save_strongest, threshold): current_time = time.strftime("%H:%M:%S") current_date = time.strftime("%d.%m.%Y") # Creating a folder where all pictures and results will be stored (if it doesn't exist) result_path = "RESULTS_pulsar_n_periods_pics_" + filename if not os.path.exists(result_path): os.makedirs(result_path) if save_strongest: best_result_path = result_path + '/Strongest_pulses' if not os.path.exists(best_result_path): os.makedirs(best_result_path) # Taking pulsar period from catalogue pulsar_ra, pulsar_dec, DM, p_bar = catalogue_pulsar(pulsar_name) # DAT file to be analyzed: filepath = common_path + filename # Timeline file to be analyzed: timeline_filepath = common_path + filename.split( '_Data_')[0] + '_Timeline.txt' # Opening DAT datafile file = open(filepath, 'rb') # Data file header read df_filesize = os.stat(filepath).st_size # Size of file df_filepath = file.read(32).decode('utf-8').rstrip( '\x00') # Initial data file name file.close() if df_filepath[-4:] == '.adr': [ df_filepath, df_filesize, df_system_name, df_obs_place, df_description, CLCfrq, df_creation_timeUTC, ReceiverMode, Mode, sumDifMode, NAvr, time_resolution, fmin, fmax, df, frequency, FFTsize, SLine, Width, BlockSize ] = FileHeaderReaderADR(filepath, 0, 0) freq_points_num = len(frequency) if df_filepath[-4:] == '.jds': # If data obtained from DSPZ receiver [ df_filepath, df_filesize, df_system_name, df_obs_place, df_description, CLCfrq, df_creation_timeUTC, SpInFile, ReceiverMode, Mode, Navr, time_resolution, fmin, fmax, df, frequency, freq_points_num, dataBlockSize ] = FileHeaderReaderJDS(filepath, 0, 1) # ************************************************************************************ # R E A D I N G D A T A * # ************************************************************************************ # Time line file reading timeline, dt_timeline = time_line_file_reader(timeline_filepath) # Calculation of the dimensions of arrays to read taking into account the pulsar period spectra_in_file = int( (df_filesize - 1024) / (8 * freq_points_num)) # int(df_filesize - 1024)/(2*4*freq_points_num) spectra_to_read = int( np.round((periods_per_fig * p_bar / time_resolution), 0)) num_of_blocks = int(np.floor(spectra_in_file / spectra_to_read)) print(' Pulsar period: ', p_bar, 's.') print(' Time resolution: ', time_resolution, 's.') print(' Number of spectra to read in', periods_per_fig, 'periods: ', spectra_to_read, ' ') print(' Number of spectra in file: ', spectra_in_file, ' ') print(' Number of', periods_per_fig, 'periods blocks in file: ', num_of_blocks, '\n') # Data reading and making figures print('\n\n *** Data reading and making figures *** \n\n') data_file = open(filepath, 'rb') data_file.seek( 1024, os.SEEK_SET ) # Jumping to 1024+number of spectra to skip byte from file beginning bar = IncrementalBar(' Making pictures of n periods: ', max=num_of_blocks, suffix='%(percent)d%%') bar.start() for block in range(num_of_blocks + 1): # Main loop by blocks of data # bar.next() # current_time = time.strftime("%H:%M:%S") # print(' * Data block # ', block + 1, ' of ', num_of_blocks + 1, ' started at: ', current_time) # Reading the last block which is less then 3 periods if block == num_of_blocks: spectra_to_read = spectra_in_file - num_of_blocks * spectra_to_read # Reading and preparing block of data (3 periods) data = np.fromfile(data_file, dtype=np.float64, count=spectra_to_read * len(frequency)) data = np.reshape(data, [len(frequency), spectra_to_read], order='F') data = 10 * np.log10(data) if normalize_response > 0: Normalization_dB(data.transpose(), len(frequency), spectra_to_read) # Preparing single averaged data profile for figure profile = data.mean(axis=0)[:] profile = profile - np.mean(profile) data = data - np.mean(data) # Time line fig_time_scale = timeline[block * spectra_to_read:(block + 1) * spectra_to_read] # Making result picture fig = plt.figure(figsize=(9.2, 4.5)) rc('font', size=5, weight='bold') ax1 = fig.add_subplot(211) ax1.plot(profile, color=u'#1f77b4', linestyle='-', alpha=1.0, linewidth='0.60', label='3 pulses time profile') ax1.legend(loc='upper right', fontsize=5) ax1.grid(b=True, which='both', color='silver', linewidth='0.50', linestyle='-') ax1.axis([0, len(profile), profile_pic_min, profile_pic_max]) ax1.set_ylabel('Amplitude, AU', fontsize=6, fontweight='bold') ax1.set_title('File: ' + filename + ' Description: ' + df_description + ' Resolution: ' + str(np.round(df / 1000, 3)) + ' kHz and ' + str(np.round(time_resolution * 1000, 3)) + ' ms.', fontsize=5, fontweight='bold') ax1.tick_params(axis='x', which='both', bottom=False, top=False, labelbottom=False) ax2 = fig.add_subplot(212) ax2.imshow(np.flipud(data), aspect='auto', cmap=colormap, vmin=spectrum_pic_min, vmax=spectrum_pic_max, extent=[0, len(profile), frequency[0], frequency[-1]]) ax2.set_xlabel('Time UTC (at the lowest frequency), HH:MM:SS.ms', fontsize=6, fontweight='bold') ax2.set_ylabel('Frequency, MHz', fontsize=6, fontweight='bold') text = ax2.get_xticks().tolist() for i in range(len(text) - 1): k = int(text[i]) text[i] = fig_time_scale[k][11:23] ax2.set_xticklabels(text, fontsize=5, fontweight='bold') fig.subplots_adjust(hspace=0.05, top=0.91) fig.suptitle('Single pulses of ' + pulsar_name + ' (DM: ' + str(DM) + r' $\mathrm{pc \cdot cm^{-3}}$' + ', Period: ' + str(p_bar) + ' s.), fig. ' + str(block + 1) + ' of ' + str(num_of_blocks + 1), fontsize=7, fontweight='bold') fig.text(0.80, 0.04, 'Processed ' + current_date + ' at ' + current_time, fontsize=3, transform=plt.gcf().transFigure) fig.text(0.09, 0.04, 'Software version: ' + Software_version + ', [email protected], IRA NASU', fontsize=3, transform=plt.gcf().transFigure) pylab.savefig(result_path + '/' + filename + ' fig. ' + str(block + 1) + ' - Combined picture.png', bbox_inches='tight', dpi=customDPI) # If the profile has points above threshold save picture also into separate folder if save_strongest and np.max(profile) > threshold: pylab.savefig(best_result_path + '/' + filename + ' fig. ' + str(block + 1) + ' - Combined picture.png', bbox_inches='tight', dpi=customDPI) plt.close('all') bar.next() bar.finish() data_file.close()
def check_if_JDS_files_of_equal_parameters(folder_path, file_list): ''' The function checks if main parameters of the ADR files are equal (are they from the same observation) Input parameters: folder_path - path to folder with files file_list - list of files in the folder to check Output parameters: equal_or_not - "1" if files have equal parameters, "0" - otherwise ''' df_system_name_list = [] df_obs_place_list = [] df_description_list = [] ReceiverMode_list = [] TimeRes_list = [] fmin_list = [] FreqPointsNum_list = [] dataBlockSize_list = [] for file_no in range(len(file_list)): filepath = folder_path + file_list[file_no] [ df_filename, df_filesize, df_system_name, df_obs_place, df_description, CLCfrq, df_creation_timeUTC, SpInFile, ReceiverMode, Mode, Navr, TimeRes, fmin, fmax, df, frequency, FreqPointsNum, dataBlockSize ] = FileHeaderReaderJDS(filepath, 0, 0) df_system_name_list.append(df_system_name) df_obs_place_list.append(df_obs_place) df_description_list.append(df_description) ReceiverMode_list.append(ReceiverMode) TimeRes_list.append(TimeRes) fmin_list.append(fmin) FreqPointsNum_list.append(FreqPointsNum) dataBlockSize_list.append(dataBlockSize) i = 0 if df_system_name_list.count( df_system_name_list[0]) == len(df_system_name_list): i = i + 1 if df_obs_place_list.count(df_obs_place_list[0]) == len(df_obs_place_list): i = i + 1 if df_description_list.count( df_description_list[0]) == len(df_description_list): i = i + 1 if ReceiverMode_list.count(ReceiverMode_list[0]) == len(ReceiverMode_list): i = i + 1 if TimeRes_list.count(TimeRes_list[0]) == len(TimeRes_list): i = i + 1 if fmin_list.count(fmin_list[0]) == len(fmin_list): i = i + 1 if FreqPointsNum_list.count( FreqPointsNum_list[0]) == len(FreqPointsNum_list): i = i + 1 if dataBlockSize_list.count( dataBlockSize_list[0]) == len(dataBlockSize_list): i = i + 1 if i == 8: equal_or_not = 1 print(' OK: all files have the same parameters!') else: equal_or_not = 0 print( '\n **********************************************************\n !!! WARNING: Parameters of files in folder differ !!! \n **********************************************************' ) print('\n * Check letteral parameters of the files in list: \n') for file_no in range(len(file_list)): print(' ', file_no + 1, ') ', df_system_name_list[file_no], ' ', df_obs_place_list[file_no], ' ', df_description_list[file_no]) print('\n * Check numerical parameters of the files in list: \n') print( ' No DSP mode Sum/Diff Time res. FFT size Start line Width Block size\n' ) for file_no in range(len(file_list)): #print(' ', file_no+1 ,') ', str(ADRmode_list[file_no]), ' ',str(sumDifMode_list[file_no]), ' ',np.round(TimeRes_list[file_no], 6), ' ', FFT_Size_list[file_no], ' ', SLine_list[file_no], Width_list[file_no], ' ', BlockSize_list[file_no]) print(' {:0>4d}'.format(file_no + 1), ' ', (ReceiverMode_list[file_no]), ' {:.6f}'.format(np.round(TimeRes_list[file_no], 6)), ' {:1.0f}'.format(fmin_list[file_no]), ' {:1.0f}'.format(FreqPointsNum_list[file_no]), ' {:6.0f}'.format(dataBlockSize_list[file_no])) return equal_or_not
def convert_wf32_to_dat_with_overlap(fname, no_of_points_for_fft_spectr, no_of_spectra_in_bunch, hanning_window): """ function converts waveform data in .wf32 format to spectra in .dat format : fname : name of .wf32 file with waveform data : no_of_points_for_fft : number of points for FFT to provide necessary time-frequency resolution : return : file_data_name - name of .dat file with result spectra """ # *** Data file header read *** [ df_filename, df_filesize, df_system_name, df_obs_place, df_description, clock_freq, df_time_utc, channel, receiver_mode, mode, n_avr, time_resolution, fmin, fmax, df, frequency, freq_points_num, data_block_size ] = FileHeaderReaderJDS(fname, 0, 0) freq_points_num = int(no_of_points_for_fft_spectr / 2) with open(fname, 'rb') as file: # *** Data file header read *** file_header = file.read(1024) # *** Creating a binary file with spectra data for long data storage *** file_data_name = fname[:-5] + '.dat' file_data = open(file_data_name, 'wb') file_data.write(file_header) file_data.seek(574) # FFT size place in header file_data.write(np.int32(no_of_points_for_fft_spectr).tobytes()) file_data.seek(624) # Lb place in header file_data.write(np.int32(0).tobytes()) file_data.seek(628) # Hb place in header file_data.write(np.int32(freq_points_num).tobytes()) file_data.seek(632) # Wb place in header file_data.write(np.int32(freq_points_num).tobytes()) file_data.seek(636) file_data.write(np.int32(1).tobytes()) # Seem to work OK file_data.close() del file_header # Calculation of number of blocks and number of spectra in the file no_of_bunches_per_file = int( (df_filesize - 1024) / ((no_of_spectra_in_bunch + 0.5) * no_of_points_for_fft_spectr * 4)) # Real time resolution of averaged spectra fine_clock_freq = int(clock_freq / 1000000.0) * 1000000.0 real_spectra_dt = float(no_of_points_for_fft_spectr / fine_clock_freq) real_spectra_df = float( (fine_clock_freq / 2) / (no_of_points_for_fft_spectr / 2)) print(' Number of spectra in bunch: ', no_of_spectra_in_bunch) print(' Sampling clock frequency: ', fine_clock_freq, ' Hz') print(' Number of bunches to read in file: ', no_of_bunches_per_file) print(' Time resolution of calculated spectra: ', round(real_spectra_dt * 1000, 3), ' ms') print(' Frequency resolution of calculated spectra: ', round(real_spectra_df / 1000, 3), ' kHz \n') # print('\n Reading data from file \n') file.seek(1024) # Jumping to 1024 byte from file beginning # *** Creating a new timeline TXT file for results *** new_tl_file_name = file_data_name.split('_Data_', 1)[0] + '_Timeline.txt' new_tl_file = open( new_tl_file_name, 'w') # Open and close to delete the file with the same name new_tl_file.close() # *** Reading timeline file *** old_tl_file_name = fname.split("_Data_", 1)[0] + '_Timeline.wtxt' old_tl_file = open(old_tl_file_name, 'r') new_tl_file = open( new_tl_file_name, 'w') # Open and close to delete the file with the same name # Making the variable for half length of the spectrum for convenience half_of_spectrum = int(no_of_points_for_fft_spectr / 2) # Making a small buffer vector to store the last half ot spectrum for the next loop step buffer = np.zeros(half_of_spectrum) bar = IncrementalBar(' Conversion from waveform to spectra: ', max=no_of_bunches_per_file - 1, suffix='%(percent)d%%') bar.start() for bunch in range(no_of_bunches_per_file - 1): # print('Bunch # ', bunch, ' of ', no_of_bunches_per_file - 1) # Read time from timeline file for the bunch time_scale_bunch = [] for line in range(no_of_spectra_in_bunch): tmp = str(old_tl_file.readline()) time_scale_bunch.append(tmp) # append the current value time_scale_bunch.append( tmp ) # append once more the same value for timing of fft with overlap # Saving time data to new file for i in range(len(time_scale_bunch)): new_tl_file.write((time_scale_bunch[i][:]) + '') # Reading and reshaping data of the bunch wf_data = np.fromfile(file, dtype='f4', count=no_of_spectra_in_bunch * no_of_points_for_fft_spectr) wf_data = np.concatenate((buffer, wf_data), axis=0) # Save new data from the end to the buffer buffer = wf_data[-half_of_spectrum:].copy() # Selecting the needed sequence of the data and reshaping to rectangular array wf_data_1 = np.reshape( wf_data[:-half_of_spectrum].copy(), [no_of_points_for_fft_spectr, no_of_spectra_in_bunch], order='F') wf_data_2 = np.reshape( wf_data[half_of_spectrum:].copy(), [no_of_points_for_fft_spectr, no_of_spectra_in_bunch], order='F') wf_data_1 = np.transpose(wf_data_1) wf_data_2 = np.transpose(wf_data_2) del wf_data # Merging 2 arrays into one rectangular array in the one by one order wf_data = np.zeros( (2 * no_of_spectra_in_bunch, no_of_points_for_fft_spectr)) wf_data[0::2, :] = wf_data_1[:, :] wf_data[1::2, :] = wf_data_2[:, :] del wf_data_1, wf_data_2 # Apply window to data for FFT if hanning_window: # window = np.hanning(no_of_points_for_fft_spectr) window = np.hamming(no_of_points_for_fft_spectr) wf_data[:] = wf_data[:] * window[:] del window # Preparing empty array for spectra spectra = np.zeros_like(wf_data, dtype=np.float64) # Calculation of spectra spectra[:] = np.power(np.abs(np.fft.fft(wf_data[:])), 2) # spectra[:] = np.abs(np.fft.fft(np.power(wf_data[:], 2))) # Does not work # spectra[:, i] = np.power(np.abs(np.fft.fft(wf_data[:, i])), 2) del wf_data # Storing only first (left) mirror part of spectra spectra = spectra[:, :int(no_of_points_for_fft_spectr / 2)] # spectra = spectra[: int(no_of_points_for_fft_spectr / 2), :] # At 33 MHz clock frequency the specter is upside down, to correct it we use flip up/down if int(clock_freq / 1000000) == 33: # spectra = np.flipud(spectra) spectra = np.fliplr(spectra) # Saving spectra data to dat file temp = spectra.copy(order='C') file_data = open(file_data_name, 'ab') file_data.write(np.float64(temp)) file_data.close() bar.next() bar.finish() file.close() # Close the data file return file_data_name
def make_long_spectra_files_from_wf(directory, fileList, result_folder): ''' Makes fft and saves spectra to the long data files ''' # Preparing long data files fname = directory + fileList[0] [df_filename, df_filesize, df_system_name, df_obs_place, df_description, CLCfrq, df_creation_timeUTC, Channel, ReceiverMode, Mode, Navr, TimeRes, fmin, fmax, df, frequency, FreqPointsNum, data_block_size] = FileHeaderReaderJDS(fname, 0, 1) no_of_blocks_in_file = (df_filesize - 1024) / data_block_size print(' Number of blocks in file: ', no_of_blocks_in_file) no_of_blocks_in_batch = int(no_of_blocks_in_file / (2 * no_of_batches_in_file)) print(' Number of blocks in batch: ', no_of_blocks_in_batch) with open(fname, 'rb') as file: # *** Data file header read *** file_header = file.read(1024) # *** Creating a name for long timeline TXT file *** TLfile_name = result_folder + '/' + df_filename + '_Timeline.txt' TLfile = open(TLfile_name, 'w') # Open and close to delete the file with the same name TLfile.close() # *** Creating a binary file with data for long data storage *** file_data_re_name = result_folder + '/' + df_filename + '_Data_WRe.dat' file_data_re = open(file_data_re_name, 'wb') file_data_re.write(file_header) file_data_re.close() file_data_im_name = result_folder + '/' + df_filename + '_Data_WIm.dat' file_data_im = open(file_data_im_name, 'wb') file_data_im.write(file_header) file_data_im.close() for fileNo in range(len(fileList)): # loop by files #print('\n\n\n * File ', str(fileNo + 1), ' of', str(len(fileList))) #print(' * File path: ', str(fileList[fileNo])) # *** Opening datafile *** fname = directory + fileList[fileNo] # ********************************************************************************* # *** Data file header read *** [df_filename, df_filesize, df_system_name, df_obs_place, df_description, CLCfrq, df_creation_timeUTC, Channel, ReceiverMode, Mode, Navr, TimeRes, fmin, fmax, df, frequency, FreqPointsNum, data_block_size] = FileHeaderReaderJDS(fname, 0, 0) # ******************************************************************************* # R E A D I N G D A T A * # ******************************************************************************* #print('\n *** Reading data from file *** \n') with open(fname, 'rb') as file: file.seek(1024) # Jumping to 1024 byte from file beginning #+ (sizeOfChunk+8) * chunkSkip TimeScaleFull = [] bar = IncrementalBar(' File ' + str(fileNo + 1) + ' of ' + str(len(fileList)) + ' progress: ', max=no_of_batches_in_file, suffix='%(percent)d%%') for batch in range(no_of_batches_in_file): # bar.next() # Reading and reshaping all data with readers if Channel == 0 or Channel == 1: # Single channel mode wf_data = np.fromfile(file, dtype='i2', count=no_of_blocks_in_batch * data_block_size) wf_data = np.reshape(wf_data, [data_block_size, no_of_blocks_in_batch], order='F') # Timing timeline_block_str = jds_waveform_time(wf_data, CLCfrq, data_block_size) #TimeScaleFig.append(timeline_block_str[-1][0:12]) for j in range (no_of_blocks_in_batch): TimeScaleFull.append(df_creation_timeUTC[0:10] + ' ' + timeline_block_str[j][0:12]) # Nulling the time blocks in waveform data wf_data[data_block_size - 4: data_block_size, :] = 0 # Scaling of the data - seems to be wrong in absolute value wf_data = wf_data / 32768.0 spectra_chA = np.zeros([data_block_size, no_of_blocks_in_batch], dtype=complex) for i in range(no_of_blocks_in_batch): spectra_chA[:, i] = np.fft.fft(wf_data[:, i]) # Storing only second (right) mirror part of spectra spectra_chA = spectra_chA[0: int(data_block_size / 2), :] if batch == 0: plt.figure(1, figsize=(10.0, 6.0)) plt.subplots_adjust(left=None, bottom=0, right=None, top=0.86, wspace=None, hspace=None) plt.plot(np.log10(np.power(np.abs(spectra_chA[:, 0]), 2)), label='First specter') plt.title('Title', fontsize=10, fontweight='bold', style='italic', y=1.025) plt.legend(loc='upper right', fontsize=10) plt.ylabel('Amplitude, a.u.', fontsize=10, fontweight='bold') plt.xlabel('Frequency, counts', fontsize=10, fontweight='bold') plt.yticks(fontsize=8, fontweight='bold') plt.xticks(fontsize=8, fontweight='bold') pylab.savefig(result_folder + '/Fig. 1.png', bbox_inches='tight', dpi=customDPI) plt.close('all') temp = np.real(spectra_chA).copy(order='C') file_data_re = open(file_data_re_name, 'ab') file_data_re.write(temp) file_data_re.close() temp = np.imag(spectra_chA).copy(order='C') file_data_im = open(file_data_im_name, 'ab') file_data_im.write(temp) file_data_im.close() # Saving time data to ling timeline file with open(TLfile_name, 'a') as TLfile: for i in range(no_of_blocks_in_batch): TLfile.write((TimeScaleFull[i][:]) + ' \n') return file_data_re_name, file_data_im_name, TLfile_name
def convert_wf32_to_dat_without_overlap(fname, no_of_points_for_fft_spectr, no_of_spectra_in_bunch): """ Converts waveform data in .wf32 format to spectra in .dat format Input parameters: fname - name of .wf32 file with waveform data no_of_points_for_fft - number of points for FFT to provide necessary time-frequency resolution Output parameters: file_data_name - name of .dat file with result spectra """ # *** Data file header read *** [ df_filename, df_filesize, df_system_name, df_obs_place, df_description, clock_freq, df_time_utc, channel, receiver_mode, mode, n_avr, time_resolution, fmin, fmax, df, frequency, freq_points_num, data_block_size ] = FileHeaderReaderJDS(fname, 0, 0) freq_points_num = int(no_of_points_for_fft_spectr / 2) with open(fname, 'rb') as file: # Data file header read file_header = file.read(1024) # Creating a binary file with spectra data for long data storage file_data_name = fname[:-5] + '.dat' file_data = open(file_data_name, 'wb') file_data.write(file_header) file_data.seek(574) # FFT size place in header file_data.write(np.int32(no_of_points_for_fft_spectr).tobytes()) file_data.seek(624) # Lb place in header file_data.write(np.int32(0).tobytes()) file_data.seek(628) # Hb place in header file_data.write(np.int32(freq_points_num).tobytes()) file_data.seek(632) # Wb place in header file_data.write(np.int32(freq_points_num).tobytes()) file_data.seek(636) # Navr place in header file_data.write(np.int32(1).tobytes()) # Works fine file_data.close() del file_header # Calculation of number of blocks and number of spectra in the file no_of_bunches_per_file = int( (df_filesize - 1024) / (no_of_spectra_in_bunch * no_of_points_for_fft_spectr * 4)) # Real time resolution of averaged spectra fine_clock_freq = (int(clock_freq / 1000000.0) * 1000000.0) real_spectra_dt = float(no_of_points_for_fft_spectr / fine_clock_freq) real_spectra_df = float( (fine_clock_freq / 2) / (no_of_points_for_fft_spectr / 2)) print(' Number of spectra in bunch: ', no_of_spectra_in_bunch) print(' Number of bunches to read in file: ', no_of_bunches_per_file) print(' Time resolution of calculated spectra: ', round(real_spectra_dt * 1000, 3), ' ms') print(' Frequency resolution of calculated spectra: ', round(real_spectra_df / 1000, 3), ' kHz') print('\n *** Reading data from file *** \n') file.seek(1024) # Jumping to 1024 byte from file beginning # *** Creating a new timeline TXT file for results *** new_tl_file_name = file_data_name.split('_Data_', 1)[0] + '_Timeline.txt' new_tl_file = open( new_tl_file_name, 'w') # Open and close to delete the file with the same name new_tl_file.close() # *** Reading timeline file *** old_tl_file_name = fname.split("_Data_", 1)[0] + '_Timeline.wtxt' old_tl_file = open(old_tl_file_name, 'r') new_tl_file = open( new_tl_file_name, 'w') # Open and close to delete the file with the same name bar = IncrementalBar(' Conversion from waveform to spectra: ', max=no_of_bunches_per_file - 1, suffix='%(percent)d%%') for bunch in range(no_of_bunches_per_file - 1): bar.next() # Read time from timeline file for the bunch time_scale_bunch = [] for line in range(no_of_spectra_in_bunch): time_scale_bunch.append(str(old_tl_file.readline())) # Saving time data to new file for i in range(len(time_scale_bunch)): new_tl_file.write((time_scale_bunch[i][:]) + '') # Reading and reshaping data of the bunch wf_data = np.fromfile(file, dtype='f4', count=no_of_spectra_in_bunch * no_of_points_for_fft_spectr) wf_data = np.reshape( wf_data, [no_of_points_for_fft_spectr, no_of_spectra_in_bunch], order='F') # # # # # # window = np.hamming(no_of_points_for_fft_spectr) # for i in range(no_of_spectra_in_bunch): # wf_data[:, i] = wf_data[:, i] * window[:] # del window # # # # # # preparing matrices for spectra spectra = np.zeros_like(wf_data) # Calculation of spectra for i in range(no_of_spectra_in_bunch): spectra[:, i] = np.power(np.abs(np.fft.fft(wf_data[:, i])), 2) # Storing only first (left) mirror part of spectra spectra = spectra[:int(no_of_points_for_fft_spectr / 2), :] # At 33 MHz the specter is usually upside down, to correct it we use flip up/down if int(clock_freq / 1000000) == 33: spectra = np.flipud(spectra) # Saving spectra data to dat file temp = spectra.transpose().copy(order='C') file_data = open(file_data_name, 'ab') file_data.write(np.float64(temp)) file_data.close() bar.finish() file.close() # Close the data file return file_data_name
def obtain_calibr_matrix_for_2_channel_wf_calibration(path_to_calibr_data, no_of_points_for_fft): """ The function reads 2-channel waveform calibration files (UTR-2 noise generator calibration with a set of attenuators) calculates the cross-spectra of two channels in each file and provides a phase difference txt file for pulsar waveform observations calibration """ file_list = find_and_check_files_in_current_folder(path_to_calibr_data, '.jds') labels = [] cross_sp_ampl = [] cross_sp_angl = [] file_names = [] spectrum_ch_1 = [] spectrum_ch_2 = [] imed_spectrum_ch_1 = [] imed_spectrum_ch_2 = [] corr_f_abs = [] corr_f_ang = [] corr_f_re = [] result_path = 'RESULTS_WF_calibration_analyzer/' if not os.path.exists(result_path): os.makedirs(result_path) # Main loop by files start for file_no in range(len(file_list)): # loop by files fname = path_to_calibr_data + file_list[file_no] # *** Data file header read *** [ df_filename, df_filesize, df_system_name, df_obs_place, df_description, clock_freq, df_creation_timeUTC, channel, receiver_mode, Mode, Navr, time_res, fmin, fmax, df, frequency, freq_points_num, data_block_size ] = FileHeaderReaderJDS(fname, 0, 0) labels.append(df_system_name + ' ' + df_description.replace('_', ' ')) file_names.append(df_filename) print('\n Processing file: ', df_description.replace('_', ' '), ', # ', file_no + 1, ' of ', len(file_list), '\n') wf32_files = convert_one_jds_wf_to_wf32(fname, result_directory, 16) ampl_corr, angle_corr, av_sp_1, av_sp_2, sp_1, sp_2, cf_abs, cf_arg, cf_re = correlate_two_wf32_signals( wf32_files[0], wf32_files[1], no_of_points_for_fft, True, False) cross_sp_ampl.append(ampl_corr) cross_sp_angl.append(angle_corr) spectrum_ch_1.append(av_sp_1) spectrum_ch_2.append(av_sp_2) imed_spectrum_ch_1.append(sp_1) imed_spectrum_ch_2.append(sp_2) corr_f_abs.append(cf_abs) corr_f_ang.append(cf_arg) corr_f_re.append(cf_re) # Figures of initial and averaged spectra for each file for i in range(len(file_list)): # Figure of averaged and non-averaged spectra rc('font', size=10, weight='bold') fig = plt.figure(figsize=(18, 10)) fig.suptitle( 'Comparison of current and averaged cross spectra of waveform signals for ' + file_list[i] + ' (' + labels[i] + ')', fontsize=12, fontweight='bold') ax1 = fig.add_subplot(211) ax1.plot(imed_spectrum_ch_1[i], linestyle='-', linewidth='1.00', label='Current spectrum') ax1.plot(spectrum_ch_1[i], linestyle='-', linewidth='1.00', label='Averaged spectrum') ax1.set(xlim=(0, no_of_points_for_fft // 2)) ax1.legend(loc='upper right', fontsize=10) ax2 = fig.add_subplot(212) ax2.plot(imed_spectrum_ch_2[i], linestyle='-', linewidth='1.00', label='Current spectrum') ax2.plot(spectrum_ch_2[i], linestyle='-', linewidth='1.00', label='Averaged spectrum') ax2.set(xlim=(0, no_of_points_for_fft // 2)) ax2.set_xlabel('Frequency channels, #', fontsize=10, fontweight='bold') ax2.legend(loc='upper right', fontsize=10) fig.subplots_adjust(hspace=0.07, top=0.94) pylab.savefig(result_path + 'Signal_spectra_' + file_names[i] + '.png', bbox_inches='tight', dpi=160) plt.close('all') # Plot cross spectra matrix rc('font', size=10, weight='bold') fig = plt.figure(figsize=(18, 10)) fig.suptitle( 'Calibration matrix of waveform signals correlation for ' + file_list[i] + ' (' + labels[i] + ')', fontsize=12, fontweight='bold') ax1 = fig.add_subplot(211) ax1.set_title('Files: ' + file_names[0] + ' - ' + file_names[-1], fontsize=12) ax1.plot(np.log10(cross_sp_ampl[i]), linestyle='-', linewidth='1.30', label='Cross spectra amplitude') ax1.legend(loc='upper right', fontsize=10) ax1.set(xlim=(0, no_of_points_for_fft // 2)) ax1.set_ylabel('Amplitude, A.U.', fontsize=10, fontweight='bold') ax2 = fig.add_subplot(212) ax2.plot(cross_sp_angl[i], linestyle='-', linewidth='1.30', label='Cross spectra phase') ax2.set(xlim=(0, no_of_points_for_fft // 2)) ax2.set_xlabel('Frequency channels, #', fontsize=10, fontweight='bold') ax2.set_ylabel('Phase, rad', fontsize=10, fontweight='bold') ax2.legend(loc='upper right', fontsize=10) fig.subplots_adjust(hspace=0.07, top=0.94) pylab.savefig(result_path + 'WF_signal_correlation_' + file_names[i] + '.png', bbox_inches='tight', dpi=160) plt.close('all') # Plot calibration spectra matrix rc('font', size=10, weight='bold') fig = plt.figure(figsize=(18, 10)) fig.suptitle('Calibration matrix of waveform signals', fontsize=12, fontweight='bold') ax1 = fig.add_subplot(211) ax1.set_title('Files: ' + file_names[0] + ' - ' + file_names[-1], fontsize=12) for i in range(len(spectrum_ch_1)): ax1.plot(spectrum_ch_1[i], linestyle='-', linewidth='1.30', label=labels[i]) ax1.legend(loc='upper right', fontsize=10) ax1.set(xlim=(0, no_of_points_for_fft // 2)) ax1.set_ylabel('Signal, A.U.', fontsize=10, fontweight='bold') ax2 = fig.add_subplot(212) for i in range(len(spectrum_ch_2)): ax2.plot(spectrum_ch_2[i], linestyle='-', linewidth='1.30', label=labels[i]) ax2.set(xlim=(0, no_of_points_for_fft // 2)) ax2.set_xlabel('Frequency channels, #', fontsize=10, fontweight='bold') ax2.set_ylabel('Signal, A.U.', fontsize=10, fontweight='bold') ax2.legend(loc='upper right', fontsize=10) fig.subplots_adjust(hspace=0.07, top=0.94) pylab.savefig(result_path + 'Calibration_matrix_wf_spectra.png', bbox_inches='tight', dpi=160) plt.close('all') # Plot cross spectra matrix rc('font', size=10, weight='bold') fig = plt.figure(figsize=(18, 10)) fig.suptitle('Cross spectra matrix of waveform signals correlation', fontsize=12, fontweight='bold') ax1 = fig.add_subplot(211) ax1.set_title('Files: ' + file_names[0] + ' - ' + file_names[-1], fontsize=12) for i in range(len(cross_sp_ampl)): ax1.plot(np.log10(cross_sp_ampl[i]), linestyle='-', linewidth='1.30', label=labels[i]) ax1.legend(loc='upper right', fontsize=10) ax1.set(xlim=(0, no_of_points_for_fft // 2)) ax1.set_ylabel('Amplitude, A.U.', fontsize=10, fontweight='bold') ax2 = fig.add_subplot(212) for i in range(len(cross_sp_angl)): ax2.plot(cross_sp_angl[i], linestyle='-', linewidth='1.30', label=labels[i]) ax2.set(xlim=(0, no_of_points_for_fft // 2)) ax2.set_xlabel('Frequency channels, #', fontsize=10, fontweight='bold') ax2.set_ylabel('Phase, rad', fontsize=10, fontweight='bold') ax2.legend(loc='upper right', fontsize=10) fig.subplots_adjust(hspace=0.07, top=0.94) pylab.savefig(result_path + 'Calibration_matrix_wf_cross_spectra.png', bbox_inches='tight', dpi=160) plt.close('all') # Plot correlation matrix rc('font', size=10, weight='bold') fig = plt.figure(figsize=(18, 10)) fig.suptitle('Cross spectra matrix of waveform signals correlation', fontsize=12, fontweight='bold') ax1 = fig.add_subplot(211) ax1.set_title('Files: ' + file_names[0] + ' - ' + file_names[-1], fontsize=12) for i in range(len(corr_f_abs)): ax1.plot(np.log10(corr_f_abs[i]), linestyle='-', linewidth='1.30', label=labels[i]) ax1.legend(loc='upper right', fontsize=10) ax1.set(xlim=(0, no_of_points_for_fft // 1)) ax1.set_ylabel('Amplitude, A.U.', fontsize=10, fontweight='bold') ax2 = fig.add_subplot(212) for i in range(len(corr_f_ang)): ax2.plot(corr_f_ang[i], linestyle='-', linewidth='1.30', label=labels[i]) ax2.set(xlim=(0, no_of_points_for_fft // 1)) ax2.set_xlabel('Frequency channels, #', fontsize=10, fontweight='bold') ax2.set_ylabel('Phase, rad', fontsize=10, fontweight='bold') ax2.legend(loc='upper right', fontsize=10) fig.subplots_adjust(hspace=0.07, top=0.94) pylab.savefig(result_path + 'Calibration_matrix_wf_correlation.png', bbox_inches='tight', dpi=160) plt.close('all') # Plot mutual correlation function rc('font', size=10, weight='bold') fig = plt.figure(figsize=(18, 10)) fig.suptitle('Mutual correlation matrix of waveform signals', fontsize=12, fontweight='bold') ax1 = fig.add_subplot(111) ax1.set_title('Files: ' + file_names[0] + ' - ' + file_names[-1], fontsize=12) for i in range(len(corr_f_re)): ax1.plot(corr_f_re[i], linestyle='-', linewidth='1.30', label=labels[i]) ax1.legend(loc='upper right', fontsize=10) ax1.set(xlim=(0, no_of_points_for_fft // 1)) ax1.set_ylabel('Amplitude, A.U.', fontsize=10, fontweight='bold') fig.subplots_adjust(top=0.94) pylab.savefig(result_path + 'Calibration_matrix_of_wf_mutual_correlation_function.png', bbox_inches='tight', dpi=160) plt.close('all') # Save phase matrix to txt files for i in range(len(file_list)): phase_txt_file = open( result_path + 'Calibration_' + file_names[i] + '_cross_spectra_phase.txt', "w") for freq in range(no_of_points_for_fft // 1): # //2 phase_txt_file.write( ''.join(' {:+12.7E}'.format(cross_sp_angl[i][freq])) + ' \n') phase_txt_file.close() return 0
def coherent_wf_to_wf_dedispersion(DM, fname, no_of_points_for_fft_dedisp): """ function reads waveform data in wf32 format, makes FFT, cuts the symmetrical half of the spectra and shifts the lines of complex data to provide coherent dedispersion. Then a symmetrcal part of spectra are made and joined to the shifted one, inverse FFT as applied and data are stored in waveform wf32 format Input parameters: DM - dispersion measure to compensate fname - name of file with initial wf32 data no_of_points_for_fft_dedisp - number of waveform data points to use for FFT Output parameters: file_data_name - name of file with processed data """ # *** Data file header read *** [ df_filename, df_filesize, df_system_name, df_obs_place, df_description, clock_freq, df_creation_timeUTC, Channel, ReceiverMode, Mode, Navr, time_resolution, fmin, fmax, df, frequency_list, freq_points_num, data_block_size ] = FileHeaderReaderJDS(fname, 0, 0) # Manually set frequencies for one channel mode freq_points_num = int(no_of_points_for_fft_dedisp / 2) # Manually set frequencies for 33 MHz clock frequency if int(clock_freq / 1000000) == 33: fmin = 16.5 fmax = 33.0 df = 16500000 / freq_points_num # Create long data files and copy first data file header to them with open(fname, 'rb') as file: # *** Data file header read *** file_header = file.read(1024) # Removing old DM from file name and updating it to current value if fname.startswith('DM_'): prev_dm_str = fname.split('_')[1] prev_dm = np.float32(prev_dm_str) new_dm = prev_dm + DM n = len('DM_' + prev_dm_str + '_') file_data_name = 'DM_' + str(np.round(new_dm, 6)) + '_' + fname[n:] else: file_data_name = 'DM_' + str(np.round(DM, 6)) + '_' + fname # *** Creating a binary file with data for long data storage *** file_data = open(file_data_name, 'wb') file_data.write(file_header) file_data.close() del file_header # *** Creating a new timeline TXT file for results *** new_tl_file_name = file_data_name.split("_Data_ch", 1)[0] + '_Timeline.wtxt' new_tl_file = open( new_tl_file_name, 'w') # Open and close to delete the file with the same name new_tl_file.close() # Calculation of the time shifts shift_vector = DM_full_shift_calc(freq_points_num, fmin, fmax, df / pow(10, 6), time_resolution, DM, 'jds') max_shift = np.abs(shift_vector[0]) # Preparing buffer array buffer_array = np.zeros((freq_points_num, 2 * max_shift), dtype='complex64') print(' Maximal shift is: ', max_shift, ' pixels ') print(' Dispersion measure: ', DM, ' pc / cm3 ') # Calculation of number of blocks and number of spectra in the file no_of_spectra_in_bunch = max_shift.copy() no_of_bunches_per_file = int( np.ceil( (df_filesize - 1024) / (no_of_spectra_in_bunch * no_of_points_for_fft_dedisp * 4))) # Real time resolution of spectra fine_clock_freq = (int(clock_freq / 1000000.0) * 1000000.0) real_spectra_dt = float(no_of_points_for_fft_dedisp / fine_clock_freq) real_spectra_df = float( (fine_clock_freq / 2) / (no_of_points_for_fft_dedisp / 2)) print(' Number of spectra in bunch: ', no_of_spectra_in_bunch) print(' Number of bunches to read in file: ', no_of_bunches_per_file) print(' Time resolution of calculated spectra: ', round(real_spectra_dt * 1000, 3), ' ms') print(' Frequency resolution of calculated spectra: ', round(real_spectra_df / 1000, 3), ' kHz \n') # !!! Fake timing. Real timing to be done!!! # *** Reading timeline file *** old_tl_file_name = fname.split("_Data_ch", 1)[0] + '_Timeline.wtxt' old_tl_file = open(old_tl_file_name, 'r') new_tl_file = open( new_tl_file_name, 'w') # Open and close to delete the file with the same name file.seek(1024) # Jumping to 1024 byte from file beginning # bar = IncrementalBar(' Coherent dispersion delay removing: ', max=no_of_bunches_per_file - 1, bar = IncrementalBar(' Coherent dispersion delay removing: ', max=no_of_bunches_per_file, suffix='%(percent)d%%') bar.start() # for bunch in range(no_of_bunches_per_file - 1): for bunch in range(no_of_bunches_per_file): # Trying to read all the file, not only integer number of bunches if bunch >= no_of_bunches_per_file - 1: no_of_spectra_in_bunch = int( ((df_filesize - 1024) - bunch * max_shift * no_of_points_for_fft_dedisp * 4) / (no_of_points_for_fft_dedisp * 4)) # print('\n Bunch No ', str(bunch+1), ' of ', no_of_bunches_per_file, ' bunches') # print('\n Number of spectra in the last bunch is: ', no_of_spectra_in_bunch) # print('\n Maximal shift is: ', max_shift) # Read time from timeline file for the bunch time_scale_bunch = [] for line in range(no_of_spectra_in_bunch): time_scale_bunch.append(str(old_tl_file.readline())) # Reading and reshaping all data with time data wf_data = np.fromfile(file, dtype='f4', count=no_of_spectra_in_bunch * no_of_points_for_fft_dedisp) ''' fig = plt.figure(figsize=(9, 5)) ax1 = fig.add_subplot(111) ax1.plot(wf_data, linestyle='-', linewidth='1.00', label='Initial waveform') ax1.legend(loc='upper right', fontsize=6) ax1.grid(b=True, which='both', color='silver', linestyle='-') ax1.set_ylabel('Intensity, a.u.', fontsize=6, fontweight='bold') pylab.savefig('00_Initial_waveform_' + str(bunch) + '.png', bbox_inches='tight', dpi=160) plt.close('all') ''' wf_data = np.reshape( wf_data, [no_of_points_for_fft_dedisp, no_of_spectra_in_bunch], order='F') # preparing matrices for spectra spectra = np.zeros( (no_of_points_for_fft_dedisp, no_of_spectra_in_bunch), dtype='complex64') # Calculation of spectra for i in range(no_of_spectra_in_bunch): spectra[:, i] = np.fft.fft(wf_data[:, i]) del wf_data ''' fig = plt.figure(figsize=(9, 5)) ax1 = fig.add_subplot(111) ax1.plot(10 * np.log10(np.power(np.abs(spectra[:, 0]), 2)), linestyle='-', linewidth='1.00', label='Initial spectra before cut') ax1.legend(loc='upper right', fontsize=6) ax1.grid(b=True, which='both', color='silver', linestyle='-') ax1.set_ylabel('Intensity, a.u.', fontsize=6, fontweight='bold') pylab.savefig('00a_Initial_doubled_imm_spectra' + str(bunch) + '.png', bbox_inches='tight', dpi=160) plt.close('all') ''' # Cut half of the spectra spectra = spectra[int(no_of_points_for_fft_dedisp / 2):, :] ''' # making figures fig = plt.figure(figsize=(9, 5)) ax1 = fig.add_subplot(111) ax1.imshow(np.flipud(10*np.log10(np.power(np.abs(spectra), 2))), aspect='auto', cmap='jet') ax1.set_ylabel('Frequency points', fontsize=6, fontweight='bold') ax1.set_xlabel('Time points', fontsize=6, fontweight='bold') pylab.savefig('01_Initial_spectra_' + str(bunch) + '.png', bbox_inches='tight', dpi=160) plt.close('all') fig = plt.figure(figsize=(9, 5)) ax1 = fig.add_subplot(111) ax1.plot(10*np.log10(np.power(np.abs(spectra[:, 0]), 2)), linestyle='-', linewidth='1.00', label='Initial waveform') ax1.legend(loc='upper right', fontsize=6) ax1.grid(b=True, which='both', color='silver', linestyle='-') ax1.set_ylabel('Intensity, a.u.', fontsize=6, fontweight='bold') pylab.savefig('02_Initial_imm_spectra' + str(bunch) + '.png', bbox_inches='tight', dpi=160) plt.close('all') ''' # Dispersion delay removing data_space = np.zeros((freq_points_num, 2 * max_shift), dtype='complex64') # if it is the last bunch - use only availble data if bunch >= no_of_bunches_per_file - 1: data_space[:, max_shift:max_shift + no_of_spectra_in_bunch] = spectra[:, :] else: data_space[:, max_shift:] = spectra[:, :] data_space = pulsar_DM_compensation_with_indices_changes( data_space, shift_vector) del spectra # Adding the next data block buffer_array += data_space # Making and filling the array with fully ready data for plotting and saving to a file if bunch >= no_of_bunches_per_file - 1: array_compensated_dm = buffer_array[:, 0:no_of_spectra_in_bunch] else: array_compensated_dm = buffer_array[:, 0:max_shift] if bunch > 0: # Saving time data to a new file for i in range(len(time_scale_bunch)): new_tl_file.write((time_scale_bunch[i][:]) + '') # Saving data with compensated DM spectra = array_compensated_dm # .copy() ''' # making figures fig = plt.figure(figsize=(9, 5)) ax1 = fig.add_subplot(111) ax1.imshow(np.flipud(10*np.log10(np.power(np.abs(spectra), 2))), aspect='auto', cmap='jet') ax1.set_ylabel('Frequency points', fontsize=6, fontweight='bold') ax1.set_xlabel('Time points', fontsize=6, fontweight='bold') pylab.savefig('03_Compensated_spectra_' + str(bunch) + '.png', bbox_inches='tight', dpi=160) plt.close('all') fig = plt.figure(figsize=(9, 5)) ax1 = fig.add_subplot(111) ax1.plot(10*np.log10(np.power(np.abs(spectra[:,0]), 2)), linestyle='-', linewidth='1.00', label='Initial waveform') ax1.legend(loc='upper right', fontsize=6) ax1.grid(b=True, which='both', color='silver', linestyle='-') ax1.set_ylabel('Intensity, a.u.', fontsize=6, fontweight='bold') pylab.savefig('04_Compensated_imm_spectra' + str(bunch) + '.png', bbox_inches='tight', dpi=160) plt.close('all') ''' wf_data = np.zeros( (no_of_points_for_fft_dedisp, no_of_spectra_in_bunch)) # Add lost half of the spectra second_spectra_half = spectra.copy() second_spectra_half = np.flipud(second_spectra_half) spectra = np.concatenate((second_spectra_half, spectra), axis=0) # Changed places!!! ''' fig = plt.figure(figsize=(9, 5)) ax1 = fig.add_subplot(111) ax1.plot(10*np.log10(np.power(np.abs(spectra[:,0]), 2)), linestyle='-', linewidth='1.00', label='Initial waveform') ax1.legend(loc='upper right', fontsize=6) ax1.grid(b=True, which='both', color='silver', linestyle='-') ax1.set_ylabel('Intensity, a.u.', fontsize=6, fontweight='bold') pylab.savefig('05_Compensated_doubled_imm_spectra' + str(bunch) + '.png', bbox_inches='tight', dpi=160) plt.close('all') ''' # Making IFFT for i in range(no_of_spectra_in_bunch): wf_data[:, i] = np.real(np.fft.ifft(spectra[:, i])) del spectra # Reshaping the waveform to single dimension (real) wf_data = np.reshape( wf_data, [no_of_points_for_fft_dedisp * no_of_spectra_in_bunch, 1], order='F') ''' # making figures fig = plt.figure(figsize=(9, 5)) ax1 = fig.add_subplot(111) ax1.plot(wf_data, linestyle='-', linewidth='1.00', label='Initial waveform') ax1.legend(loc='upper right', fontsize=6) ax1.grid(b=True, which='both', color='silver', linestyle='-') ax1.set_ylabel('Intensity, a.u.', fontsize=6, fontweight='bold') pylab.savefig('06_Compensated_waveform_' + str(bunch) + '.png', bbox_inches='tight', dpi=160) plt.close('all') ''' # Saving waveform data to wf32 file file_data = open(file_data_name, 'ab') file_data.write( np.float32(wf_data).transpose().copy(order='C')) file_data.close() # !!! Saving time data to timeline file !!! # Rolling temp_array to put current data first buffer_array = np.roll(buffer_array, -max_shift) buffer_array[:, max_shift:] = 0 bar.next() bar.finish() old_tl_file.close() new_tl_file.close() return file_data_name
if filename[0:3] == 'ADR': [df_filename, df_filesize, df_system_name, df_obs_place, df_description, F_ADC, df_creation_timeUTC, ReceiverMode, ADRmode, sumDifMode, NAvr, TimeRes, fmin, fmax, df, frequency_list, FFTsize, SLine, Width, BlockSize] = FileHeaderReaderADR(filepath, smd_filesize - 1024 - 131096, 1) record_date_time_dt = datetime(int('20' + df_filename[1:3]), int(df_filename[3:5]), int(df_filename[5:7]), int(df_creation_timeUTC[0:2]), int(df_creation_timeUTC[3:5]), int(df_creation_timeUTC[6:8]), int(df_creation_timeUTC[9:12]) * 1000) record_date_time = str(record_date_time_dt) telescope = 'GURT' if filename[0:3] == 'DSP': [df_filename, df_filesize, df_system_name, df_obs_place, df_description, CLCfrq, df_creation_timeUTC, SpInFile, ReceiverMode, Mode, Navr, TimeRes, fmin, fmax, df, frequency_list, FFTsize, BlockSize] = FileHeaderReaderJDS(filepath, smd_filesize - 1024, 1) telescope = 'UTR-2' record_date_time_dt = datetime(int('20' + df_filename[5:7]), int(df_filename[3:5]), int(df_filename[1:3]), int(df_creation_timeUTC[11:13]), int(df_creation_timeUTC[14:16]), int(df_creation_timeUTC[17:19]), 0) record_date_time = str(record_date_time_dt) df = df / pow(10,6) freq_num = len(frequency_list) file = open(filepath, 'rb') # *** Reading pulsar period and number of samples per period ***
def JDS_file_reader(fileList, result_path, MaxNsp, spSkip, RFImeanConst, Vmin, Vmax, VminNorm, VmaxNorm, VminCorrMag, VmaxCorrMag, colormap, customDPI, CorrelationProcess, longFileSaveAch, longFileSaveBch, longFileSaveCRI, longFileSaveCMP, DynSpecSaveInitial, DynSpecSaveCleaned, CorrSpecSaveInitial, CorrSpecSaveCleaned, SpecterFileSaveSwitch, ImmediateSpNo): currentTime = time.strftime("%H:%M:%S") currentDate = time.strftime("%d.%m.%Y") # *** Creating a folder where all pictures and results will be stored (if it doen't exist) *** if not os.path.exists(result_path): os.makedirs(result_path) if not os.path.exists(result_path + '/Service'): os.makedirs(result_path + '/Service') if DynSpecSaveInitial == 1: if not os.path.exists(result_path + '/Initial_spectra'): os.makedirs(result_path + '/Initial_spectra') if (DynSpecSaveCleaned == 1 and CorrelationProcess == 1): if not os.path.exists(result_path + '/Correlation_spectra'): os.makedirs(result_path + '/Correlation_spectra') # Main loop for fileNo in range(len(fileList)): # loop by files print('\n\n\n * File ', str(fileNo + 1), ' of', str(len(fileList))) print(' * File path: ', str(fileList[fileNo])) #********************************************************************************* # *** Opening datafile *** fname = '' if len(fname) < 1: fname = fileList[fileNo] # *** Data file header read *** [ df_filename, df_filesize, df_system_name, df_obs_place, df_description, CLCfrq, df_creation_timeUTC, SpInFile, ReceiverMode, Mode, Navr, TimeRes, fmin, fmax, df, frequency, FreqPointsNum, dataBlockSize ] = FileHeaderReaderJDS(fname, 0, 0) # Initial time line settings TimeScaleStartDate = datetime(int(df_creation_timeUTC[0:4]), int(df_creation_timeUTC[5:7]), int(df_creation_timeUTC[8:10]), 0, 0, 0, 0) timeLineMS = np.zeros( int(SpInFile)) # List of ms values from ends of spectra # *** Creating a name for long timeline TXT file *** if fileNo == 0 and (longFileSaveAch == 1 or longFileSaveBch == 1 or longFileSaveCRI == 1 or longFileSaveCMP == 1): TLfile_name = df_filename + '_Timeline.txt' TLfile = open( TLfile_name, 'wb') # Open and close to delete the file with the same name TLfile.close() with open(fname, 'rb') as file: # *** If it is the first file - write the header to long data file if ((longFileSaveAch == 1 or longFileSaveBch == 1 or longFileSaveCRI == 1 or longFileSaveCMP == 1) and fileNo == 0): file.seek(0) file_header = file.read(1024) DAT_file_name = df_filename DAT_file_list = [] # *** Creating a binary file with data for long data storage *** if ((Mode == 1 or Mode == 2) and longFileSaveAch == 1): Data_A_name = df_filename + '_Data_chA.dat' Data_AFile = open(Data_A_name, 'wb') Data_AFile.write(file_header) Data_AFile.close() DAT_file_list.append('chA') if (longFileSaveBch == 1 and (Mode == 1 or Mode == 2)): Data_B_name = df_filename + '_Data_chB.dat' Data_BFile = open(Data_B_name, 'wb') Data_BFile.write(file_header) Data_BFile.close() DAT_file_list.append('chB') if (longFileSaveCRI == 1 and CorrelationProcess == 1 and Mode == 2): Data_CRe_name = df_filename + '_Data_CRe.dat' Data_CReFile = open(Data_CRe_name, 'wb') Data_CReFile.write(file_header) Data_CReFile.close() DAT_file_list.append('CRe') Data_CIm_name = df_filename + '_Data_CIm.dat' Data_CImFile = open(Data_CIm_name, 'wb') Data_CImFile.write(file_header) Data_CImFile.close() DAT_file_list.append('CIm') if (longFileSaveCMP == 1 and CorrelationProcess == 1 and Mode == 2): Data_Cm_name = df_filename + '_Data_C_m.dat' Data_CmFile = open(Data_Cm_name, 'wb') Data_CmFile.write(file_header) Data_CmFile.close() DAT_file_list.append('C_m') Data_Cp_name = df_filename + '_Data_C_p.dat' Data_CpFile = open(Data_Cp_name, 'wb') Data_CpFile.write(file_header) Data_CpFile.close() DAT_file_list.append('C_p') del file_header #******************************************************************************* # R E A D I N G D A T A * #******************************************************************************* #print ('\n *** Reading data from file *** \n') file.seek(1024) # Jumping to 1024 byte from file beginning if Mode == 0: print( '\n\n Data in waveform mode, use appropriate program!!! \n\n\n' ) if Mode > 0 and Mode < 3: # Spectra modes figID = -1 figMAX = int(math.ceil((SpInFile - spSkip) / MaxNsp)) if figMAX < 1: figMAX = 1 for fig in range(figMAX): Time1 = time.time() # Timing figID = figID + 1 currentTime = time.strftime("%H:%M:%S") print(' File # ', str(fileNo + 1), ' of ', str(len(fileList)), ', figure # ', figID + 1, ' of ', figMAX, ' started at: ', currentTime) if (SpInFile - spSkip - MaxNsp * figID) < MaxNsp: Nsp = int(SpInFile - spSkip - MaxNsp * figID) else: Nsp = MaxNsp # *** Preparing empty matrices *** if Mode == 1 or Mode == 2: Data_ChA = np.zeros((Nsp, FreqPointsNum)) if Mode == 1 or Mode == 2: Data_ChB = np.zeros((Nsp, FreqPointsNum)) if Mode == 2: Data_CRe = np.zeros((Nsp, FreqPointsNum)) Data_CIm = np.zeros((Nsp, FreqPointsNum)) CorrModule = np.zeros((Nsp, FreqPointsNum)) CorrPhase = np.zeros((Nsp, FreqPointsNum)) # *** Reading and reshaping all data for figure *** if Mode == 1: raw = np.fromfile(file, dtype='u4', count=(2 * Nsp * FreqPointsNum)) raw = np.reshape(raw, [2 * FreqPointsNum, Nsp], order='F') Data_ChA = raw[0:(FreqPointsNum * 2):2, :].transpose() Data_ChB = raw[1:(FreqPointsNum * 2):2, :].transpose() if Mode == 2: raw = np.fromfile(file, dtype='u4', count=(4 * Nsp * FreqPointsNum)) raw = np.reshape(raw, [4 * FreqPointsNum, Nsp], order='F') Data_ChA = raw[0:(FreqPointsNum * 4):4, :].transpose() Data_ChB = raw[1:(FreqPointsNum * 4):4, :].transpose() Data_CRe = raw[2:(FreqPointsNum * 4):4, :].transpose() Data_CIm = raw[3:(FreqPointsNum * 4):4, :].transpose() del raw # *** Single out timing from data *** counterA2 = np.uint64(Data_ChA[:, -1]) counterB2 = np.uint64(Data_ChB[:, -1]) counterA1 = np.uint64(Data_ChA[:, -2]) counterB1 = np.uint64(Data_ChB[:, -2]) A = np.uint64(int('01111111111111111111111111111111', 2)) msCount = np.uint32(np.bitwise_and( counterB2, A)) # number of ms since record started ftCount = np.uint32(np.bitwise_and( counterA2, A)) # number of specter since record started A = np.uint64(int('00000111111111111111111111111111', 2)) phaOfSec = np.uint32(np.bitwise_and( counterA1, A)) # phase of second for the spectr A = np.uint64(int('00000000000000011111111111111111', 2)) secOfDay = np.uint32(np.bitwise_and( counterB1, A)) # second of the day for the specter # *** Time line arranging *** # Preparing/cleaning matrices for time scales TimeScale = [] # New for each file TimeFigureScale = [ ] # Timelime (new) for each figure (Nsp) # Calculations FigStartTime = timedelta( 0, int(secOfDay[0]), int(1000000 * phaOfSec[0] / CLCfrq)) for i in range(Nsp): TimeAdd = timedelta( 0, int(secOfDay[i]), int(1000000 * phaOfSec[i] / CLCfrq)) TimeScale.append(str(str(TimeScaleStartDate + TimeAdd))) TimeFigureScale.append(str((TimeAdd - FigStartTime))) TimeFigureScaleFig = np.empty_like(TimeFigureScale) TimeScaleFig = np.empty_like(TimeScale) for i in range(len(TimeFigureScale)): TimeFigureScaleFig[i] = TimeFigureScale[i][0:11] TimeScaleFig[i] = TimeScale[i][11:23] # *** Converting from FPGA to PC float format *** if Mode == 1 or Mode == 2: Data_ChA = FPGAtoPCarrayJDS(Data_ChA, Navr) Data_ChB = FPGAtoPCarrayJDS(Data_ChB, Navr) if (Mode == 2 and CorrelationProcess == 1): Data_CRe = FPGAtoPCarrayJDS(Data_CRe, Navr) Data_CIm = FPGAtoPCarrayJDS(Data_CIm, Navr) ''' # *** Absolute correlation specter plot *** if Mode == 2 and figID == 0: # Immediate correlation spectrum channels A & B TwoImmedSpectraPlot(frequency, Data_CRe[1][:], Data_CIm[1][:], 'Channel A', 'Channel B', frequency[0], frequency[FreqPointsNum-1], -0.001, 0.001, 'Frequency, MHz', 'Amplitude, dB', 'Immediate spectrum '+str(df_filename[0:18])+ ' channels A & B', 'Initial parameters: dt = '+str(round(TimeRes,3))+' Sec, df = '+str(round(df/1000,3))+' kHz', 'JDS_Results/Service/'+df_filename[0:14]+' Correlation Spectrum Re and Im before log.png') ''' # *** Saving data to a long-term file *** if (Mode == 1 or Mode == 2) and longFileSaveAch == 1: Data_AFile = open(Data_A_name, 'ab') Data_AFile.write(Data_ChA) Data_AFile.close() if (Mode == 1 or Mode == 2) and longFileSaveBch == 1: Data_BFile = open(Data_B_name, 'ab') Data_BFile.write(Data_ChB) Data_BFile.close() if Mode == 2 and longFileSaveCRI == 1 and CorrelationProcess == 1: Data_CReFile = open(Data_CRe_name, 'ab') Data_CReFile.write(np.float64(Data_CRe)) Data_CReFile.close() Data_CImFile = open(Data_CIm_name, 'ab') Data_CImFile.write(np.float64(Data_CIm)) Data_CImFile.close() if (longFileSaveAch == 1 or longFileSaveBch == 1 or longFileSaveCRI == 1 or longFileSaveCMP == 1): with open(TLfile_name, 'a') as TLfile: for i in range(Nsp): TLfile.write( (TimeScale[i][:] + ' \n')) #str.encode # *** Converting to logarythmic scale matrices *** if (Mode == 1 or Mode == 2): with np.errstate(invalid='ignore'): Data_ChA = 10 * np.log10(Data_ChA) Data_ChB = 10 * np.log10(Data_ChB) Data_ChA[np.isnan(Data_ChA)] = -120 Data_ChB[np.isnan(Data_ChB)] = -120 if (Mode == 2 and CorrelationProcess == 1): with np.errstate(invalid='ignore', divide='ignore'): CorrModule = 10 * np.log10( ((Data_CRe)**2 + (Data_CIm)**2)**(0.5)) CorrPhase = np.arctan2(Data_CIm, Data_CRe) CorrPhase[np.isnan(CorrPhase)] = 0 CorrModule[np.isinf(CorrModule)] = -135.5 # *** Saving correlation data to a long-term module and phase files *** if (Mode == 2 and CorrelationProcess == 1 and longFileSaveCMP == 1): Data_CmFile = open(Data_Cm_name, 'ab') Data_CmFile.write(np.float64(CorrModule)) Data_CmFile.close() Data_CpFile = open(Data_Cp_name, 'ab') Data_CpFile.write(np.float64(CorrPhase)) Data_CpFile.close() # *** Saving immediate spectrum to file *** if (SpecterFileSaveSwitch == 1 and figID == 0): SpFile = open( 'JDS_Results/Service/Specter_' + df_filename[0:14] + '.txt', 'w') for i in range(FreqPointsNum - 1): if Mode == 1: SpFile.write( str('{:10.6f}'.format(frequency[i])) + ' ' + str('{:16.10f}'.format( Data_ChA[ImmediateSpNo][i])) + ' ' + str('{:16.10f}'.format( Data_ChB[ImmediateSpNo][i])) + ' \n') if Mode == 2: SpFile.write( str(frequency[i]) + ' ' + str(Data_ChA[ImmediateSpNo][i]) + ' ' + str(Data_ChB[ImmediateSpNo][i]) + ' ' + str(Data_CRe[ImmediateSpNo][i]) + ' ' + str(Data_CIm[ImmediateSpNo][i]) + ' \n') SpFile.close() #******************************************************************************* # F I G U R E S * #******************************************************************************* # *** Plotting immediate spectra before cleaning and normalizing *** if (Mode == 1 or Mode == 2) and figID == 0: Suptitle = ('Immediate spectrum ' + str(df_filename[0:18]) + ' channels A & B') Title = ('Place: ' + str(df_obs_place) + ', Receiver: ' + str(df_system_name) + '. Initial parameters: dt = ' + str(round(TimeRes, 3)) + ' Sec, df = ' + str(round(df / 1000, 3)) + ' kHz ' + 'Description: ' + str(df_description)) Filename = ( result_path + '/Service/' + df_filename[0:14] + ' Channels A and B Immediate Spectrum before cleaning and normalizing.png' ) TwoOrOneValuePlot( 2, frequency, Data_ChA[0][:], Data_ChB[0][:], 'Channel A', 'Channel B', frequency[0], frequency[FreqPointsNum - 1], -120, -20, -120, -20, 'Frequency, MHz', 'Intensity, dB', 'Intensity, dB', Suptitle, Title, Filename, currentDate, currentTime, Software_version) if Mode == 2 and CorrelationProcess == 1 and figID == 0: Suptitle = ('Immediate correlation spectrum ' + str(df_filename[0:18]) + ' channels A & B') Title = ('Place: ' + str(df_obs_place) + ', Receiver: ' + str(df_system_name) + '. Initial parameters: dt = ' + str(round(TimeRes, 3)) + ' Sec, df = ' + str(round(df / 1000, 3)) + ' kHz ' + 'Description: ' + str(df_description)) Filename = ( result_path + '/Service/' + df_filename[0:14] + ' Channels A and B Correlation Immedaiate Spectrum before cleaning and normalizing.png' ) TwoOrOneValuePlot( 2, frequency, CorrModule[0][:], CorrPhase[0][:], 'Correlation module', 'Correlation phase', frequency[0], frequency[FreqPointsNum - 1], VminCorrMag, VmaxCorrMag, -4, 4, 'Frequency, MHz', 'Amplitude, dB', 'Phase, deg', Suptitle, Title, Filename, currentDate, currentTime, Software_version) # *** FIGURE Initial dynamic spectrum channels A and B *** if (Mode == 1 or Mode == 2) and DynSpecSaveInitial == 1: Suptitle = ('Dynamic spectrum (initial) ' + str(df_filename) + ' - Fig. ' + str(figID + 1) + ' of ' + str(figMAX) + '\n Initial parameters: dt = ' + str(round(TimeRes * 1000, 3)) + ' ms, df = ' + str(round(df / 1000., 3)) + ' kHz, Receiver: ' + str(df_system_name) + ', Place: ' + str(df_obs_place) + '\n' + ReceiverMode + ', Description: ' + str(df_description)) fig_file_name = (result_path + '/Initial_spectra/' + df_filename[0:14] + ' Initial dynamic spectrum fig.' + str(figID + 1) + '.png') TwoDynSpectraPlot(Data_ChA.transpose(), Data_ChB.transpose(), Vmin, Vmax, Vmin, Vmax, Suptitle, 'Intensity, dB', 'Intensity, dB', Nsp, TimeFigureScaleFig, TimeScaleFig, frequency, FreqPointsNum, colormap, 'Channel A', 'Channel B', fig_file_name, currentDate, currentTime, Software_version, customDPI) # *** FIGURE Initial correlation spectrum Module and Phase (python 3 new version) *** if (Mode == 2 and CorrSpecSaveInitial == 1 and CorrelationProcess == 1): Suptitle = ('Correlation dynamic spectrum (initial) ' + str(df_filename) + ' - Fig. ' + str(figID + 1) + ' of ' + str(figMAX) + '\n Initial parameters: dt = ' + str(round(TimeRes * 1000, 3)) + ' ms, df = ' + str(round(df / 1000., 3)) + ' kHz, Receiver: ' + str(df_system_name) + ', Place: ' + str(df_obs_place) + '\n' + ReceiverMode + ', Description: ' + str(df_description)) fig_file_name = (result_path + '/Correlation_spectra/' + df_filename[0:14] + ' Correlation dynamic spectrum fig.' + str(figID + 1) + '.png') TwoDynSpectraPlot(CorrModule.transpose(), CorrPhase.transpose(), VminCorrMag, VmaxCorrMag, -3.15, 3.15, Suptitle, 'Intensity, dB', 'Phase, rad', Nsp, TimeFigureScaleFig, TimeScaleFig, frequency, FreqPointsNum, colormap, 'Correlation module', 'Correlation phase', fig_file_name, currentDate, currentTime, Software_version, customDPI) # *** Normalizing amplitude-frequency responce *** if Mode == 1 or Mode == 2: Normalization_dB(Data_ChA, FreqPointsNum, Nsp) Normalization_dB(Data_ChB, FreqPointsNum, Nsp) if Mode == 2 and CorrelationProcess == 1 and CorrSpecSaveCleaned == 1: Normalization_dB(CorrModule, FreqPointsNum, Nsp) # *** Deleting cahnnels with strong RFI *** if Mode == 1 or Mode == 2: simple_channel_clean(Data_ChA, RFImeanConst) simple_channel_clean(Data_ChB, RFImeanConst) if Mode == 2 and CorrelationProcess == 1 and CorrSpecSaveCleaned == 1: simple_channel_clean(CorrModule, 2 * RFImeanConst) # *** Immediate spectra *** (only for first figure in data file) if (Mode == 1 or Mode == 2 ) and figID == 0: # Immediate spectrum channels A & B Suptitle = ( 'Cleaned and normalized immediate spectrum ' + str(df_filename[0:18]) + ' channels A & B') Title = ('Place: ' + str(df_obs_place) + ', Receiver: ' + str(df_system_name) + '. Initial parameters: dt = ' + str(round(TimeRes, 3)) + ' Sec, df = ' + str(round(df / 1000, 3)) + ' kHz ' + 'Description: ' + str(df_description)) Filename = ( result_path + '/Service/' + df_filename[0:14] + ' Channels A and B Immediate Spectrum after cleaning and normalizing.png' ) TwoOrOneValuePlot( 2, frequency, Data_ChA[1][:], Data_ChB[1][:], 'Channel A', 'Channel B', frequency[0], frequency[FreqPointsNum - 1], VminNorm - 5, VmaxNorm, VminNorm - 5, VmaxNorm, 'Frequency, MHz', 'Intensity, dB', 'Intensity, dB', Suptitle, Title, Filename, currentDate, currentTime, Software_version) # *** FIGURE Normalized dynamic spectrum channels A and B *** if (Mode == 1 or Mode == 2) and DynSpecSaveCleaned == 1: Suptitle = ('Dynamic spectrum (normalized) ' + str(df_filename) + ' - Fig. ' + str(figID + 1) + ' of ' + str(figMAX) + '\n Initial parameters: dt = ' + str(round(TimeRes * 1000, 3)) + ' ms, df = ' + str(round(df / 1000., 3)) + ' kHz, Receiver: ' + str(df_system_name) + ', Place: ' + str(df_obs_place) + '\n' + ReceiverMode + ', Description: ' + str(df_description)) fig_file_name = (result_path + '/' + df_filename[0:14] + ' Dynamic spectra fig.' + str(figID + 1) + '.png') TwoDynSpectraPlot( Data_ChA.transpose(), Data_ChB.transpose(), VminNorm, VmaxNorm, VminNorm, VmaxNorm, Suptitle, 'Intensity, dB', 'Intensity, dB', Nsp, TimeFigureScaleFig, TimeScaleFig, frequency, FreqPointsNum, colormap, 'Channel A', 'Channel B', fig_file_name, currentDate, currentTime, Software_version, customDPI) # *** FIGURE Normalized correlation spectrum Module and Phase *** if (Mode == 2 and CorrSpecSaveCleaned == 1 and CorrelationProcess == 1): Suptitle = ( 'Correlation dynamic spectrum (normalized) ' + str(df_filename) + ' - Fig. ' + str(figID + 1) + ' of ' + str(figMAX) + '\n Initial parameters: dt = ' + str(round(TimeRes * 1000, 3)) + ' ms, df = ' + str(round(df / 1000., 3)) + ' kHz, Receiver: ' + str(df_system_name) + ', Place: ' + str(df_obs_place) + '\n' + ReceiverMode + ', Description: ' + str(df_description)) fig_file_name = ( result_path + '/Correlation_spectra/' + df_filename[0:14] + ' Correlation dynamic spectra cleaned fig.' + str(figID + 1) + '.png') TwoDynSpectraPlot(CorrModule.transpose(), CorrPhase.transpose(), 2 * VminNorm, 2 * VmaxNorm, -3.15, 3.15, Suptitle, 'Intensity, dB', 'Phase, rad', Nsp, TimeFigureScaleFig, TimeScaleFig, frequency, FreqPointsNum, colormap, 'Normalized correlation module', 'Correlation phase', fig_file_name, currentDate, currentTime, Software_version, customDPI) ''' # Check of second counter data for linearity OneImmedSpecterPlot(list(range(ChunksInFile)), timeLineSecond, 'timeLineSecond', 0, ChunksInFile, 0, 2000, 'Time, sec', 'Second counter, sec', 'Second counter', ' ', 'ADR_Results/Service/' + df_filename[0:14] + ' Second counter fig.' + str(figID+1) + '.png') ''' gc.collect() #print ('\n Position in file: ', file.tell(), ' File size: ', df_filesize) #if (file.tell() == df_filesize): print ('\n File was read till the end \n') if (file.tell() < df_filesize): print(' The difference is ', (df_filesize - file.tell()), ' bytes') print('\n File was NOT read till the end!!! ERROR') file.close() #Here we close the data file ok = 1 return ok, DAT_file_name, DAT_file_list
def pulsar_incoherent_dedispersion( common_path, filename, pulsar_name, average_const, profile_pic_min, profile_pic_max, cleaning_Iana, cleaning, no_of_iterations, std_lines_clean, pic_in_line, std_pixels_clean, SpecFreqRange, freqStart, freqStop, save_profile_txt, save_compensated_data, customDPI, colormap): previousTime = time.time() currentTime = time.strftime("%H:%M:%S") currentDate = time.strftime("%d.%m.%Y") rc('font', size=6, weight='bold') data_filename = common_path + filename # *** Creating a folder where all pictures and results will be stored (if it doesn't exist) *** newpath = 'RESULTS_pulsar_single_pulses_' + pulsar_name + '_' + filename if not os.path.exists(newpath): os.makedirs(newpath) # Path to timeline file to be analyzed: time_line_file_name = common_path + filename[-31:-13] + '_Timeline.txt' if save_profile_txt > 0: # *** Creating a name for long timeline TXT file *** profile_file_name = newpath + '/' + filename + '_time_profile.txt' profile_txt_file = open( profile_file_name, 'w') # Open and close to delete the file with the same name profile_txt_file.close() # *** Opening DAT datafile *** file = open(data_filename, 'rb') # reading FHEADER df_filesize = os.stat(data_filename).st_size # Size of file df_filename = file.read(32).decode('utf-8').rstrip( '\x00') # Initial data file name file.close() receiver_type = df_filename[-4:] # Reading file header to obtain main parameters of the file if receiver_type == '.adr': [TimeRes, fmin, fmax, df, frequency_list, FFTsize] = FileHeaderReaderADR(data_filename, 0, 1) if receiver_type == '.jds': [ df_filename, df_filesize, df_system_name, df_obs_place, df_description, CLCfrq, df_creation_timeUTC, sp_in_file, ReceiverMode, Mode, Navr, TimeRes, fmin, fmax, df, frequency_list, FFTsize, dataBlockSize ] = FileHeaderReaderJDS(data_filename, 0, 1) # Manually set frequencies for two channels mode if int(CLCfrq / 1000000) == 33: #FFTsize = 8192 fmin = 16.5 fmax = 33.0 frequency_list = np.linspace(fmin, fmax, FFTsize) sp_in_file = int( ((df_filesize - 1024) / (len(frequency_list) * 8) )) # the second dimension of the array: file size - 1024 bytes pulsar_ra, pulsar_dec, DM, p_bar = catalogue_pulsar(pulsar_name) # ************************************************************************************ # R E A D I N G D A T A * # ************************************************************************************ # Time line file reading timeline, dt_timeline = time_line_file_reader(time_line_file_name) # Selecting the frequency range of data to be analyzed if SpecFreqRange == 1: A = [] B = [] for i in range(len(frequency_list)): A.append(abs(frequency_list[i] - freqStart)) B.append(abs(frequency_list[i] - freqStop)) ifmin = A.index(min(A)) ifmax = B.index(min(B)) shift_vector = DM_full_shift_calc(ifmax - ifmin, frequency_list[ifmin], frequency_list[ifmax], df / pow(10, 6), TimeRes, DM, receiver_type) print(' Number of frequency channels: ', ifmax - ifmin) else: shift_vector = DM_full_shift_calc( len(frequency_list) - 4, fmin, fmax, df / pow(10, 6), TimeRes, DM, receiver_type) print(' Number of frequency channels: ', len(frequency_list) - 4) ifmin = 0 ifmax = int(len(frequency_list) - 4) if save_compensated_data > 0: with open(data_filename, 'rb') as file: file_header = file.read(1024) # Data file header read # *** Creating a binary file with data for long data storage *** new_data_file_name = pulsar_name + '_DM_' + str(DM) + '_' + filename new_data_file = open(new_data_file_name, 'wb') new_data_file.write(file_header) new_data_file.seek(624) # Lb place in header new_data_file.write(np.int32(ifmin).tobytes()) new_data_file.seek(628) # Hb place in header new_data_file.write(np.int32(ifmax).tobytes()) new_data_file.seek(632) # Wb place in header new_data_file.write( np.int32(ifmax - ifmin).tobytes()) # bytes([np.int32(ifmax - ifmin)])) new_data_file.close() # *** Creating a name for long timeline TXT file *** new_TLfile_name = pulsar_name + '_DM_' + str( DM) + '_' + data_filename[:-13] + '_Timeline.txt' new_TLfile = open( new_TLfile_name, 'w') # Open and close to delete the file with the same name new_TLfile.close() del file_header max_shift = np.abs(shift_vector[0]) if SpecFreqRange == 1: buffer_array = np.zeros((ifmax - ifmin, 2 * max_shift)) else: buffer_array = np.zeros((len(frequency_list) - 4, 2 * max_shift)) num_of_blocks = int(sp_in_file / (1 * max_shift)) print(' Number of spectra in file: ', sp_in_file, ' ') print(' Maximal shift is: ', max_shift, ' pixels ') print(' Number of blocks in file: ', num_of_blocks, ' ') print(' Dispersion measure: ', DM, ' pc / cm3 \n') print(' Pulsar name: ', pulsar_name, ' \n') if receiver_type == '.jds': num_frequencies_initial = len(frequency_list) - 4 frequency_list_initial = np.empty_like(frequency_list) frequency_list_initial[:] = frequency_list[:] dat_file = open(data_filename, 'rb') dat_file.seek(1024) # Jumping to 1024 byte from file beginning for block in range(num_of_blocks): # main loop by number of blocks in file print( '\n * Data block # ', block + 1, ' of ', num_of_blocks, '\n ******************************************************************' ) # Time line arrangements: fig_time_scale = [] fig_date_time_scale = [] for i in range(block * max_shift, (block + 1) * max_shift ): # Shows the time of pulse end (at lowest frequency) fig_time_scale.append(timeline[i][11:23]) fig_date_time_scale.append(timeline[i][:]) print(' Time: ', fig_time_scale[0], ' - ', fig_time_scale[-1], ', number of points: ', len(fig_time_scale)) # Data block reading if receiver_type == '.jds': data = np.fromfile(dat_file, dtype=np.float64, count=(num_frequencies_initial + 4) * 1 * max_shift) # 2 data = np.reshape(data, [(num_frequencies_initial + 4), 1 * max_shift], order='F') # 2 data = data[: num_frequencies_initial, :] # To delete the last channels of DSP data where time is stored # Cutting the array in predefined frequency range if SpecFreqRange == 1: data, frequency_list, fi_start, fi_stop = specify_frequency_range( data, frequency_list_initial, freqStart, freqStop) num_frequencies = len(frequency_list) else: num_frequencies = num_frequencies_initial # Normalization of data Normalization_lin(data, num_frequencies, 1 * max_shift) nowTime = time.time() print('\n *** Preparation of data took: ', round((nowTime - previousTime), 2), 'seconds ') previousTime = nowTime if cleaning_Iana > 0: data = survey_cleaning( data) # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! if cleaning > 0: # Cleaning vertical and horizontal lines of RFI data, mask, cleaned_pixels_num = clean_lines_of_pixels( data, no_of_iterations, std_lines_clean, pic_in_line) plt.figure(1, figsize=(10.0, 6.0)) plt.subplots_adjust(left=None, bottom=0, right=None, top=0.86, wspace=None, hspace=None) ImA = plt.imshow(mask, aspect='auto', vmin=0, vmax=1, cmap='Greys') plt.title('Full log initial data', fontsize=10, fontweight='bold', style='italic', y=1.025) plt.ylabel('One dimension', fontsize=10, fontweight='bold') plt.xlabel('Second dimensions', fontsize=10, fontweight='bold') plt.colorbar() plt.yticks(fontsize=8, fontweight='bold') plt.xticks(fontsize=8, fontweight='bold') pylab.savefig(newpath + '/00_10' + ' fig. ' + str(block + 1) + ' - Result mask after lines cleaning.png', bbox_inches='tight', dpi=300) plt.close('all') # Cleaning remaining 1 pixel splashes of RFI data, mask, cleaned_pixels_num = array_clean_by_STD_value( data, std_pixels_clean) plt.figure(1, figsize=(10.0, 6.0)) plt.subplots_adjust(left=None, bottom=0, right=None, top=0.86, wspace=None, hspace=None) ImA = plt.imshow(mask, aspect='auto', vmin=0, vmax=1, cmap='Greys') plt.title('Full log initial data', fontsize=10, fontweight='bold', style='italic', y=1.025) plt.ylabel('One dimension', fontsize=10, fontweight='bold') plt.xlabel('Second dimensions', fontsize=10, fontweight='bold') plt.colorbar() plt.yticks(fontsize=8, fontweight='bold') plt.xticks(fontsize=8, fontweight='bold') pylab.savefig(newpath + '/00_11' + ' fig. ' + str(block + 1) + ' - Mask after fine STD cleaning.png', bbox_inches='tight', dpi=300) plt.close('all') nowTime = time.time() print('\n *** Normalization and cleaning took: ', round((nowTime - previousTime), 2), 'seconds ') previousTime = nowTime ''' # Logging the data with np.errstate(invalid='ignore'): data[:,:] = 10 * np.log10(data[:,:]) data[np.isnan(data)] = 0 # Normalizing log data data = data - np.mean(data) ''' # Dispersion delay removing data_space = np.zeros((num_frequencies, 2 * max_shift)) data_space[:, max_shift:] = data[:, :] temp_array = pulsar_DM_compensation_with_indices_changes( data_space, shift_vector) del data, data_space nowTime = time.time() # print('\n *** Dispersion compensation took: ', round((nowTime - previousTime), 2), 'seconds ') print('\n *** Dispersion delay removing took: ', round((nowTime - previousTime), 2), 'seconds ') previousTime = nowTime # Adding the next data block buffer_array += temp_array # Making and filling the array with fully ready data for plotting and saving to a file array_compensated_DM = buffer_array[:, 0:max_shift] if block > 0: # Saving data with compensated DM to DAT file if save_compensated_data > 0 and block > 0: temp = array_compensated_DM.transpose().copy(order='C') new_data_file = open(new_data_file_name, 'ab') new_data_file.write(temp) new_data_file.close() # Saving time data to ling timeline file with open(new_TLfile_name, 'a') as new_TLfile: for i in range(max_shift): new_TLfile.write((fig_date_time_scale[i][:])) # str # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! # Logging the data with np.errstate(divide='ignore'): array_compensated_DM[:, :] = 10 * np.log10( array_compensated_DM[:, :]) array_compensated_DM[array_compensated_DM == -np.inf] = 0 # Normalizing log data array_compensated_DM = array_compensated_DM - np.mean( array_compensated_DM) # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! # Preparing single averaged data profile for figure profile = array_compensated_DM.mean(axis=0)[:] profile = profile - np.mean(profile) # Save full profile to TXT file if save_profile_txt > 0: profile_txt_file = open(profile_file_name, 'a') for i in range(len(profile)): profile_txt_file.write(str(profile[i]) + ' \n') profile_txt_file.close() # Averaging of the array with pulses for figure averaged_array = average_some_lines_of_array( array_compensated_DM, int(num_frequencies / average_const)) freq_resolution = (df * int(num_frequencies / average_const)) / 1000. max_time_shift = max_shift * TimeRes # NEW start averaged_array = averaged_array - np.mean(averaged_array) # NEW stop plot_ready_data(profile, averaged_array, frequency_list, num_frequencies, fig_time_scale, newpath, filename, pulsar_name, DM, freq_resolution, TimeRes, max_time_shift, block, num_of_blocks - 1, block, profile_pic_min, profile_pic_max, df_description, colormap, customDPI, currentDate, currentTime, Software_version) # Rolling temp_array to put current data first buffer_array = np.roll(buffer_array, -max_shift) buffer_array[:, max_shift:] = 0 dat_file.close() # Fourier analysis of the obtained time profile of pulses if save_profile_txt > 0: print('\n\n *** Making Fourier transform of the time profile...') pulsar_pulses_time_profile_FFT(newpath + '/', filename + '_time_profile.txt', pulsar_name, TimeRes, profile_pic_min, profile_pic_max, customDPI, colormap) return new_data_file_name
def jds_wf_true_resolution(source_directory, result_directory, no_of_points_for_fft, no_of_bunches_per_file): # *** Search JDS files in the source_directory *** fileList = find_files_only_in_current_folder(source_directory, '.jds', 1) print('') if len( fileList ) > 1: # Check if files have same parameters if there are more then one file in list # Check if all files (except the last) have same size same_or_not = check_if_all_files_of_same_size(source_directory, fileList, 1) # Check if all files in this folder have the same parameters in headers equal_or_not = check_if_JDS_files_of_equal_parameters( source_directory, fileList) if same_or_not and equal_or_not: print( '\n\n\n :-) All files seem to be of the same parameters! :-) \n\n\n' ) else: print( '\n\n\n ************************************************************************************* \n * *' ) print( ' * Seems files in folders are different check the errors and restart the script! *' ) print( ' * * ' '\n ************************************************************************************* \n\n\n' ) decision = int( input( '* Enter "1" to start processing, or "0" to stop the script: ' )) if decision != 1: sys.exit('\n\n\n *** Program stopped! *** \n\n\n') # To print in console the header of first file print('\n First file header parameters: \n') # *** Data file header read *** [ df_filename, df_filesize, df_system_name, df_obs_place, df_description, CLCfrq, df_creation_timeUTC, Channel, ReceiverMode, Mode, Navr, TimeRes, fmin, fmax, df, frequency, FreqPointsNum, data_block_size ] = FileHeaderReaderJDS(source_directory + fileList[0], 0, 1) if Channel == 0 or Channel == 1: # Single channel mode wf_data_chA = np.empty([0]) else: wf_data_chA = np.empty([0]) wf_data_chB = np.empty([0]) # CLCfrq = 80 # Main loop by files start for fileNo in range(len(fileList)): # loop by files print('\n\n * File ', str(fileNo + 1), ' of', str(len(fileList))) print(' * File path: ', str(fileList[fileNo])) # *** Opening datafile *** fname = source_directory + fileList[fileNo] # ********************************************************************************* # *** Data file header read *** [ df_filename, df_filesize, df_system_name, df_obs_place, df_description, CLCfrq, df_creation_timeUTC, Channel, ReceiverMode, Mode, Navr, TimeRes, fmin, fmax, df, frequency, FreqPointsNum, data_block_size ] = FileHeaderReaderJDS(fname, 0, 0) # !!! Make automatic calculations of time and frequency resolutions for waveform mode!!! # Manually set frequencies for one channel mode FreqPointsNum = int(no_of_points_for_fft / 2) ''' if (Channel == 0 and int(CLCfrq/1000000) == 66) or (Channel == 1 and int(CLCfrq/1000000) == 66): # FreqPointsNum = 8192 frequency = np.linspace(0.0, 33.0, FreqPointsNum) # Manually set frequencies for two channels mode if Channel == 2 or (Channel == 0 and int(CLCfrq/1000000) == 33) or (Channel == 1 and int(CLCfrq/1000000) == 33): #FreqPointsNum = 8192 frequency = np.linspace(16.5, 33.0, FreqPointsNum) # For new receiver (temporary): if Channel == 2 and int(CLCfrq/1000000) == 80: #FreqPointsNum = 8192 frequency = np.linspace(0.0, 40.0, FreqPointsNum) ''' # Create long data files and copy first data file header to them if fileNo == 0: #''' with open(fname, 'rb') as file: # *** Data file header read *** file_header = file.read(1024) # *** Creating a name for long timeline TXT file *** TLfile_name = df_filename + '_Timeline.txt' TLfile = open( TLfile_name, 'w') # Open and close to delete the file with the same name TLfile.close() # *** Creating a binary file with data for long data storage *** file_data_A_name = df_filename + '_Data_chA.dat' file_data_A = open(file_data_A_name, 'wb') file_data_A.write(file_header) file_data_A.seek(574) # FFT size place in header file_data_A.write(np.int32(no_of_points_for_fft).tobytes()) file_data_A.seek(624) # Lb place in header file_data_A.write(np.int32(0).tobytes()) file_data_A.seek(628) # Hb place in header file_data_A.write(np.int32(FreqPointsNum).tobytes()) file_data_A.seek(632) # Wb place in header file_data_A.write(np.int32(FreqPointsNum).tobytes()) file_data_A.seek(636) # Navr place in header #file_data_A.write(bytes([np.int32(Navr)])) # !!! To correct !!! #file_data_A.write(np.int32(no_of_points_for_fft/8192).tobytes()) # !!! Check for correctness !!! file_data_A.write( np.int32(1).tobytes()) # !!! Check for correctness !!! file_data_A.close() if Channel == 2: file_data_B_name = df_filename + '_Data_chB.dat' file_data_B = open(file_data_B_name, 'wb') file_data_B.write(file_header) file_data_B.seek(574) # FFT size place in header file_data_B.write(np.int32(no_of_points_for_fft).tobytes()) file_data_B.seek(624) # Lb place in header file_data_B.write(np.int32(0).tobytes()) file_data_B.seek(628) # Hb place in header file_data_B.write(np.int32(FreqPointsNum).tobytes()) file_data_B.seek(632) # Wb place in header file_data_B.write(np.int32(FreqPointsNum).tobytes()) file_data_B.seek(636) # Navr place in header #file_data_B.write(bytes([np.int32(Navr)])) # !!! To correct !!! file_data_B.write(np.int32(1).tobytes()) file_data_B.close() del file_header #''' # Calculation of number of blocks and number of spectra in the file if Channel == 0 or Channel == 1: # Single channel mode no_of_spectra_in_bunch = int( (df_filesize - 1024) / (no_of_bunches_per_file * 2 * data_block_size)) else: # Two channels mode no_of_spectra_in_bunch = int( (df_filesize - 1024) / (no_of_bunches_per_file * 4 * data_block_size)) no_of_blocks_in_file = (df_filesize - 1024) / data_block_size fine_CLCfrq = (int(CLCfrq / 1000000.0) * 1000000.0) # Real time resolution of averaged spectra real_spectra_dt = float(no_of_points_for_fft / fine_CLCfrq) real_spectra_df = float((fine_CLCfrq / 2) / (no_of_points_for_fft / 2)) if fileNo == 0: print(' Number of blocks in file: ', no_of_blocks_in_file) print(' Number of spectra in bunch: ', no_of_spectra_in_bunch) print(' Number of bunches to read in file: ', no_of_bunches_per_file) print(' Time resolution of calculated spectra: ', round(real_spectra_dt * 1000, 3), ' ms') print(' Frequency resolution of calculated spectra: ', round(real_spectra_df / 1000, 3), ' kHz') print('\n *** Reading data from file *** \n') # ******************************************************************************* # R E A D I N G D A T A * # ******************************************************************************* with open(fname, 'rb') as file: file.seek(1024) # Jumping to 1024 byte from file beginning # *** DATA READING process *** # Preparing arrays for dynamic spectra #dyn_spectra_chA = np.zeros((int(data_block_size/2), no_of_bunches_per_file), float) #if Channel == 2: # Two channels mode # dyn_spectra_chB = np.zeros((int(data_block_size/2), no_of_bunches_per_file), float) # !!! Fake timing. Real timing to be done!!! TimeFigureScaleFig = np.linspace(0, no_of_bunches_per_file, no_of_bunches_per_file + 1) for i in range(no_of_bunches_per_file): TimeFigureScaleFig[i] = str(TimeFigureScaleFig[i]) time_scale_bunch = [] #bar = IncrementalBar(' File ' + str(fileNo+1) + ' of ' + str(len(fileList)) + ' reading: ', # max=no_of_bunches_per_file, suffix='%(percent)d%%') for bunch in range(no_of_bunches_per_file): #bar.next() # Reading and reshaping all data with time data if Channel == 0 or Channel == 1: # Single channel mode wf_data = np.fromfile(file, dtype='i2', count=no_of_spectra_in_bunch * data_block_size) wf_data = np.reshape( wf_data, [data_block_size, no_of_spectra_in_bunch], order='F') if Channel == 2: # Two channels mode wf_data = np.fromfile(file, dtype='i2', count=2 * no_of_spectra_in_bunch * data_block_size) wf_data = np.reshape( wf_data, [data_block_size, 2 * no_of_spectra_in_bunch], order='F') # Timing timeline_block_str = jds_waveform_time(wf_data, CLCfrq, data_block_size) if Channel == 2: # Two channels mode timeline_block_str = timeline_block_str[0:int( len(timeline_block_str) / 2)] # Cut the timeline of second channel for i in range(len(timeline_block_str)): time_scale_bunch.append(df_creation_timeUTC[0:10] + ' ' + timeline_block_str[i]) # [0:12] # deleting the time blocks from waveform data real_data_block_size = data_block_size - 4 wf_data = wf_data[0:real_data_block_size, :] # *** !!! Not sure if it is necessary now !!! *** # Scaling of the data - seems to be wrong in absolute value wf_data = wf_data / 32768 # .0 # Separation data into channels if Channel == 0 or Channel == 1: # Single channel mode wf_data_chA = np.append( wf_data_chA, np.reshape( wf_data, [real_data_block_size * no_of_spectra_in_bunch, 1], order='F')) del wf_data # Deleting unnecessary array name just in case if Channel == 2: # Two channels mode # Separating the data into two channels #print(wf_data.size, wf_data.shape) wf_data = np.reshape( wf_data, [2 * real_data_block_size * no_of_spectra_in_bunch, 1], order='F') wf_data_chA = np.append(wf_data_chA, wf_data[0:( 2 * real_data_block_size * no_of_spectra_in_bunch):2]) # Separation to channel A wf_data_chB = np.append(wf_data_chB, wf_data[1:( 2 * real_data_block_size * no_of_spectra_in_bunch):2]) # Separation to channel B del wf_data no_of_spectra_to_compute = int( np.floor(len(wf_data_chA) / no_of_points_for_fft)) print(' Bunch # ', bunch + 1, ' Number of true spectra in bunch: ', no_of_spectra_to_compute) # Cutting the full array and saving residuals to buffer ready_wf_array_chA = wf_data_chA[:no_of_spectra_to_compute * no_of_points_for_fft] ready_wf_array_chA = np.reshape( ready_wf_array_chA, [no_of_points_for_fft, no_of_spectra_to_compute], order='F') wf_data_chA = wf_data_chA[no_of_spectra_to_compute * no_of_points_for_fft:] if Channel == 2: ready_wf_array_chB = wf_data_chB[: no_of_spectra_to_compute * no_of_points_for_fft] ready_wf_array_chB = np.reshape( ready_wf_array_chB, [no_of_points_for_fft, no_of_spectra_to_compute], order='F') wf_data_chB = wf_data_chB[no_of_spectra_to_compute * no_of_points_for_fft:] # preparing matrices for spectra spectra_chA = np.zeros_like(ready_wf_array_chA) if Channel == 2: spectra_chB = np.zeros_like(ready_wf_array_chB) # Calculation of spectra for i in range(no_of_spectra_to_compute): spectra_chA[:, i] = np.power( np.abs(np.fft.fft(ready_wf_array_chA[:, i])), 2) if Channel == 2: # Two channels mode spectra_chB[:, i] = np.power( np.abs(np.fft.fft(ready_wf_array_chB[:, i])), 2) # Storing only first (left) mirror part of spectra spectra_chA = spectra_chA[:int(no_of_points_for_fft / 2), :] if Channel == 2: spectra_chB = spectra_chB[:int(no_of_points_for_fft / 2), :] # At 33 MHz the specter is usually upside down, to correct it we use flip up/down if int(CLCfrq / 1000000) == 33: spectra_chA = np.flipud(spectra_chA) if Channel == 2: spectra_chB = np.flipud(spectra_chB) ''' ''' # Saving spectra data to dat file temp = spectra_chA.transpose().copy(order='C') file_data_A = open(file_data_A_name, 'ab') file_data_A.write(temp) file_data_A.close() if Channel == 2: temp = spectra_chB.transpose().copy(order='C') file_data_B = open(file_data_B_name, 'ab') file_data_B.write(temp) file_data_B.close() # Saving time data to ling timeline file with open(TLfile_name, 'a') as TLfile: for i in range(no_of_spectra_in_bunch): TLfile.write( (str(time_scale_bunch[i][:])) + ' \n') # str #bar.finish() file.close() # Close the data file