Exemple #1
0
def ADR_file_reader(fileList, result_path, MaxNim, RFImeanConst, Vmin, Vmax,
                    VminNorm, VmaxNorm, VminCorrMag, VmaxCorrMag, customDPI,
                    colormap, CorrelationProcess, Sum_Diff_Calculate,
                    longFileSaveAch, longFileSaveBch, longFileSaveCMP,
                    longFileSaveCRI, longFileSaveSSD, DynSpecSaveInitial,
                    DynSpecSaveCleaned, CorrSpecSaveInitial,
                    CorrSpecSaveCleaned, SpecterFileSaveSwitch, ImmediateSpNo):

    currentTime = time.strftime("%H:%M:%S")
    currentDate = time.strftime("%d.%m.%Y")

    if not os.path.exists(result_path):
        os.makedirs(result_path)
    if not os.path.exists(result_path + '/Service'):
        os.makedirs(result_path + '/Service')
    if DynSpecSaveInitial == 1:
        if not os.path.exists(result_path + '/Initial_spectra'):
            os.makedirs(result_path + '/Initial_spectra')
    if (DynSpecSaveCleaned == 1 and CorrelationProcess == 1):
        if not os.path.exists(result_path + '/Correlation_spectra'):
            os.makedirs(result_path + '/Correlation_spectra')

    for fileNo in range(len(fileList)):  # loop by files

        # *** Opening datafile ***
        fname = ''
        if len(fname) < 1: fname = fileList[fileNo]

        # Reading the file header
        [
            df_filename, df_filesize, df_system_name, df_obs_place,
            df_description, F_ADC, df_creation_timeUTC, ReceiverMode, ADRmode,
            sumDifMode, NAvr, TimeRes, fmin, fmax, df, frequency, FFT_Size,
            SLine, Width, BlockSize
        ] = FileHeaderReaderADR(fname, 0, 0)

        # Reading the chunk header
        [
            SpInFile, SpInFrame, FrameInChunk, ChunksInFile, sizeOfChunk,
            frm_sec, frm_phase
        ] = ChunkHeaderReaderADR(fname, 0, BlockSize, 0)

        FreqPointsNum = int(Width * 1024)

        # *** Setting the time reference (file beginning) ***
        TimeFirstFramePhase = float(frm_phase) / F_ADC
        TimeFirstFrameFloatSec = frm_sec + TimeFirstFramePhase
        TimeScaleStartTime = datetime(int('20' + df_filename[1:3]),
                                      int(df_filename[3:5]),
                                      int(df_filename[5:7]),
                                      int(df_creation_timeUTC[0:2]),
                                      int(df_creation_timeUTC[3:5]),
                                      int(df_creation_timeUTC[6:8]),
                                      int(df_creation_timeUTC[9:12]) * 1000)

        with open(fname, 'rb') as file:

            # *** Reading indexes of data from index file '*.fft' ***
            indexes = []
            ifname = 'package_ra_data_files_formats/' + str(int(
                FFT_Size / 2)) + '.fft'
            indexfile = open(ifname, 'r')
            num = 0
            for line in indexfile:
                ind = int(line)
                if (ind >= SLine * 1024) & (ind < ((SLine + Width) * 1024)):
                    indexes.append(ind - SLine * 1024)
                num = num + 1
            indexfile.close()

            timeLineSecond = np.zeros(
                ChunksInFile)  # List of second values from DSP_INF field

            # *** If it is the first file - write the header to long data file ***
            if ((longFileSaveAch == 1 or longFileSaveBch == 1
                 or longFileSaveCRI == 1 or longFileSaveCMP == 1
                 or longFileSaveSSD == 1) and fileNo == 0):
                file.seek(0)
                file_header = file.read(1024)

                # *** Creating a name for long timeline TXT file ***
                TLfile_name = df_filename + '_Timeline.txt'
                TLfile = open(
                    TLfile_name, 'w'
                )  # Open and close to delete the file with the same name
                TLfile.close()

                DAT_file_name = df_filename
                DAT_file_list = []

                # *** Creating a binary file with data for long data storage ***
                if (longFileSaveAch == 1
                        and (ADRmode == 3 or ADRmode == 5 or ADRmode == 6)):
                    fileData_A_name = df_filename + '_Data_chA.dat'
                    fileData_A = open(fileData_A_name, 'wb')
                    fileData_A.write(file_header)
                    fileData_A.close()
                    DAT_file_list.append('chA')
                if (longFileSaveBch == 1
                        and (ADRmode == 4 or ADRmode == 5 or ADRmode == 6)):
                    fileData_B_name = df_filename + '_Data_chB.dat'
                    fileData_B = open(fileData_B_name, 'wb')
                    fileData_B.write(file_header)
                    fileData_B.close()
                    DAT_file_list.append('chB')
                if (CorrelationProcess == 1 and longFileSaveCRI == 1
                        and ADRmode == 6):
                    fileData_CRe_name = df_filename + '_Data_CRe.dat'
                    fileData_C_Re = open(fileData_CRe_name, 'wb')
                    fileData_C_Re.write(file_header)
                    fileData_C_Re.close()
                    DAT_file_list.append('CRe')
                    fileData_CIm_name = df_filename + '_Data_CIm.dat'
                    fileData_C_Im = open(fileData_CIm_name, 'wb')
                    fileData_C_Im.write(file_header)
                    fileData_C_Im.close()
                    DAT_file_list.append('CIm')
                if (CorrelationProcess == 1 and longFileSaveCMP == 1
                        and ADRmode == 6):
                    fileData_CM_name = df_filename + '_Data_C_m.dat'
                    fileData_C_M = open(fileData_CM_name, 'wb')
                    fileData_C_M.write(file_header)
                    fileData_C_M.close()
                    DAT_file_list.append('C_m')
                    fileData_CP_name = df_filename + '_Data_C_p.dat'
                    fileData_C_P = open(fileData_CP_name, 'wb')
                    fileData_C_P.write(file_header)
                    fileData_C_P.close()
                    DAT_file_list.append('C_p')
                if (Sum_Diff_Calculate == 1 and longFileSaveSSD == 1
                        and (ADRmode == 5 or ADRmode == 6)):
                    fileData_Sum_name = df_filename + '_Data_Sum.dat'
                    fileData_Sum = open(fileData_Sum_name, 'wb')
                    fileData_Sum.write(file_header)
                    fileData_Sum.close()
                    fileData_Dif_name = df_filename + '_Data_Dif.dat'
                    fileData_Dif = open(fileData_Dif_name, 'wb')
                    fileData_Dif.write(file_header)
                    fileData_Dif.close()

                del file_header

            #************************************************************************************
            #                            R E A D I N G   D A T A                                *
            #************************************************************************************

            file.seek(1024)  # Jumping to 1024 byte from file beginning

            if ADRmode > 2 and ADRmode < 7:  # Specter modes
                figID = -1
                figMAX = int(math.ceil((ChunksInFile) / MaxNim))
                if figMAX < 1: figMAX = 1
                for fig in range(figMAX):
                    Time1 = time.time()  # Timing
                    figID = figID + 1
                    currentTime = time.strftime("%H:%M:%S")
                    print('   File # ', str(fileNo + 1), ' of ',
                          str(len(fileList)), ', figure # ', figID + 1, ' of ',
                          figMAX, '   started at: ', currentTime)
                    if (ChunksInFile - MaxNim * figID) < MaxNim:
                        Nim = (ChunksInFile - MaxNim * figID)
                    else:
                        Nim = MaxNim
                    SpectrNum = Nim * SpInFrame * FrameInChunk  # Number of specra in the figure

                    # *** Preparing empty matrices ***
                    if ADRmode == 3 or ADRmode == 5 or ADRmode == 6:
                        Data_Ch_A = np.zeros(
                            (Nim * SpInFrame * FrameInChunk, FreqPointsNum))
                        Data_Ch_A0 = np.zeros(
                            (Nim * SpInFrame * FrameInChunk, FreqPointsNum))
                    if ADRmode == 4 or ADRmode == 5 or ADRmode == 6:
                        Data_Ch_B = np.zeros(
                            (Nim * SpInFrame * FrameInChunk, FreqPointsNum))
                        Data_Ch_B0 = np.zeros(
                            (Nim * SpInFrame * FrameInChunk, FreqPointsNum))
                    if ADRmode == 6:
                        Data_C_Im = np.zeros(
                            (Nim * SpInFrame * FrameInChunk, FreqPointsNum))
                        Data_C_Re = np.zeros(
                            (Nim * SpInFrame * FrameInChunk, FreqPointsNum))
                        Data_C_Im0 = np.zeros(
                            (Nim * SpInFrame * FrameInChunk, FreqPointsNum))
                        Data_C_Re0 = np.zeros(
                            (Nim * SpInFrame * FrameInChunk, FreqPointsNum))
                        CorrModule = np.zeros(
                            (Nim * SpInFrame * FrameInChunk, FreqPointsNum))
                        CorrPhase = np.zeros(
                            (Nim * SpInFrame * FrameInChunk, FreqPointsNum))

                    TimeScale = []
                    TimeFigureScale = []  # Timelime (new) for each figure
                    TimeFigureStartTime = datetime(2016, 1, 1, 0, 0, 0, 0)

                    # *** DATA READING process ***

                    # Reading and reshaping all data with readers
                    raw = np.fromfile(file,
                                      dtype='i4',
                                      count=int((Nim * (sizeOfChunk + 8)) / 4))
                    raw = np.reshape(raw, [int((sizeOfChunk + 8) / 4), Nim],
                                     order='F')

                    # Splitting headers from data
                    headers = raw[0:1024, :]
                    data = raw[1024:, :]
                    del raw

                    # Arranging data in right order
                    if ADRmode == 3:
                        data = np.reshape(
                            data,
                            [FreqPointsNum, Nim * FrameInChunk * SpInFrame],
                            order='F')
                        Data_Ch_A0 = data[0:FreqPointsNum:1, :].transpose()
                    if ADRmode == 4:
                        data = np.reshape(
                            data,
                            [FreqPointsNum, Nim * FrameInChunk * SpInFrame],
                            order='F')
                        Data_Ch_B0 = data[0:FreqPointsNum:1, :].transpose()
                    if ADRmode == 5:
                        data = np.reshape(data, [
                            FreqPointsNum * 2, Nim * FrameInChunk * SpInFrame
                        ],
                                          order='F')
                        Data_Ch_B0 = data[0:(FreqPointsNum *
                                             2):2, :].transpose()
                        Data_Ch_A0 = data[1:(FreqPointsNum *
                                             2):2, :].transpose()
                    if (ADRmode == 6):
                        data = np.reshape(data, [
                            FreqPointsNum * 4, Nim * FrameInChunk * SpInFrame
                        ],
                                          order='F')
                        Data_C_Im0 = data[0:(FreqPointsNum *
                                             4):4, :].transpose()
                        Data_C_Re0 = data[1:(FreqPointsNum *
                                             4):4, :].transpose()
                        Data_Ch_B0 = data[2:(FreqPointsNum *
                                             4):4, :].transpose()
                        Data_Ch_A0 = data[3:(FreqPointsNum *
                                             4):4, :].transpose()
                    del data

                    # *** TimeLine calculations ***
                    for i in range(Nim):

                        # *** DSP_INF ***
                        frm_count = headers[3][i]
                        frm_sec = headers[4][i]
                        frm_phase = headers[5][i]

                        # * Abosolute time calculation *
                        timeLineSecond[
                            figID * MaxNim +
                            i] = frm_sec  # to check the linearity of seconds
                        TimeCurrentFramePhase = float(frm_phase) / F_ADC
                        TimeCurrentFrameFloatSec = frm_sec + TimeCurrentFramePhase
                        TimeSecondDiff = TimeCurrentFrameFloatSec - TimeFirstFrameFloatSec
                        TimeAdd = timedelta(
                            0, int(np.fix(TimeSecondDiff)),
                            int(
                                np.fix(
                                    (TimeSecondDiff -
                                     int(np.fix(TimeSecondDiff))) * 1000000)))

                        # Adding of time point to time line is in loop by spectra because
                        # for each spectra in frame there is one time point but it should
                        # appear for all spectra to fit the dimensions of arrays

                        # * Time from figure start calculation *
                        if (i == 0): TimeFigureStart = TimeCurrentFrameFloatSec
                        TimeFigureSecondDiff = TimeCurrentFrameFloatSec - TimeFigureStart
                        TimeFigureAdd = timedelta(
                            0, int(np.fix(TimeFigureSecondDiff)),
                            int(
                                np.fix((TimeFigureSecondDiff -
                                        int(np.fix(TimeFigureSecondDiff))) *
                                       1000000)))

                        for iframe in range(0, SpInFrame):
                            TimeScale.append(
                                str((TimeScaleStartTime + TimeAdd)))  #.time()
                            TimeFigureScale.append(
                                str((TimeFigureStartTime +
                                     TimeFigureAdd).time()))

                    # Exact string timescales to show on plots
                    TimeFigureScaleFig = np.empty_like(TimeFigureScale)
                    TimeScaleFig = np.empty_like(TimeScale)
                    for i in range(len(TimeFigureScale)):
                        TimeFigureScaleFig[i] = TimeFigureScale[i][0:11]
                        TimeScaleFig[i] = TimeScale[i][11:23]

                    # *** Performing index changes ***
                    for i in range(0, FreqPointsNum):
                        n = indexes[i]
                        if ADRmode == 3 or ADRmode == 5 or ADRmode == 6:
                            Data_Ch_A[:, n] = Data_Ch_A0[:, i]
                        if ADRmode == 4 or ADRmode == 5 or ADRmode == 6:
                            Data_Ch_B[:, n] = Data_Ch_B0[:, i]
                        if (ADRmode == 6 and CorrelationProcess == 1):
                            Data_C_Im[:, n] = Data_C_Im0[:, i]
                            Data_C_Re[:, n] = Data_C_Re0[:, i]

                    # *** Deleting matrices which were nessesary for index changes ***
                    del n

                    if ADRmode == 3 or ADRmode == 5 or ADRmode == 6:
                        del Data_Ch_A0
                    if ADRmode == 4 or ADRmode == 5 or ADRmode == 6:
                        del Data_Ch_B0
                    if (ADRmode == 6 and CorrelationProcess == 1):
                        del Data_C_Im0, Data_C_Re0

                    # *** Converting from FPGA to PC float format ***
                    if ADRmode == 3 or ADRmode == 5 or ADRmode == 6:
                        Data_Ch_A = FPGAtoPCarrayADR(Data_Ch_A, NAvr)
                    if ADRmode == 4 or ADRmode == 5 or ADRmode == 6:
                        Data_Ch_B = FPGAtoPCarrayADR(Data_Ch_B, NAvr)
                    if (ADRmode == 6 and CorrelationProcess == 1):
                        Data_C_Re = FPGAtoPCarrayADR(Data_C_Re, NAvr)
                        Data_C_Im = FPGAtoPCarrayADR(Data_C_Im, NAvr)

                    # *** Calculating Sum and Difference of A and B channels ***
                    if ((ADRmode == 5 or ADRmode == 6)
                            and Sum_Diff_Calculate == 1):
                        Data_Sum = Data_Ch_A + Data_Ch_B
                        Data_Dif = abs(Data_Ch_A - Data_Ch_B)

                    # *** Saving data to a long-term file ***
                    if (ADRmode == 3 or ADRmode == 5
                            or ADRmode == 6) and longFileSaveAch == 1:
                        fileData_A = open(fileData_A_name, 'ab')
                        fileData_A.write(Data_Ch_A)
                        fileData_A.close()
                    if (ADRmode == 4 or ADRmode == 5
                            or ADRmode == 6) and longFileSaveBch == 1:
                        fileData_B = open(fileData_B_name, 'ab')
                        fileData_B.write(Data_Ch_B)
                        fileData_B.close()
                    if ADRmode == 6 and longFileSaveCRI == 1 and CorrelationProcess == 1:
                        fileData_C_Re = open(fileData_CRe_name, 'ab')
                        fileData_C_Re.write(Data_C_Re)
                        fileData_C_Re.close()
                        fileData_C_Im = open(fileData_CIm_name, 'ab')
                        fileData_C_Im.write(Data_C_Im)
                        fileData_C_Im.close()
                    if ((ADRmode == 5 or ADRmode == 6)
                            and Sum_Diff_Calculate == 1
                            and longFileSaveSSD == 1):
                        fileData_Sum = open(fileData_Sum_name, 'ab')
                        fileData_Sum.write(Data_Sum)
                        fileData_Sum.close()
                        fileData_Dif = open(fileData_Dif_name, 'ab')
                        fileData_Dif.write(Data_Dif)
                        fileData_Dif.close()
                        del Data_Sum, Data_Dif

                    if (longFileSaveAch == 1 or longFileSaveBch == 1
                            or longFileSaveCRI == 1 or longFileSaveCMP == 1
                            or longFileSaveSSD == 1):
                        with open(TLfile_name, 'a') as TLfile:
                            for i in range(SpInFrame * FrameInChunk * Nim):
                                TLfile.write((TimeScale[i][:]) + ' \n')  # str

                    # *** Converting to logarythmic scale matrices ***
                    if ADRmode == 3 or ADRmode == 5 or ADRmode == 6:
                        with np.errstate(divide='ignore'):
                            Data_Ch_A = 10 * np.log10(Data_Ch_A)
                    if ADRmode == 4 or ADRmode == 5 or ADRmode == 6:
                        with np.errstate(divide='ignore'):
                            Data_Ch_B = 10 * np.log10(Data_Ch_B)
                    if (ADRmode == 6 and CorrelationProcess == 1):
                        with np.errstate(divide='ignore'):
                            CorrModule = ((Data_C_Re)**2 +
                                          (Data_C_Im)**2)**(0.5)
                            CorrModule = 10 * np.log10(CorrModule)
                            CorrPhase = np.arctan2(Data_C_Im, Data_C_Re)
                        CorrModule[np.isnan(CorrModule)] = 0
                        CorrPhase[np.isnan(CorrPhase)] = 0

                    # *** Writing correlation data to long files ***
                    if (ADRmode == 6 and longFileSaveCMP == 1
                            and CorrelationProcess == 1):
                        fileData_C_M = open(fileData_CM_name, 'ab')
                        fileData_C_M.write(np.float64(CorrModule))
                        fileData_C_M.close()
                        fileData_C_P = open(fileData_CP_name, 'ab')
                        fileData_C_P.write(np.float64(CorrPhase))
                        fileData_C_P.close()

                    # *** Saving immediate spectrum to file ***
                    if (SpecterFileSaveSwitch == 1 and figID == 0):
                        SpFile = open(
                            result_path + '/Service/Specter_' +
                            df_filename[0:14] + '.txt', 'w')
                        for i in range(FreqPointsNum - 1):
                            if ADRmode == 3:
                                SpFile.write(
                                    str('{:10.6f}'.format(frequency[i])) +
                                    '  ' + str('{:16.10f}'.format(
                                        Data_Ch_A[ImmediateSpNo][i])) + ' \n')
                            if ADRmode == 4:
                                SpFile.write(
                                    str('{:10.6f}'.format(frequency[i])) +
                                    '  ' + str('{:16.10f}'.format(
                                        Data_Ch_B[ImmediateSpNo][i])) + ' \n')
                            if ADRmode == 5 or ADRmode == 6:
                                SpFile.write(
                                    str('{:10.6f}'.format(frequency[i])) +
                                    '  ' + str('{:16.10f}'.format(
                                        Data_Ch_A[ImmediateSpNo][i])) + '  ' +
                                    str('{:16.10f}'.format(
                                        Data_Ch_B[ImmediateSpNo][i])) + ' \n')
                        SpFile.close()

                    # *** FIGURE Immediate spectra before cleaning and normalizing ***
                    if figID == 0:
                        if ADRmode == 3:
                            Data_1 = Data_Ch_A[0][:]
                            Legend_1 = 'Channel A'
                        if ADRmode == 4:
                            Data_1 = Data_Ch_B[0][:]
                            Legend_1 = 'Channel B'
                        if ADRmode == 3 or ADRmode == 4:
                            no_of_sets = 1
                            Data_2 = []
                            Suptitle = ('Immediate spectrum ' +
                                        str(df_filename[0:18]) + ' ' +
                                        Legend_1)
                            Title = ('Initial parameters: dt = ' +
                                     str(round(TimeRes * 1000, 3)) +
                                     ' ms, df = ' + str(round(df / 1000., 3)) +
                                     ' kHz' + sumDifMode + ', Description: ' +
                                     str(df_description))
                            Filename = (
                                result_path + '/Service/' + df_filename[0:14] +
                                ' ' + Legend_1 +
                                ' Immediate Spectrum before cleaning and normalizing.png'
                            )

                        if (ADRmode == 5 or ADRmode
                                == 6):  # Immediate spectrum channels A & B
                            Data_1 = Data_Ch_A[0][:]
                            Data_2 = Data_Ch_B[0][:]
                            Legend_1 = 'Channel A'
                            no_of_sets = 2
                            Suptitle = ('Immediate spectrum ' +
                                        str(df_filename[0:18]) +
                                        ' channels A & B')
                            Title = ('Initial parameters: dt = ' +
                                     str(round(TimeRes * 1000, 3)) +
                                     ' ms, df = ' + str(round(df / 1000., 3)) +
                                     ' kHz,' + sumDifMode + ' Description: ' +
                                     str(df_description))
                            Filename = (
                                result_path + '/Service/' + df_filename[0:14] +
                                ' Channels A and B Immediate Spectrum before cleaning and normalizing.png'
                            )

                        TwoOrOneValuePlot(
                            no_of_sets, frequency, Data_1, Data_2, Legend_1,
                            'Channel B', frequency[0],
                            frequency[FreqPointsNum - 1], -120, -20, -120, -20,
                            'Frequency, MHz', 'Intensity, dB', 'Intensity, dB',
                            Suptitle, Title, Filename, currentDate,
                            currentTime, Software_version)

                    # *** FIGURE Correlation amplitude and phase immediate spectrum ***
                    if (ADRmode == 6 and figID == 0 and CorrelationProcess == 1
                        ):  #  Immediate correlation spectrum channels A & B

                        Suptitle = ('Immediate correlation spectrum ' +
                                    str(df_filename[0:18]) + ' channels A & B')
                        Title = ('Initial parameters: dt = ' +
                                 str(round(TimeRes * 1000, 3)) + ' ms, df = ' +
                                 str(round(df / 1000., 3)) + ' kHz,' +
                                 sumDifMode + ' Description: ' +
                                 str(df_description))
                        Filename = (
                            result_path + '/Service/' + df_filename[0:14] +
                            ' Channels A and B Correlation module and phase spectrum.png'
                        )

                        TwoOrOneValuePlot(
                            2, frequency, CorrModule[0][:], CorrPhase[0][:],
                            'Correlation module', 'Correlation phase',
                            frequency[0], frequency[FreqPointsNum - 1], -150,
                            -20, -4, 4, 'Frequency, MHz', 'Intensity, dB',
                            'Phase, rad', Suptitle, Title, Filename,
                            currentDate, currentTime, Software_version)

                    # *** FIGURE Initial dynamic spectrum of 1 channel (A or B) ***
                    if ((ADRmode == 3 or ADRmode == 4)
                            and DynSpecSaveInitial == 1):
                        if ADRmode == 3:
                            Data = Data_Ch_A.transpose()
                        if ADRmode == 4:
                            Data = Data_Ch_B.transpose()

                        Suptitle = ('Dynamic spectrum (initial) ' +
                                    str(df_filename[0:18]) + ' - Fig. ' +
                                    str(figID + 1) + ' of ' + str(figMAX) +
                                    '\n Initial parameters: dt = ' +
                                    str(round(TimeRes * 1000, 3)) +
                                    ' ms, df = ' + str(round(df / 1000., 3)) +
                                    ' kHz, ' + sumDifMode + ' Receiver: ' +
                                    str(df_system_name) + ', Place: ' +
                                    str(df_obs_place) + '\n Description: ' +
                                    str(df_description))

                        fig_file_name = (result_path + '/Initial_spectra/' +
                                         df_filename[0:14] +
                                         ' Initial dynamic spectrum fig.' +
                                         str(figID + 1) + '.png')

                        OneDynSpectraPlot(
                            Data, -120, -30, Suptitle, 'Intensity, dB',
                            Nim * SpInFrame * FrameInChunk, TimeScaleFig,
                            frequency, FreqPointsNum, colormap,
                            'UTC Time, HH:MM:SS.msec', fig_file_name,
                            currentDate, currentTime, Software_version,
                            customDPI)

                    # *** FIGURE Initial dynamic spectrum channels A and B ***
                    if ((ADRmode == 5 or ADRmode == 6)
                            and DynSpecSaveInitial == 1):

                        fig_file_name = (result_path + '/Initial_spectra/' +
                                         df_filename[0:14] +
                                         ' Initial dynamic spectrum fig.' +
                                         str(figID + 1) + '.png')
                        Suptitle = ('Dynamic spectrum (initial) ' +
                                    str(df_filename) + ' - Fig. ' +
                                    str(figID + 1) + ' of ' + str(figMAX) +
                                    '\n Initial parameters: dt = ' +
                                    str(round(TimeRes * 1000, 3)) +
                                    ' ms, df = ' + str(round(df / 1000., 3)) +
                                    ' kHz, ' + sumDifMode + ' Receiver: ' +
                                    str(df_system_name) + ', Place: ' +
                                    str(df_obs_place) + '\n' + ReceiverMode +
                                    ', Description: ' + str(df_description))

                        TwoDynSpectraPlot(
                            Data_Ch_A.transpose(), Data_Ch_B.transpose(), Vmin,
                            Vmax, Vmin, Vmax, Suptitle, 'Intensity, dB',
                            'Intensity, dB', Nim * SpInFrame * FrameInChunk,
                            TimeFigureScaleFig, TimeScaleFig, frequency,
                            FreqPointsNum, colormap, 'Channel A', 'Channel B',
                            fig_file_name, currentDate, currentTime,
                            Software_version, customDPI)

                    # *** FIGURE Initial correlation spectrum module and phase ***
                    if (ADRmode == 6 and CorrSpecSaveInitial == 1
                            and CorrelationProcess == 1):

                        fig_file_name = (result_path +
                                         '/Correlation_spectra/' +
                                         df_filename[0:14] +
                                         ' Correlation dynamic spectrum fig.' +
                                         str(figID + 1) + '.png')
                        Suptitle = ('Correlation dynamic spectrum (initial) ' +
                                    str(df_filename) + ' - Fig. ' +
                                    str(figID + 1) + ' of ' + str(figMAX) +
                                    '\n Initial parameters: dt = ' +
                                    str(round(TimeRes * 1000, 3)) +
                                    ' ms, df = ' + str(round(df / 1000., 3)) +
                                    ' kHz, ' + sumDifMode + ' Receiver: ' +
                                    str(df_system_name) + ', Place: ' +
                                    str(df_obs_place) + '\n' + ReceiverMode +
                                    ', Description: ' + str(df_description))

                        TwoDynSpectraPlot(CorrModule.transpose(),
                                          CorrPhase.transpose(), VminCorrMag,
                                          VmaxCorrMag, -3.15, 3.15, Suptitle,
                                          'Intensity, dB', 'Phase, rad',
                                          Nim * SpInFrame * FrameInChunk,
                                          TimeFigureScaleFig, TimeScaleFig,
                                          frequency, FreqPointsNum, colormap,
                                          'Correlation module',
                                          'Correlation phase', fig_file_name,
                                          currentDate, currentTime,
                                          Software_version, customDPI)

                    # *** Normalizing amplitude-frequency responce ***
                    if (ADRmode == 3 or ADRmode == 5
                            or ADRmode == 6) and DynSpecSaveCleaned == 1:
                        Normalization_dB(Data_Ch_A, FreqPointsNum,
                                         Nim * SpInFrame * FrameInChunk)
                    if (ADRmode == 4 or ADRmode == 5
                            or ADRmode == 6) and DynSpecSaveCleaned == 1:
                        Normalization_dB(Data_Ch_B, FreqPointsNum,
                                         Nim * SpInFrame * FrameInChunk)
                    if ADRmode == 6 and CorrelationProcess == 1 and CorrSpecSaveCleaned == 1:
                        Normalization_dB(CorrModule, FreqPointsNum,
                                         Nim * SpInFrame * FrameInChunk)

                    # *** Deleting cahnnels with strong RFI ***
                    if (ADRmode == 3 or ADRmode == 5
                            or ADRmode == 6) and DynSpecSaveCleaned == 1:
                        simple_channel_clean(Data_Ch_A, RFImeanConst)
                    if (ADRmode == 4 or ADRmode == 5
                            or ADRmode == 6) and DynSpecSaveCleaned == 1:
                        simple_channel_clean(Data_Ch_B, RFImeanConst)
                    if ADRmode == 6 and CorrelationProcess == 1 and CorrSpecSaveCleaned == 1:
                        simple_channel_clean(CorrModule, 2 * RFImeanConst)

                    #   *** Immediate spectra of normalyzed data ***    (only for first figure in data file)
                    if figID == 0 and DynSpecSaveCleaned == 1:
                        if ADRmode == 3:
                            Data_1 = Data_Ch_A[0][:]
                            Legend_1 = 'Channel A'
                        if ADRmode == 4:
                            Data_1 = Data_Ch_B[0][:]
                            Legend_1 = 'Channel B'
                        if ADRmode == 3 or ADRmode == 4:
                            no_of_sets = 1
                            Data_2 = []
                            Suptitle = ('Normalized immediate spectrum ' +
                                        str(df_filename[0:18]) + ' ' +
                                        Legend_1)
                            Title = ('Initial parameters: dt = ' +
                                     str(round(TimeRes * 1000, 3)) +
                                     ' ms, df = ' + str(round(df / 1000., 3)) +
                                     ' kHz' + sumDifMode + ', Description: ' +
                                     str(df_description))
                            Filename = (
                                result_path + '/Service/' + df_filename[0:14] +
                                ' ' + Legend_1 +
                                ' Immediate Spectrum after cleaning and normalizing.png'
                            )

                        if (ADRmode == 5 or ADRmode
                                == 6):  # Immediate spectrum channels A & B
                            no_of_sets = 2
                            Data_1 = Data_Ch_A[0][:]
                            Data_2 = Data_Ch_B[0][:]
                            Legend_1 = 'Channel A'
                            Suptitle = ('Normalized immediate spectrum ' +
                                        str(df_filename[0:18]) +
                                        ' channels A & B')
                            Title = ('Initial parameters: dt = ' +
                                     str(round(TimeRes * 1000, 3)) +
                                     ' ms, df = ' + str(round(df / 1000., 3)) +
                                     ' kHz' + sumDifMode + ', Description: ' +
                                     str(df_description))
                            Filename = (
                                result_path + '/Service/' + df_filename[0:14] +
                                ' Channels A and B Immediate Spectrum after cleaning and normalizing.png'
                            )

                        TwoOrOneValuePlot(
                            no_of_sets, frequency, Data_1, Data_2, Legend_1,
                            'Channel B', frequency[0],
                            frequency[FreqPointsNum - 1], -10, 40, -10, 40,
                            'Frequency, MHz', 'Intensity, dB', 'Intensity, dB',
                            Suptitle, Title, Filename, currentDate,
                            currentTime, Software_version)

                    # *** FIGURE Cleaned and normalized dynamic spectrum of 1 channel A or B
                    if ((ADRmode == 3 or ADRmode == 4)
                            and DynSpecSaveCleaned == 1):
                        if ADRmode == 3:
                            Data = Data_Ch_A.transpose()
                        if ADRmode == 4:
                            Data = Data_Ch_B.transpose()

                        Suptitle = ('Dynamic spectrum (normalized) ' +
                                    str(df_filename[0:18]) + ' - Fig. ' +
                                    str(figID + 1) + ' of ' + str(figMAX) +
                                    '\n Initial parameters: dt = ' +
                                    str(round(TimeRes * 1000, 3)) +
                                    ' ms, df = ' + str(round(df / 1000., 3)) +
                                    ' kHz, ' + sumDifMode + ' Receiver: ' +
                                    str(df_system_name) + ', Place: ' +
                                    str(df_obs_place) + '\n Description: ' +
                                    str(df_description))

                        fig_file_name = (result_path + '/' +
                                         df_filename[0:14] +
                                         ' Dynamic spectrum fig.' +
                                         str(figID + 1) + '.png')

                        OneDynSpectraPlot(
                            Data, VminNorm, VmaxNorm, Suptitle,
                            'Intensity, dB', Nim * SpInFrame * FrameInChunk,
                            TimeScaleFig, frequency, FreqPointsNum, colormap,
                            'UTC Time, HH:MM:SS.msec', fig_file_name,
                            currentDate, currentTime, Software_version,
                            customDPI)

                    # *** FIGURE Dynamic spectrum channels A and B cleaned and normalized (python 3 new version) ***
                    if ((ADRmode == 5 or ADRmode == 6)
                            and DynSpecSaveCleaned == 1):
                        fig_file_name = (result_path + '/' +
                                         df_filename[0:14] +
                                         ' Dynamic spectrum fig.' +
                                         str(figID + 1) + '.png')
                        Suptitle = ('Dynamic spectrum (normalized) ' +
                                    str(df_filename) + ' - Fig. ' +
                                    str(figID + 1) + ' of ' + str(figMAX) +
                                    '\n Initial parameters: dt = ' +
                                    str(round(TimeRes * 1000, 3)) +
                                    ' ms, df = ' + str(round(df / 1000., 3)) +
                                    ' kHz, ' + sumDifMode + ' Receiver: ' +
                                    str(df_system_name) + ', Place: ' +
                                    str(df_obs_place) + '\n' + ReceiverMode +
                                    ', Description: ' + str(df_description))

                        TwoDynSpectraPlot(
                            Data_Ch_A.transpose(), Data_Ch_B.transpose(),
                            VminNorm, VmaxNorm, VminNorm, VmaxNorm, Suptitle,
                            'Intensity, dB', 'Intensity, dB',
                            Nim * SpInFrame * FrameInChunk, TimeFigureScaleFig,
                            TimeScaleFig, frequency, FreqPointsNum, colormap,
                            'Channel A', 'Channel B', fig_file_name,
                            currentDate, currentTime, Software_version,
                            customDPI)

                    # *** FIGURE Correlation spectrum module and phase cleaned and normalized (python 3 new version) ***
                    if (ADRmode == 6 and CorrSpecSaveCleaned == 1
                            and CorrelationProcess == 1):
                        Suptitle = 'Correlation dynamic spectrum (normalized) ' + str(
                            df_filename
                        ) + ' - Fig. ' + str(figID + 1) + ' of ' + str(
                            figMAX
                        ) + '\n Initial parameters: dt = ' + str(
                            round(TimeRes * 1000, 3)) + ' ms, df = ' + str(
                                round(df / 1000., 3)
                            ) + ' kHz, ' + sumDifMode + ' Receiver: ' + str(
                                df_system_name
                            ) + ', Place: ' + str(
                                df_obs_place
                            ) + '\n' + ReceiverMode + ', Description: ' + str(
                                df_description)
                        fig_file_name = result_path + '/Correlation_spectra/' + df_filename[
                            0:
                            14] + ' Correlation dynamic spectrum cleaned fig.' + str(
                                figID + 1) + '.png'
                        TwoDynSpectraPlot(
                            CorrModule.transpose(), CorrPhase.transpose(),
                            VminNorm, 3 * VmaxNorm, -3.15, 3.15, Suptitle,
                            'Intensity, dB', 'Phase, rad',
                            Nim * SpInFrame * FrameInChunk, TimeFigureScaleFig,
                            TimeScaleFig, frequency, FreqPointsNum, colormap,
                            'Normalized and cleaned correlation module',
                            'Correlation phase', fig_file_name, currentDate,
                            currentTime, Software_version, customDPI)

                gc.collect()
            del timeLineSecond
            #print ('\n  Position in file: ', file.tell(), ' File size: ', df_filesize)
            #if (file.tell() == df_filesize): print ('\n  File was read till the end')
            #if (file.tell() < df_filesize):  print ('\n  File was NOT read till the end!!! ERROR')

        # Here we close the data file
    ok = 1
    return ok, DAT_file_name, DAT_file_list
Exemple #2
0
                    Suptitle = ('Dynamic spectrum (initial) ' +
                                str(df_filename) + ' - Fig. ' +
                                str(fig_id + 1) + ' of ' + str(figMAX) +
                                '\n Initial parameters: dt = ' +
                                str(round(TimeRes * 1000, 3)) + ' ms, df = ' +
                                str(round(df / 1000., 3)) + ' kHz, ' +
                                sumDifMode + ' Receiver: ' +
                                str(df_system_name) + ', Place: ' +
                                str(df_obs_place) + '\n' + receiver_mode +
                                ', Description: ' + str(df_description))

                    TwoDynSpectraPlot(Data_Ch_A.transpose(),
                                      Data_Ch_B.transpose(), Vmin, Vmax, Vmin,
                                      Vmax, Suptitle, 'Intensity, dB',
                                      'Intensity, dB',
                                      Nim * sp_in_frame * FrameInChunk,
                                      TimeFigureScaleFig, TimeScaleFig,
                                      frequency, freq_points_num, colormap,
                                      'Channel A', 'Channel B', fig_file_name,
                                      current_date, current_time,
                                      Software_version, customDPI)

                # *** FIGURE Initial correlation spectrum module and phase ***
                if adr_mode == 6 and CorrSpecSaveInitial == 1 and CorrelationProcess == 1:

                    fig_file_name = (result_path + '/Correlation_spectra/' +
                                     df_filename[0:14] +
                                     ' Correlation dynamic spectrum fig.' +
                                     str(fig_id + 1) + '.png')
                    Suptitle = ('Correlation dynamic spectrum (initial) ' +
                                str(df_filename) + ' - Fig. ' +
                                str(fig_id + 1) + ' of ' + str(figMAX) +
                            ' Initial dynamic spectrum fig.' + str(figID + 1) + '.png'

            Suptitle = ('Dynamic spectrum (initial) ' + str(fname[-11:]) +
                        ' - Fig. ' + str(figID + 1) + ' of ' + str(figMAX) +
                        '\n Initial parameters: dt = ' +
                        str(round(time_res * 1000, 3)) + ' ms, df = ' +
                        str(round(df * 1000., 3)) + ' kHz, ' + sumDifMode +
                        ' Receiver: ' + str(df_system_name) + ', Place: ' +
                        str(df_obs_place) + '\n' + ReceiverMode +
                        ' Local solar culmination at ' + solar_culm[0:2] +
                        ' hour ' + solar_culm[2:4] + ' min, UTC')

            TwoDynSpectraPlot(dataLHP, dataRHP, VminL, VmaxL, VminR, VmaxR,
                              Suptitle, 'Intensity, dB', 'Intensity, dB', Nsp,
                              TimeFigureScaleFig, TimeScaleFig, frequency,
                              freq_points_num, colormap,
                              'Left-hand circular polarization',
                              'Right-hand circular polarization',
                              fig_file_name, currentDate, current_time,
                              software_version, custom_dpi)

            if DynSpecSaveCleaned == 1:

                # *** Normalizing amplitude-frequency response ***
                Normalization_dB(dataLHP.transpose(), freq_points_num, Nsp)
                Normalization_dB(dataRHP.transpose(), freq_points_num, Nsp)

                # *** Deleting channels with strong RFI ***
                simple_channel_clean(dataLHP.transpose(), RFImeanConst)
                simple_channel_clean(dataRHP.transpose(), RFImeanConst)

                # *** FIGURE Dynamic spectrum channels A and B cleaned and normalized (python 3 new version) ***
Suptitle = 'Dynamic spectrum (initial) ' + str(filename)+' - Fig. ' + str(figID+1)+' of ' + \
           str(figMAX) + '\n Initial parameters: dt = '+str(round(TimeRes*1000, 3)) + ' ms, df = ' + \
           str(round(df/1000., 3))+' kHz, ' + sumDifMode + ' Receiver: ' + str(df_system_name) + \
           ', Place: ' + str(df_obs_place) + '\n' + ReceiverMode + ', Description: ' + str(df_description)

TimeFigureScaleFig = np.empty_like(time_line_str)
TimeScaleFig = np.empty_like(time_line_str)
for i in range(len(time_line_str)):
    TimeFigureScaleFig[i] = time_line_str[i][0:11]
    TimeScaleFig[i] = time_line_str[i][11:23]

# *** FIGURE Initial dynamic spectra channels A and B ***
TwoDynSpectraPlot(dynamic_spectra1.transpose(), dynamic_spectra2.transpose(),
                  np.min(dynamic_spectra1), np.max(dynamic_spectra1),
                  np.min(dynamic_spectra2), np.max(dynamic_spectra2), Suptitle,
                  'Intensity, dB', 'Intensity, dB', no_of_spectra,
                  TimeFigureScaleFig, TimeScaleFig, frequency[:],
                  freq_points_num, colormap, 'Channel A - ' + Label01,
                  'Channel B - ' + Label02, fig_file_name, currentDate,
                  currentTime, Software_version, customDPI)

# Normalization of data (extracting the frequency response of the signal path)
Normalization_dB(dynamic_spectra1, freq_points_num, nt)
Normalization_dB(dynamic_spectra2, freq_points_num, nt)

# Preparing variables for figure
fig_file_name = 'FITS_Results/' + filename[0:19] + ' ' + str(df_description).replace('"', '') + \
                ' Normalized dynamic spectrum fig.' + str(figID+1) + '.png'
Suptitle = 'Dynamic spectrum (normalized) ' + str(filename) + ' - Fig. ' + str(figID+1) + ' of ' + str(figMAX) + \
           '\n Initial parameters: dt = ' + str(round(TimeRes*1000, 3)) + ' ms, df = ' + str(round(df/1000., 3)) + \
           ' kHz, ' + sumDifMode + ' Receiver: ' + str(df_system_name) + ', Place: ' + str(df_obs_place) + \
           '\n' + ReceiverMode + ', Description: ' + str(df_description)
Exemple #5
0
                                str(round(TimeRes * 1000, 3)) + ' ms, df = ' +
                                str(round(df / 1000., 3)) +
                                ' kHz, Receiver: ' + str(df_system_name) +
                                ', Place: ' + str(df_obs_place) + '\n' +
                                ReceiverMode + ', Description: ' +
                                str(df_description))

                    fig_file_name = (result_path + '/Initial_spectra/' +
                                     df_filename[0:14] +
                                     ' Initial dynamic spectrum fig.' +
                                     str(figID + 1) + '.png')

                    TwoDynSpectraPlot(Data_ChA.transpose(),
                                      Data_ChB.transpose(), Vmin, Vmax, Vmin,
                                      Vmax, Suptitle, 'Intensity, dB',
                                      'Intensity, dB', Nsp, TimeFigureScaleFig,
                                      TimeScaleFig, frequency, FreqPointsNum,
                                      colormap, 'Channel A', 'Channel B',
                                      fig_file_name, currentDate, currentTime,
                                      Software_version, customDPI)

                # *** FIGURE Initial correlation spectrum Module and Phase (python 3 new version) ***
                if (Mode == 2 and CorrSpecSaveInitial == 1
                        and CorrelationProcess == 1):

                    Suptitle = ('Correlation dynamic spectrum (initial) ' +
                                str(df_filename) + ' - Fig. ' +
                                str(figID + 1) + ' of ' + str(figMAX) +
                                '\n Initial parameters: dt = ' +
                                str(round(TimeRes * 1000, 3)) + ' ms, df = ' +
                                str(round(df / 1000., 3)) +
                                ' kHz, Receiver: ' + str(df_system_name) +
    fig_file_name = (initial_spectra_folder + '/' + df_filename[0:14] +
                     ' Initial dynamic spectrum fig.' + str(0 + 1) + '.png')

    if Channel == 0 or Channel == 1:  # Single channel mode
        OneDynSpectraPlot(dyn_spectra_chA, VminA, VmaxA, Suptitle,
                          'Intensity, dB', no_of_av_spectra_per_file,
                          TimeScaleFig, frequency, FreqPointsNum, colormap,
                          'UTC Time, HH:MM:SS.msec', fig_file_name,
                          currentDate, currentTime, Software_version,
                          customDPI)

    if Channel == 2:
        TwoDynSpectraPlot(dyn_spectra_chA, dyn_spectra_chB, VminA, VmaxA,
                          VminB, VmaxB, Suptitle, 'Intensity, dB',
                          'Intensity, dB', no_of_av_spectra_per_file,
                          TimeScaleFig, TimeScaleFig, frequency, FreqPointsNum,
                          colormap, 'Channel A', 'Channel B', fig_file_name,
                          currentDate, currentTime, Software_version,
                          customDPI)

    # Normalization and cleaning of data

    Normalization_dB(dyn_spectra_chA.transpose(), FreqPointsNum,
                     no_of_av_spectra_per_file)
    if Channel == 2:
        Normalization_dB(dyn_spectra_chB.transpose(), FreqPointsNum,
                         no_of_av_spectra_per_file)

    simple_channel_clean(dyn_spectra_chA, 8)
    if Channel == 2: simple_channel_clean(dyn_spectra_chB, 8)
def JDS_file_reader(fileList, result_path, MaxNsp, spSkip, RFImeanConst, Vmin,
                    Vmax, VminNorm, VmaxNorm, VminCorrMag, VmaxCorrMag,
                    colormap, customDPI, CorrelationProcess, longFileSaveAch,
                    longFileSaveBch, longFileSaveCRI, longFileSaveCMP,
                    DynSpecSaveInitial, DynSpecSaveCleaned,
                    CorrSpecSaveInitial, CorrSpecSaveCleaned,
                    SpecterFileSaveSwitch, ImmediateSpNo):
    currentTime = time.strftime("%H:%M:%S")
    currentDate = time.strftime("%d.%m.%Y")

    # *** Creating a folder where all pictures and results will be stored (if it doen't exist) ***
    if not os.path.exists(result_path):
        os.makedirs(result_path)
    if not os.path.exists(result_path + '/Service'):
        os.makedirs(result_path + '/Service')
    if DynSpecSaveInitial == 1:
        if not os.path.exists(result_path + '/Initial_spectra'):
            os.makedirs(result_path + '/Initial_spectra')
    if (DynSpecSaveCleaned == 1 and CorrelationProcess == 1):
        if not os.path.exists(result_path + '/Correlation_spectra'):
            os.makedirs(result_path + '/Correlation_spectra')

    # Main loop
    for fileNo in range(len(fileList)):  # loop by files
        print('\n\n\n  *  File ', str(fileNo + 1), ' of', str(len(fileList)))
        print('  *  File path: ', str(fileList[fileNo]))

        #*********************************************************************************

        # *** Opening datafile ***
        fname = ''
        if len(fname) < 1: fname = fileList[fileNo]

        # *** Data file header read ***
        [
            df_filename, df_filesize, df_system_name, df_obs_place,
            df_description, CLCfrq, df_creation_timeUTC, SpInFile,
            ReceiverMode, Mode, Navr, TimeRes, fmin, fmax, df, frequency,
            FreqPointsNum, dataBlockSize
        ] = FileHeaderReaderJDS(fname, 0, 0)

        # Initial time line settings
        TimeScaleStartDate = datetime(int(df_creation_timeUTC[0:4]),
                                      int(df_creation_timeUTC[5:7]),
                                      int(df_creation_timeUTC[8:10]), 0, 0, 0,
                                      0)

        timeLineMS = np.zeros(
            int(SpInFile))  # List of ms values from ends of spectra

        # *** Creating a name for long timeline TXT file ***
        if fileNo == 0 and (longFileSaveAch == 1 or longFileSaveBch == 1
                            or longFileSaveCRI == 1 or longFileSaveCMP == 1):
            TLfile_name = df_filename + '_Timeline.txt'
            TLfile = open(
                TLfile_name,
                'wb')  # Open and close to delete the file with the same name
            TLfile.close()

        with open(fname, 'rb') as file:

            # *** If it is the first file - write the header to long data file
            if ((longFileSaveAch == 1 or longFileSaveBch == 1
                 or longFileSaveCRI == 1 or longFileSaveCMP == 1)
                    and fileNo == 0):
                file.seek(0)
                file_header = file.read(1024)

                DAT_file_name = df_filename
                DAT_file_list = []

                # *** Creating a binary file with data for long data storage ***
                if ((Mode == 1 or Mode == 2) and longFileSaveAch == 1):
                    Data_A_name = df_filename + '_Data_chA.dat'
                    Data_AFile = open(Data_A_name, 'wb')
                    Data_AFile.write(file_header)
                    Data_AFile.close()
                    DAT_file_list.append('chA')
                if (longFileSaveBch == 1 and (Mode == 1 or Mode == 2)):
                    Data_B_name = df_filename + '_Data_chB.dat'
                    Data_BFile = open(Data_B_name, 'wb')
                    Data_BFile.write(file_header)
                    Data_BFile.close()
                    DAT_file_list.append('chB')
                if (longFileSaveCRI == 1 and CorrelationProcess == 1
                        and Mode == 2):
                    Data_CRe_name = df_filename + '_Data_CRe.dat'
                    Data_CReFile = open(Data_CRe_name, 'wb')
                    Data_CReFile.write(file_header)
                    Data_CReFile.close()
                    DAT_file_list.append('CRe')
                    Data_CIm_name = df_filename + '_Data_CIm.dat'
                    Data_CImFile = open(Data_CIm_name, 'wb')
                    Data_CImFile.write(file_header)
                    Data_CImFile.close()
                    DAT_file_list.append('CIm')
                if (longFileSaveCMP == 1 and CorrelationProcess == 1
                        and Mode == 2):
                    Data_Cm_name = df_filename + '_Data_C_m.dat'
                    Data_CmFile = open(Data_Cm_name, 'wb')
                    Data_CmFile.write(file_header)
                    Data_CmFile.close()
                    DAT_file_list.append('C_m')
                    Data_Cp_name = df_filename + '_Data_C_p.dat'
                    Data_CpFile = open(Data_Cp_name, 'wb')
                    Data_CpFile.write(file_header)
                    Data_CpFile.close()
                    DAT_file_list.append('C_p')

                del file_header

    #*******************************************************************************
    #                         R E A D I N G   D A T A                              *
    #*******************************************************************************
    #print ('\n  *** Reading data from file *** \n')
            file.seek(1024)  # Jumping to 1024 byte from file beginning
            if Mode == 0:
                print(
                    '\n\n  Data in waveform mode, use appropriate program!!! \n\n\n'
                )

            if Mode > 0 and Mode < 3:  # Spectra modes
                figID = -1
                figMAX = int(math.ceil((SpInFile - spSkip) / MaxNsp))
                if figMAX < 1: figMAX = 1
                for fig in range(figMAX):
                    Time1 = time.time()  # Timing
                    figID = figID + 1
                    currentTime = time.strftime("%H:%M:%S")
                    print(' File # ', str(fileNo + 1), ' of ',
                          str(len(fileList)), ', figure # ', figID + 1, ' of ',
                          figMAX, '   started at: ', currentTime)
                    if (SpInFile - spSkip - MaxNsp * figID) < MaxNsp:
                        Nsp = int(SpInFile - spSkip - MaxNsp * figID)
                    else:
                        Nsp = MaxNsp

                    # *** Preparing empty matrices ***
                    if Mode == 1 or Mode == 2:
                        Data_ChA = np.zeros((Nsp, FreqPointsNum))

                    if Mode == 1 or Mode == 2:
                        Data_ChB = np.zeros((Nsp, FreqPointsNum))

                    if Mode == 2:
                        Data_CRe = np.zeros((Nsp, FreqPointsNum))
                        Data_CIm = np.zeros((Nsp, FreqPointsNum))
                        CorrModule = np.zeros((Nsp, FreqPointsNum))
                        CorrPhase = np.zeros((Nsp, FreqPointsNum))

                    # *** Reading and reshaping all data for figure ***
                    if Mode == 1:
                        raw = np.fromfile(file,
                                          dtype='u4',
                                          count=(2 * Nsp * FreqPointsNum))
                        raw = np.reshape(raw, [2 * FreqPointsNum, Nsp],
                                         order='F')
                        Data_ChA = raw[0:(FreqPointsNum * 2):2, :].transpose()
                        Data_ChB = raw[1:(FreqPointsNum * 2):2, :].transpose()

                    if Mode == 2:
                        raw = np.fromfile(file,
                                          dtype='u4',
                                          count=(4 * Nsp * FreqPointsNum))
                        raw = np.reshape(raw, [4 * FreqPointsNum, Nsp],
                                         order='F')
                        Data_ChA = raw[0:(FreqPointsNum * 4):4, :].transpose()
                        Data_ChB = raw[1:(FreqPointsNum * 4):4, :].transpose()
                        Data_CRe = raw[2:(FreqPointsNum * 4):4, :].transpose()
                        Data_CIm = raw[3:(FreqPointsNum * 4):4, :].transpose()

                    del raw

                    # *** Single out timing from data ***
                    counterA2 = np.uint64(Data_ChA[:, -1])
                    counterB2 = np.uint64(Data_ChB[:, -1])
                    counterA1 = np.uint64(Data_ChA[:, -2])
                    counterB1 = np.uint64(Data_ChB[:, -2])

                    A = np.uint64(int('01111111111111111111111111111111', 2))
                    msCount = np.uint32(np.bitwise_and(
                        counterB2, A))  # number of ms since record started
                    ftCount = np.uint32(np.bitwise_and(
                        counterA2,
                        A))  # number of specter since record started

                    A = np.uint64(int('00000111111111111111111111111111', 2))
                    phaOfSec = np.uint32(np.bitwise_and(
                        counterA1, A))  # phase of second for the spectr
                    A = np.uint64(int('00000000000000011111111111111111', 2))
                    secOfDay = np.uint32(np.bitwise_and(
                        counterB1, A))  # second of the day for the specter

                    # *** Time line arranging ***

                    # Preparing/cleaning matrices for time scales
                    TimeScale = []  # New for each file
                    TimeFigureScale = [
                    ]  # Timelime (new) for each figure (Nsp)
                    # Calculations
                    FigStartTime = timedelta(
                        0, int(secOfDay[0]),
                        int(1000000 * phaOfSec[0] / CLCfrq))
                    for i in range(Nsp):
                        TimeAdd = timedelta(
                            0, int(secOfDay[i]),
                            int(1000000 * phaOfSec[i] / CLCfrq))
                        TimeScale.append(str(str(TimeScaleStartDate +
                                                 TimeAdd)))
                        TimeFigureScale.append(str((TimeAdd - FigStartTime)))

                    TimeFigureScaleFig = np.empty_like(TimeFigureScale)
                    TimeScaleFig = np.empty_like(TimeScale)
                    for i in range(len(TimeFigureScale)):
                        TimeFigureScaleFig[i] = TimeFigureScale[i][0:11]
                        TimeScaleFig[i] = TimeScale[i][11:23]

                    # *** Converting from FPGA to PC float format ***
                    if Mode == 1 or Mode == 2:
                        Data_ChA = FPGAtoPCarrayJDS(Data_ChA, Navr)
                        Data_ChB = FPGAtoPCarrayJDS(Data_ChB, Navr)
                    if (Mode == 2 and CorrelationProcess == 1):
                        Data_CRe = FPGAtoPCarrayJDS(Data_CRe, Navr)
                        Data_CIm = FPGAtoPCarrayJDS(Data_CIm, Navr)
                    '''
                    # *** Absolute correlation specter plot ***
                    if Mode == 2 and figID == 0:   #  Immediate correlation spectrum channels A & B
                        TwoImmedSpectraPlot(frequency, Data_CRe[1][:], Data_CIm[1][:], 'Channel A', 'Channel B',
                                            frequency[0], frequency[FreqPointsNum-1], -0.001, 0.001,
                                            'Frequency, MHz', 'Amplitude, dB',
                                            'Immediate spectrum '+str(df_filename[0:18])+ ' channels A & B',
                                            'Initial parameters: dt = '+str(round(TimeRes,3))+' Sec, df = '+str(round(df/1000,3))+' kHz',
                                            'JDS_Results/Service/'+df_filename[0:14]+' Correlation Spectrum Re and Im before log.png')
                    '''

                    # *** Saving data to a long-term file ***
                    if (Mode == 1 or Mode == 2) and longFileSaveAch == 1:
                        Data_AFile = open(Data_A_name, 'ab')
                        Data_AFile.write(Data_ChA)
                        Data_AFile.close()
                    if (Mode == 1 or Mode == 2) and longFileSaveBch == 1:
                        Data_BFile = open(Data_B_name, 'ab')
                        Data_BFile.write(Data_ChB)
                        Data_BFile.close()
                    if Mode == 2 and longFileSaveCRI == 1 and CorrelationProcess == 1:
                        Data_CReFile = open(Data_CRe_name, 'ab')
                        Data_CReFile.write(np.float64(Data_CRe))
                        Data_CReFile.close()
                        Data_CImFile = open(Data_CIm_name, 'ab')
                        Data_CImFile.write(np.float64(Data_CIm))
                        Data_CImFile.close()

                    if (longFileSaveAch == 1 or longFileSaveBch == 1
                            or longFileSaveCRI == 1 or longFileSaveCMP == 1):
                        with open(TLfile_name, 'a') as TLfile:
                            for i in range(Nsp):
                                TLfile.write(
                                    (TimeScale[i][:] + ' \n'))  #str.encode

                    # *** Converting to logarythmic scale matrices ***
                    if (Mode == 1 or Mode == 2):
                        with np.errstate(invalid='ignore'):
                            Data_ChA = 10 * np.log10(Data_ChA)
                            Data_ChB = 10 * np.log10(Data_ChB)
                        Data_ChA[np.isnan(Data_ChA)] = -120
                        Data_ChB[np.isnan(Data_ChB)] = -120
                    if (Mode == 2 and CorrelationProcess == 1):
                        with np.errstate(invalid='ignore', divide='ignore'):
                            CorrModule = 10 * np.log10(
                                ((Data_CRe)**2 + (Data_CIm)**2)**(0.5))
                            CorrPhase = np.arctan2(Data_CIm, Data_CRe)
                        CorrPhase[np.isnan(CorrPhase)] = 0
                        CorrModule[np.isinf(CorrModule)] = -135.5

                    # *** Saving correlation data to a long-term module and phase files ***
                    if (Mode == 2 and CorrelationProcess == 1
                            and longFileSaveCMP == 1):
                        Data_CmFile = open(Data_Cm_name, 'ab')
                        Data_CmFile.write(np.float64(CorrModule))
                        Data_CmFile.close()
                        Data_CpFile = open(Data_Cp_name, 'ab')
                        Data_CpFile.write(np.float64(CorrPhase))
                        Data_CpFile.close()

                    # *** Saving immediate spectrum to file ***
                    if (SpecterFileSaveSwitch == 1 and figID == 0):
                        SpFile = open(
                            'JDS_Results/Service/Specter_' +
                            df_filename[0:14] + '.txt', 'w')
                        for i in range(FreqPointsNum - 1):
                            if Mode == 1:
                                SpFile.write(
                                    str('{:10.6f}'.format(frequency[i])) +
                                    '  ' + str('{:16.10f}'.format(
                                        Data_ChA[ImmediateSpNo][i])) + '  ' +
                                    str('{:16.10f}'.format(
                                        Data_ChB[ImmediateSpNo][i])) + ' \n')
                            if Mode == 2:
                                SpFile.write(
                                    str(frequency[i]) + '  ' +
                                    str(Data_ChA[ImmediateSpNo][i]) + '  ' +
                                    str(Data_ChB[ImmediateSpNo][i]) + '  ' +
                                    str(Data_CRe[ImmediateSpNo][i]) + '  ' +
                                    str(Data_CIm[ImmediateSpNo][i]) + ' \n')

                        SpFile.close()

    #*******************************************************************************
    #                                  F I G U R E S                               *
    #*******************************************************************************

    # *** Plotting immediate spectra before cleaning and normalizing ***
                    if (Mode == 1 or Mode == 2) and figID == 0:

                        Suptitle = ('Immediate spectrum ' +
                                    str(df_filename[0:18]) + ' channels A & B')
                        Title = ('Place: ' + str(df_obs_place) +
                                 ', Receiver: ' + str(df_system_name) +
                                 '. Initial parameters: dt = ' +
                                 str(round(TimeRes, 3)) + ' Sec, df = ' +
                                 str(round(df / 1000, 3)) + ' kHz ' +
                                 'Description: ' + str(df_description))
                        Filename = (
                            result_path + '/Service/' + df_filename[0:14] +
                            ' Channels A and B Immediate Spectrum before cleaning and normalizing.png'
                        )

                        TwoOrOneValuePlot(
                            2, frequency, Data_ChA[0][:], Data_ChB[0][:],
                            'Channel A', 'Channel B', frequency[0],
                            frequency[FreqPointsNum - 1], -120, -20, -120, -20,
                            'Frequency, MHz', 'Intensity, dB', 'Intensity, dB',
                            Suptitle, Title, Filename, currentDate,
                            currentTime, Software_version)

                    if Mode == 2 and CorrelationProcess == 1 and figID == 0:

                        Suptitle = ('Immediate correlation spectrum ' +
                                    str(df_filename[0:18]) + ' channels A & B')
                        Title = ('Place: ' + str(df_obs_place) +
                                 ', Receiver: ' + str(df_system_name) +
                                 '. Initial parameters: dt = ' +
                                 str(round(TimeRes, 3)) + ' Sec, df = ' +
                                 str(round(df / 1000, 3)) + ' kHz ' +
                                 'Description: ' + str(df_description))
                        Filename = (
                            result_path + '/Service/' + df_filename[0:14] +
                            ' Channels A and B Correlation Immedaiate Spectrum before cleaning and normalizing.png'
                        )

                        TwoOrOneValuePlot(
                            2, frequency, CorrModule[0][:], CorrPhase[0][:],
                            'Correlation module', 'Correlation phase',
                            frequency[0], frequency[FreqPointsNum - 1],
                            VminCorrMag, VmaxCorrMag, -4, 4, 'Frequency, MHz',
                            'Amplitude, dB', 'Phase, deg', Suptitle, Title,
                            Filename, currentDate, currentTime,
                            Software_version)

                    # *** FIGURE Initial dynamic spectrum channels A and B ***
                    if (Mode == 1 or Mode == 2) and DynSpecSaveInitial == 1:

                        Suptitle = ('Dynamic spectrum (initial) ' +
                                    str(df_filename) + ' - Fig. ' +
                                    str(figID + 1) + ' of ' + str(figMAX) +
                                    '\n Initial parameters: dt = ' +
                                    str(round(TimeRes * 1000, 3)) +
                                    ' ms, df = ' + str(round(df / 1000., 3)) +
                                    ' kHz, Receiver: ' + str(df_system_name) +
                                    ', Place: ' + str(df_obs_place) + '\n' +
                                    ReceiverMode + ', Description: ' +
                                    str(df_description))

                        fig_file_name = (result_path + '/Initial_spectra/' +
                                         df_filename[0:14] +
                                         ' Initial dynamic spectrum fig.' +
                                         str(figID + 1) + '.png')

                        TwoDynSpectraPlot(Data_ChA.transpose(),
                                          Data_ChB.transpose(), Vmin, Vmax,
                                          Vmin, Vmax, Suptitle,
                                          'Intensity, dB', 'Intensity, dB',
                                          Nsp, TimeFigureScaleFig,
                                          TimeScaleFig, frequency,
                                          FreqPointsNum, colormap, 'Channel A',
                                          'Channel B', fig_file_name,
                                          currentDate, currentTime,
                                          Software_version, customDPI)

                    # *** FIGURE Initial correlation spectrum Module and Phase (python 3 new version) ***
                    if (Mode == 2 and CorrSpecSaveInitial == 1
                            and CorrelationProcess == 1):

                        Suptitle = ('Correlation dynamic spectrum (initial) ' +
                                    str(df_filename) + ' - Fig. ' +
                                    str(figID + 1) + ' of ' + str(figMAX) +
                                    '\n Initial parameters: dt = ' +
                                    str(round(TimeRes * 1000, 3)) +
                                    ' ms, df = ' + str(round(df / 1000., 3)) +
                                    ' kHz, Receiver: ' + str(df_system_name) +
                                    ', Place: ' + str(df_obs_place) + '\n' +
                                    ReceiverMode + ', Description: ' +
                                    str(df_description))

                        fig_file_name = (result_path +
                                         '/Correlation_spectra/' +
                                         df_filename[0:14] +
                                         ' Correlation dynamic spectrum fig.' +
                                         str(figID + 1) + '.png')

                        TwoDynSpectraPlot(CorrModule.transpose(),
                                          CorrPhase.transpose(), VminCorrMag,
                                          VmaxCorrMag, -3.15, 3.15, Suptitle,
                                          'Intensity, dB', 'Phase, rad', Nsp,
                                          TimeFigureScaleFig, TimeScaleFig,
                                          frequency, FreqPointsNum, colormap,
                                          'Correlation module',
                                          'Correlation phase', fig_file_name,
                                          currentDate, currentTime,
                                          Software_version, customDPI)

                    # *** Normalizing amplitude-frequency responce ***
                    if Mode == 1 or Mode == 2:
                        Normalization_dB(Data_ChA, FreqPointsNum, Nsp)
                        Normalization_dB(Data_ChB, FreqPointsNum, Nsp)
                    if Mode == 2 and CorrelationProcess == 1 and CorrSpecSaveCleaned == 1:
                        Normalization_dB(CorrModule, FreqPointsNum, Nsp)

                    # *** Deleting cahnnels with strong RFI ***
                    if Mode == 1 or Mode == 2:
                        simple_channel_clean(Data_ChA, RFImeanConst)
                        simple_channel_clean(Data_ChB, RFImeanConst)
                    if Mode == 2 and CorrelationProcess == 1 and CorrSpecSaveCleaned == 1:
                        simple_channel_clean(CorrModule, 2 * RFImeanConst)

                    #   *** Immediate spectra ***    (only for first figure in data file)
                    if (Mode == 1 or Mode == 2
                        ) and figID == 0:  # Immediate spectrum channels A & B

                        Suptitle = (
                            'Cleaned and normalized immediate spectrum ' +
                            str(df_filename[0:18]) + ' channels A & B')
                        Title = ('Place: ' + str(df_obs_place) +
                                 ', Receiver: ' + str(df_system_name) +
                                 '. Initial parameters: dt = ' +
                                 str(round(TimeRes, 3)) + ' Sec, df = ' +
                                 str(round(df / 1000, 3)) + ' kHz ' +
                                 'Description: ' + str(df_description))
                        Filename = (
                            result_path + '/Service/' + df_filename[0:14] +
                            ' Channels A and B Immediate Spectrum after cleaning and normalizing.png'
                        )

                        TwoOrOneValuePlot(
                            2, frequency, Data_ChA[1][:], Data_ChB[1][:],
                            'Channel A', 'Channel B', frequency[0],
                            frequency[FreqPointsNum - 1], VminNorm - 5,
                            VmaxNorm, VminNorm - 5, VmaxNorm, 'Frequency, MHz',
                            'Intensity, dB', 'Intensity, dB', Suptitle, Title,
                            Filename, currentDate, currentTime,
                            Software_version)

                    # *** FIGURE Normalized dynamic spectrum channels A and B ***
                    if (Mode == 1 or Mode == 2) and DynSpecSaveCleaned == 1:

                        Suptitle = ('Dynamic spectrum (normalized) ' +
                                    str(df_filename) + ' - Fig. ' +
                                    str(figID + 1) + ' of ' + str(figMAX) +
                                    '\n Initial parameters: dt = ' +
                                    str(round(TimeRes * 1000, 3)) +
                                    ' ms, df = ' + str(round(df / 1000., 3)) +
                                    ' kHz, Receiver: ' + str(df_system_name) +
                                    ', Place: ' + str(df_obs_place) + '\n' +
                                    ReceiverMode + ', Description: ' +
                                    str(df_description))

                        fig_file_name = (result_path + '/' +
                                         df_filename[0:14] +
                                         ' Dynamic spectra fig.' +
                                         str(figID + 1) + '.png')

                        TwoDynSpectraPlot(
                            Data_ChA.transpose(), Data_ChB.transpose(),
                            VminNorm, VmaxNorm, VminNorm, VmaxNorm, Suptitle,
                            'Intensity, dB', 'Intensity, dB', Nsp,
                            TimeFigureScaleFig, TimeScaleFig, frequency,
                            FreqPointsNum, colormap, 'Channel A', 'Channel B',
                            fig_file_name, currentDate, currentTime,
                            Software_version, customDPI)

                    # *** FIGURE Normalized correlation spectrum Module and Phase ***
                    if (Mode == 2 and CorrSpecSaveCleaned == 1
                            and CorrelationProcess == 1):

                        Suptitle = (
                            'Correlation dynamic spectrum (normalized) ' +
                            str(df_filename) + ' - Fig. ' + str(figID + 1) +
                            ' of ' + str(figMAX) +
                            '\n Initial parameters: dt = ' +
                            str(round(TimeRes * 1000, 3)) + ' ms, df = ' +
                            str(round(df / 1000., 3)) + ' kHz, Receiver: ' +
                            str(df_system_name) + ', Place: ' +
                            str(df_obs_place) + '\n' + ReceiverMode +
                            ', Description: ' + str(df_description))

                        fig_file_name = (
                            result_path + '/Correlation_spectra/' +
                            df_filename[0:14] +
                            ' Correlation dynamic spectra cleaned fig.' +
                            str(figID + 1) + '.png')
                        TwoDynSpectraPlot(CorrModule.transpose(),
                                          CorrPhase.transpose(), 2 * VminNorm,
                                          2 * VmaxNorm, -3.15, 3.15, Suptitle,
                                          'Intensity, dB', 'Phase, rad', Nsp,
                                          TimeFigureScaleFig, TimeScaleFig,
                                          frequency, FreqPointsNum, colormap,
                                          'Normalized correlation module',
                                          'Correlation phase', fig_file_name,
                                          currentDate, currentTime,
                                          Software_version, customDPI)
                '''
                # Check of second counter data for linearity
                OneImmedSpecterPlot(list(range(ChunksInFile)), timeLineSecond, 'timeLineSecond',
                                    0, ChunksInFile, 0, 2000,
                                    'Time, sec', 'Second counter, sec',
                                    'Second counter',
                                    ' ',
                                    'ADR_Results/Service/' + df_filename[0:14] + ' Second counter fig.' + str(figID+1) + '.png')

                '''

                gc.collect()

            #print ('\n  Position in file: ', file.tell(), ' File size: ', df_filesize)
            #if (file.tell() == df_filesize): print ('\n  File was read till the end \n')
            if (file.tell() < df_filesize):
                print('    The difference is ', (df_filesize - file.tell()),
                      ' bytes')
                print('\n  File was NOT read till the end!!! ERROR')

        file.close()  #Here we close the data file

    ok = 1
    return ok, DAT_file_name, DAT_file_list
Exemple #8
0
def jds_wf_simple_reader(directory, no_of_spectra_to_average, skip_data_blocks,
                         VminNorm, VmaxNorm, colormap, custom_dpi,
                         save_long_file_aver, dyn_spectr_save_init,
                         dyn_spectr_save_norm):

    current_time = time.strftime("%H:%M:%S")
    current_date = time.strftime("%d.%m.%Y")

    # *** Creating a folder where all pictures and results will be stored (if it doesn't exist) ***
    result_folder = 'RESULTS_JDS_waveform_' + directory.split('/')[-2]
    if not os.path.exists(result_folder):
        os.makedirs(result_folder)
    service_folder = result_folder + '/Service'
    if not os.path.exists(service_folder):
        os.makedirs(service_folder)
    if dyn_spectr_save_init == 1:
        initial_spectra_folder = result_folder + '/Initial spectra'
        if not os.path.exists(initial_spectra_folder):
            os.makedirs(initial_spectra_folder)

    # *** Search JDS files in the directory ***

    file_list = find_files_only_in_current_folder(directory, '.jds', 1)
    print('')

    if len(
            file_list
    ) > 1:  # Check if files have same parameters if there are more then one file in list
        # Check if all files (except the last) have same size
        same_or_not = check_if_all_files_of_same_size(directory, file_list, 1)

        # Check if all files in this folder have the same parameters in headers
        equal_or_not = check_if_JDS_files_of_equal_parameters(
            directory, file_list)

        if same_or_not and equal_or_not:
            print(
                '\n\n\n        :-)  All files seem to be of the same parameters!  :-) \n\n\n'
            )
        else:
            print(
                '\n\n\n ************************************************************************************* '
            )
            print(
                ' *                                                                                   *'
            )
            print(
                ' *   Seems files in folders are different check the errors and restart the script!   *'
            )
            print(
                ' *                                                                                   *  '
                '\n ************************************************************************************* \n\n\n'
            )

            decision = int(
                input(
                    '* Enter "1" to start processing, or "0" to stop the script:     '
                ))
            if decision != 1:
                sys.exit(
                    '\n\n\n              ***  Program stopped! *** \n\n\n')

    # To print in console the header of first file
    print('\n  First file header parameters: \n')

    # *** Data file header read ***
    [
        df_filename, df_filesize, df_system_name, df_obs_place, df_description,
        CLCfrq, df_creation_timeUTC, Channel, ReceiverMode, Mode, Navr,
        TimeRes, fmin, fmax, df, frequency, freq_points_num, data_block_size
    ] = FileHeaderReaderJDS(directory + file_list[0], 0, 1)

    # Main loop by files start
    for file_no in range(len(file_list)):  # loop by files

        # *** Opening datafile ***
        fname = directory + file_list[file_no]

        # *********************************************************************************

        # *** Data file header read ***
        [
            df_filename, df_filesize, df_system_name, df_obs_place,
            df_description, CLCfrq, df_creation_timeUTC, Channel, ReceiverMode,
            Mode, Navr, TimeRes, fmin, fmax, df, frequency, freq_points_num,
            data_block_size
        ] = FileHeaderReaderJDS(fname, 0, 0)

        # Create long data files and copy first data file header to them
        if file_no == 0 and save_long_file_aver == 1:

            with open(fname, 'rb') as file:
                # *** Data file header read ***
                file_header = file.read(1024)

            # *** Creating a name for long timeline TXT file ***
            tl_file_name = df_filename + '_Timeline.txt'
            tl_file = open(
                tl_file_name,
                'w')  # Open and close to delete the file with the same name
            tl_file.close()

            # *** Creating a binary file with data for long data storage ***
            file_data_a_name = df_filename + '_Data_chA.dat'
            file_data_a = open(file_data_a_name, 'wb')
            file_data_a.write(file_header)
            file_data_a.seek(574)  # FFT size place in header
            file_data_a.write(np.int32(data_block_size).tobytes())
            file_data_a.seek(624)  # Lb place in header
            file_data_a.write(np.int32(0).tobytes())
            file_data_a.seek(628)  # Hb place in header
            file_data_a.write(np.int32(data_block_size / 2).tobytes())
            file_data_a.seek(632)  # Wb place in header
            file_data_a.write(np.int32(data_block_size / 2).tobytes())
            file_data_a.seek(636)  # Navr place in header
            file_data_a.write(
                bytes([np.int32(Navr * no_of_spectra_to_average)]))
            file_data_a.close()

            if Channel == 2:
                file_data_b_name = df_filename + '_Data_chB.dat'
                file_data_b = open(file_data_b_name, 'wb')
                file_data_b.write(file_header)
                file_data_b.seek(574)  # FFT size place in header
                file_data_b.write(np.int32(data_block_size).tobytes())
                file_data_b.seek(624)  # Lb place in header
                file_data_b.write(np.int32(0).tobytes())
                file_data_b.seek(628)  # Hb place in header
                file_data_b.write(np.int32(data_block_size / 2).tobytes())
                file_data_b.seek(632)  # Wb place in header
                file_data_b.write(np.int32(data_block_size / 2).tobytes())
                file_data_b.seek(636)  # Navr place in header
                file_data_b.write(
                    bytes([np.int32(Navr * no_of_spectra_to_average)]))
                file_data_b.close()

            del file_header

        # !!! Make automatic calculations of time and frequency resolutions for waveform mode!!!

        # Manually set frequencies for one channel mode

        if (Channel == 0 and int(CLCfrq / 1000000)
                == 66) or (Channel == 1 and int(CLCfrq / 1000000) == 66):
            freq_points_num = 8192
            frequency = np.linspace(0.0, 33.0, freq_points_num)

        # Manually set frequencies for two channels mode
        if Channel == 2 or (Channel == 0 and int(CLCfrq / 1000000) == 33) or (
                Channel == 1 and int(CLCfrq / 1000000) == 33):
            freq_points_num = 8192
            frequency = np.linspace(16.5, 33.0, freq_points_num)
        # For new receiver (temporary):
        if Channel == 2 and int(CLCfrq / 1000000) == 80:
            freq_points_num = 8192
            frequency = np.linspace(0.0, 40.0, freq_points_num)

        # Calculation of number of blocks and number of spectra in the file
        if Channel == 0 or Channel == 1:  # Single channel mode
            no_of_av_spectra_per_file = (df_filesize - 1024) / (
                2 * data_block_size * no_of_spectra_to_average)
        else:  # Two channels mode
            no_of_av_spectra_per_file = (df_filesize - 1024) / (
                4 * data_block_size * no_of_spectra_to_average)

        no_of_blocks_in_file = (df_filesize - 1024) / data_block_size

        no_of_av_spectra_per_file = int(no_of_av_spectra_per_file)
        fine_clock_frq = (int(CLCfrq / 1000000.0) * 1000000.0)

        # Real time resolution of averaged spectra
        real_av_spectra_dt = (1 / fine_clock_frq) * (
            data_block_size - 4) * no_of_spectra_to_average

        if file_no == 0:
            print(' Number of blocks in file:             ',
                  no_of_blocks_in_file)
            print(' Number of spectra to average:         ',
                  no_of_spectra_to_average)
            print(' Number of averaged spectra in file:   ',
                  no_of_av_spectra_per_file)
            print(' Time resolution of averaged spectrum: ',
                  round(real_av_spectra_dt * 1000, 3), ' ms.')
            print('\n  *** Reading data from file *** \n')

        # *******************************************************************************
        #                           R E A D I N G   D A T A                             *
        # *******************************************************************************

        with open(fname, 'rb') as file:
            file.seek(
                1024 + data_block_size * 4 *
                skip_data_blocks)  # Jumping to 1024 byte from file beginning

            # *** DATA READING process ***

            # Preparing arrays for dynamic spectra
            dyn_spectra_ch_a = np.zeros(
                (int(data_block_size / 2), no_of_av_spectra_per_file), float)
            if Channel == 2:  # Two channels mode
                dyn_spectra_ch_b = np.zeros(
                    (int(data_block_size / 2), no_of_av_spectra_per_file),
                    float)

            # !!! Fake timing. Real timing to be done!!!
            # TimeFigureScaleFig = np.linspace(0, no_of_av_spectra_per_file, no_of_av_spectra_per_file+1)
            # for i in range(no_of_av_spectra_per_file):
            #     TimeFigureScaleFig[i] = str(TimeFigureScaleFig[i])

            time_scale_fig = []
            time_scale_full = []

            bar = IncrementalBar(' File ' + str(file_no + 1) + ' of ' +
                                 str(len(file_list)) + ' reading: ',
                                 max=no_of_av_spectra_per_file,
                                 suffix='%(percent)d%%')

            for av_sp in range(no_of_av_spectra_per_file):

                bar.next()

                # Reading and reshaping all data with readers
                if Channel == 0 or Channel == 1:  # Single channel mode
                    wf_data = np.fromfile(file,
                                          dtype='i2',
                                          count=no_of_spectra_to_average *
                                          data_block_size)
                    wf_data = np.reshape(
                        wf_data, [data_block_size, no_of_spectra_to_average],
                        order='F')
                if Channel == 2:  # Two channels mode
                    wf_data = np.fromfile(file,
                                          dtype='i2',
                                          count=2 * no_of_spectra_to_average *
                                          data_block_size)
                    wf_data = np.reshape(
                        wf_data,
                        [data_block_size, 2 * no_of_spectra_to_average],
                        order='F')

                # Timing
                timeline_block_str = jds_waveform_time(wf_data, CLCfrq,
                                                       data_block_size)
                time_scale_fig.append(timeline_block_str[-1][0:12])
                time_scale_full.append(df_creation_timeUTC[0:10] + ' ' +
                                       timeline_block_str[-1][0:12])

                # Nulling the time blocks in waveform data
                wf_data[data_block_size - 4:data_block_size, :] = 0

                # Scaling of the data - seems to be wrong in absolute value
                wf_data = wf_data / 32768.0

                if Channel == 0 or Channel == 1:  # Single channel mode
                    wf_data_ch_a = wf_data  # All the data is channel A data
                    del wf_data  # Deleting unnecessary array to free the memory

                if Channel == 2:  # Two channels mode

                    # Resizing to obtain the matrix for separation of channels
                    wf_data_new = np.zeros(
                        (2 * data_block_size, no_of_spectra_to_average))
                    for i in range(2 * no_of_spectra_to_average):
                        if i % 2 == 0:
                            wf_data_new[0:data_block_size,
                                        int(i / 2)] = wf_data[:, i]  # Even
                        else:
                            wf_data_new[data_block_size:2 * data_block_size,
                                        int(i / 2)] = wf_data[:, i]  # Odd
                    del wf_data  # Deleting unnecessary array to free the memory

                    # Separating the data into two channels
                    wf_data_ch_a = np.zeros(
                        (data_block_size,
                         no_of_spectra_to_average))  # Preparing empty array
                    wf_data_ch_b = np.zeros(
                        (data_block_size,
                         no_of_spectra_to_average))  # Preparing empty array
                    wf_data_ch_a[:, :] = wf_data_new[0:(
                        2 * data_block_size):2, :]  # Separation to channel A
                    wf_data_ch_b[:, :] = wf_data_new[1:(
                        2 * data_block_size):2, :]  # Separation to channel B
                    del wf_data_new

                # preparing matrices for spectra
                spectra_ch_a = np.zeros_like(wf_data_ch_a)
                if Channel == 2:
                    spectra_ch_b = np.zeros_like(wf_data_ch_b)

                # Calculation of spectra
                for i in range(no_of_spectra_to_average):
                    spectra_ch_a[:, i] = np.power(
                        np.abs(np.fft.fft(wf_data_ch_a[:, i])), 2)
                    if Channel == 2:  # Two channels mode
                        spectra_ch_b[:, i] = np.power(
                            np.abs(np.fft.fft(wf_data_ch_b[:, i])), 2)

                # Storing only first (left) mirror part of spectra
                spectra_ch_a = spectra_ch_a[:int(data_block_size / 2), :]
                if Channel == 2:
                    spectra_ch_b = spectra_ch_b[:int(data_block_size / 2), :]

                # At 33 MHz the specter is usually upside down, to correct it we use flip up/down
                if int(CLCfrq / 1000000) == 33:
                    spectra_ch_a = np.flipud(spectra_ch_a)
                    if Channel == 2:
                        spectra_ch_b = np.flipud(spectra_ch_b)

                # Plotting first waveform block and first immediate spectrum in a file
                if av_sp == 0:  # First data block in a file
                    i = 0  # First immediate spectrum in a block

                    # Prepare parameters for plot
                    data_1 = wf_data_ch_a[:, i]
                    if Channel == 0 or Channel == 1:  # Single channel mode
                        no_of_sets = 1
                        data_2 = []
                    if Channel == 2:
                        no_of_sets = 2
                        data_2 = wf_data_ch_b[:, i]

                    suptitle = ('Waveform data, first block in file ' +
                                str(df_filename))
                    Title = (ReceiverMode + ', Fclock = ' +
                             str(round(CLCfrq / 1000000, 1)) +
                             ' MHz, Description: ' + str(df_description))

                    TwoOrOneValuePlot(
                        no_of_sets,
                        np.linspace(no_of_sets, data_block_size,
                                    data_block_size), data_1, data_2,
                        'Channel A', 'Channel B', 1, data_block_size, -0.6,
                        0.6, -0.6, 0.6, 'ADC clock counts', 'Amplitude, V',
                        'Amplitude, V', suptitle, Title, service_folder + '/' +
                        df_filename[0:14] + ' Waveform first data block.png',
                        current_date, current_time, software_version)

                    # Prepare parameters for plot
                    data_1 = 10 * np.log10(spectra_ch_a[:, i])
                    if Channel == 0 or Channel == 1:  # Single channel mode
                        no_of_sets = 1
                        data_2 = []
                    if Channel == 2:
                        no_of_sets = 2
                        data_2 = 10 * np.log10(spectra_ch_b[:, i])

                    suptitle = ('Immediate spectrum, first in file ' +
                                str(df_filename))
                    Title = (ReceiverMode + ', Fclock = ' +
                             str(round(CLCfrq / 1000000, 1)) +
                             ' MHz, Description: ' + str(df_description))

                    TwoOrOneValuePlot(
                        no_of_sets, frequency, data_1, data_2, 'Channel A',
                        'Channel B', frequency[0], frequency[-1], -80, 60, -80,
                        60, 'Frequency, MHz', 'Intensity, dB', 'Intensity, dB',
                        suptitle, Title,
                        service_folder + '/' + df_filename[0:14] +
                        ' Immediate spectrum first in file.png', current_date,
                        current_time, software_version)

                # Deleting the unnecessary matrices
                del wf_data_ch_a
                if Channel == 2:
                    del wf_data_ch_b

                # Calculation the averaged spectrum
                aver_spectra_ch_a = spectra_ch_a.mean(axis=1)[:]
                if Channel == 2:
                    aver_spectra_ch_b = spectra_ch_b.mean(axis=1)[:]

                # Plotting only first averaged spectrum
                if av_sp == 0:

                    # Prepare parameters for plot
                    data_1 = 10 * np.log10(aver_spectra_ch_a)
                    if Channel == 0 or Channel == 1:  # Single channel mode
                        no_of_sets = 1
                        data_2 = []
                    if Channel == 2:
                        no_of_sets = 2
                        data_2 = 10 * np.log10(aver_spectra_ch_b)

                    suptitle = ('Average spectrum, first data block in file ' +
                                str(df_filename))
                    Title = (ReceiverMode + ', Fclock = ' +
                             str(round(CLCfrq / 1000000, 1)) +
                             ' MHz, Avergaed spectra: ' +
                             str(no_of_spectra_to_average) +
                             ', Description: ' + str(df_description))

                    TwoOrOneValuePlot(
                        no_of_sets, frequency, data_1, data_2, 'Channel A',
                        'Channel B', frequency[0], frequency[-1], -80, 60, -80,
                        60, 'Frequency, MHz', 'Intensity, dB', 'Intensity, dB',
                        suptitle, Title,
                        service_folder + '/' + df_filename[0:14] +
                        ' Average spectrum first data block in file.png',
                        current_date, current_time, software_version)

                # Adding calculated averaged spectrum to dynamic spectra array
                dyn_spectra_ch_a[:, av_sp] = aver_spectra_ch_a[:]
                if Channel == 2:
                    dyn_spectra_ch_b[:, av_sp] = aver_spectra_ch_b[:]

            bar.finish()

        # file.close()  # Close the data file

        # Saving averaged spectra to long data files
        if save_long_file_aver == 1:
            temp = dyn_spectra_ch_a.transpose().copy(order='C')
            file_data_a = open(file_data_a_name, 'ab')
            file_data_a.write(temp)
            file_data_a.close()
            if Channel == 2:
                temp = dyn_spectra_ch_b.transpose().copy(order='C')
                file_data_b = open(file_data_b_name, 'ab')
                file_data_b.write(temp)
                file_data_b.close()

            # Saving time data to ling timeline file
            with open(tl_file_name, 'a') as tl_file:
                for i in range(no_of_av_spectra_per_file):
                    tl_file.write((time_scale_full[i][:]) + ' \n')  # str
            del time_scale_full

        # Log data (make dB scale)
        with np.errstate(invalid='ignore', divide='ignore'):
            dyn_spectra_ch_a = 10 * np.log10(dyn_spectra_ch_a)
            if Channel == 2:
                dyn_spectra_ch_b = 10 * np.log10(dyn_spectra_ch_b)

        # If the data contains minus infinity values change them to particular values
        dyn_spectra_ch_a[np.isinf(dyn_spectra_ch_a)] = 40
        if Channel == 2:
            dyn_spectra_ch_b[np.isinf(dyn_spectra_ch_b)] = 40

        # *******************************************************************************
        #             P L O T T I N G    D Y N A M I C    S P E C T R A                 *
        # *******************************************************************************

        # if dyn_spectr_save_init == 1 or dyn_spectr_save_norm == 1:
        #    print('\n  *** Making figures of dynamic spectra *** \n')

        if dyn_spectr_save_init == 1:
            # Plot of initial dynamic spectra

            v_min_a = np.min(dyn_spectra_ch_a)
            v_max_a = np.max(dyn_spectra_ch_a)
            v_min_b = v_min_a
            v_max_b = v_max_a
            if Channel == 2:
                v_min_b = np.min(dyn_spectra_ch_b)
                v_max_b = np.max(dyn_spectra_ch_b)

            if Channel == 0 or Channel == 1:  # Single channel mode
                dyn_spectra_ch_b = dyn_spectra_ch_a

            suptitle = ('Dynamic spectrum (initial) ' + str(df_filename) +
                        ' - Fig. ' + str(1) + ' of ' + str(1) +
                        '\n Initial parameters: dt = ' +
                        str(round(TimeRes * 1000., 3)) + ' ms, df = ' +
                        str(round(df / 1000., 3)) + ' kHz, Receiver: ' +
                        str(df_system_name) + ', Place: ' + str(df_obs_place) +
                        '\n' + ReceiverMode + ', Fclock = ' +
                        str(round(CLCfrq / 1000000, 1)) +
                        ' MHz, Avergaed spectra: ' +
                        str(no_of_spectra_to_average) + ' (' +
                        str(round(no_of_spectra_to_average * TimeRes, 3)) +
                        ' sec.), Description: ' + str(df_description))

            fig_file_name = (initial_spectra_folder + '/' + df_filename[0:14] +
                             ' Initial dynamic spectrum fig.' + str(0 + 1) +
                             '.png')

            if Channel == 0 or Channel == 1:  # Single channel mode
                OneDynSpectraPlot(dyn_spectra_ch_a, v_min_a, v_max_a, suptitle,
                                  'Intensity, dB', no_of_av_spectra_per_file,
                                  time_scale_fig, frequency, freq_points_num,
                                  colormap, 'UTC Time, HH:MM:SS.msec',
                                  fig_file_name, current_date, current_time,
                                  software_version, custom_dpi)

            if Channel == 2:
                TwoDynSpectraPlot(dyn_spectra_ch_a, dyn_spectra_ch_b, v_min_a,
                                  v_max_a, v_min_b, v_max_b, suptitle,
                                  'Intensity, dB', 'Intensity, dB',
                                  no_of_av_spectra_per_file, time_scale_fig,
                                  time_scale_fig, frequency, freq_points_num,
                                  colormap, 'Channel A', 'Channel B',
                                  fig_file_name, current_date, current_time,
                                  software_version, custom_dpi)

        if dyn_spectr_save_norm == 1:

            # Normalization and cleaning of data

            Normalization_dB(dyn_spectra_ch_a.transpose(), freq_points_num,
                             no_of_av_spectra_per_file)
            if Channel == 2:
                Normalization_dB(dyn_spectra_ch_b.transpose(), freq_points_num,
                                 no_of_av_spectra_per_file)

            simple_channel_clean(dyn_spectra_ch_a, 8)
            if Channel == 2:
                simple_channel_clean(dyn_spectra_ch_b, 8)

            # Plot of normalized and cleaned dynamic spectra

            suptitle = ('Normalized and cleaned dynamic spectrum (initial) ' +
                        str(df_filename) + ' - Fig. ' + str(0 + 1) + ' of ' +
                        str(1) + '\n Initial parameters: dt = ' +
                        str(round(TimeRes * 1000, 3)) + ' ms, df = ' +
                        str(round(df / 1000., 3)) + ' kHz, Receiver: ' +
                        str(df_system_name) + ', Place: ' + str(df_obs_place) +
                        '\n' + ReceiverMode + ', Fclock = ' +
                        str(round(CLCfrq / 1000000, 1)) +
                        ' MHz, Avergaed spectra: ' +
                        str(no_of_spectra_to_average) + ' (' +
                        str(round(no_of_spectra_to_average * TimeRes, 3)) +
                        ' sec.), Description: ' + str(df_description))

            fig_file_name = (result_folder + '/' + df_filename[0:14] +
                             ' Normalized and cleaned dynamic spectrum fig.' +
                             str(0 + 1) + '.png')

            if Channel == 0 or Channel == 1:  # Single channel mode
                OneDynSpectraPlot(dyn_spectra_ch_a, VminNorm, VmaxNorm,
                                  suptitle, 'Intensity, dB',
                                  no_of_av_spectra_per_file, time_scale_fig,
                                  frequency, freq_points_num, colormap,
                                  'UTC Time, HH:MM:SS.msec', fig_file_name,
                                  current_date, current_time, software_version,
                                  custom_dpi)
            if Channel == 2:
                TwoDynSpectraPlot(dyn_spectra_ch_a, dyn_spectra_ch_b, VminNorm,
                                  VmaxNorm, VminNorm, VmaxNorm, suptitle,
                                  'Intensity, dB', 'Intensity, dB',
                                  no_of_av_spectra_per_file, time_scale_fig,
                                  time_scale_fig, frequency, freq_points_num,
                                  colormap, 'Channel A', 'Channel B',
                                  fig_file_name, current_date, current_time,
                                  software_version, custom_dpi)
        del time_scale_fig, file_data_a
        if Channel == 2:
            del file_data_b

    results_files_list = []
    results_files_list.append(file_data_a_name)
    if Channel == 2:
        results_files_list.append(file_data_b_name)

    return results_files_list
Exemple #9
0
def jds_wf_simple_reader(directory, no_of_spectra_to_average, skip_data_blocks,
                         VminNorm, VmaxNorm, colormap, custom_dpi,
                         save_long_file_aver, dyn_spectr_save_init,
                         dyn_spectr_save_norm):
    """
    Does not seem to work better or faster, takes a lot of RAM (32 GB) but works
    Is not used in any other scripts and is more a demonstration
    The same functions in non-fast file works approximately the same time but consumes less memory
    The only advantage of this function is reading the whole file at once
    """

    current_time = time.strftime("%H:%M:%S")
    current_date = time.strftime("%d.%m.%Y")

    # *** Creating a folder where all pictures and results will be stored (if it doesn't exist) ***
    # result_folder = 'RESULTS_JDS_waveform_' + directory.split('/')[-2]
    result_folder = 'RESULTS_JDS_waveform'
    if not os.path.exists(result_folder):
        os.makedirs(result_folder)
    if dyn_spectr_save_init == 1:
        initial_spectra_folder = result_folder + '/Initial spectra'
        if not os.path.exists(initial_spectra_folder):
            os.makedirs(initial_spectra_folder)

    # *** Search JDS files in the directory ***

    file_list = find_files_only_in_current_folder(directory, '.jds', 1)
    print('')

    if len(
            file_list
    ) > 1:  # Check if files have same parameters if there are more then one file in list
        # Check if all files (except the last) have same size
        same_or_not = check_if_all_files_of_same_size(directory, file_list, 1)

        # Check if all files in this folder have the same parameters in headers
        equal_or_not = check_if_JDS_files_of_equal_parameters(
            directory, file_list)

        if same_or_not and equal_or_not:
            print(
                '\n\n\n        :-)  All files seem to be of the same parameters!  :-) \n\n\n'
            )
        else:
            print(
                '\n\n\n ************************************************************************************* '
            )
            print(
                ' *                                                                                   *'
            )
            print(
                ' *   Seems files in folders are different check the errors and restart the script!   *'
            )
            print(
                ' *                                                                                   *  '
                '\n ************************************************************************************* \n\n\n'
            )

            decision = int(
                input(
                    '* Enter "1" to start processing, or "0" to stop the script:     '
                ))
            if decision != 1:
                sys.exit(
                    '\n\n\n              ***  Program stopped! *** \n\n\n')

    # To print in console the header of first file
    print('\n  First file header parameters: \n')

    # *** Data file header read ***
    [
        df_filename, df_filesize, df_system_name, df_obs_place, df_description,
        CLCfrq, df_creation_timeUTC, Channel, ReceiverMode, Mode, Navr,
        TimeRes, fmin, fmax, df, frequency, freq_points_num, data_block_size
    ] = FileHeaderReaderJDS(directory + file_list[0], 0, 1)

    # Main loop by files start
    for fileNo in range(len(file_list)):  # loop by files

        # *** Opening datafile ***
        fname = directory + file_list[fileNo]

        # *********************************************************************************

        # *** Data file header read ***
        [
            df_filename, df_filesize, df_system_name, df_obs_place,
            df_description, CLCfrq, df_creation_timeUTC, Channel, ReceiverMode,
            Mode, Navr, TimeRes, fmin, fmax, df, frequency, freq_points_num,
            data_block_size
        ] = FileHeaderReaderJDS(fname, 0, 0)

        # Create long data files and copy first data file header to them
        if fileNo == 0 and save_long_file_aver == 1:

            with open(fname, 'rb') as file:
                # *** Data file header read ***
                file_header = file.read(1024)

            # *** Creating a name for long timeline TXT file ***
            TLfile_name = df_filename + '_Timeline.txt'
            TLfile = open(
                TLfile_name,
                'w')  # Open and close to delete the file with the same name
            TLfile.close()

            # *** Creating a binary file with data for long data storage ***
            file_data_A_name = df_filename + '_Data_chA.dat'
            file_data_A = open(file_data_A_name, 'wb')
            file_data_A.write(file_header)
            file_data_A.seek(574)  # FFT size place in header
            file_data_A.write(np.int32(data_block_size).tobytes())
            file_data_A.seek(624)  # Lb place in header
            file_data_A.write(np.int32(0).tobytes())
            file_data_A.seek(628)  # Hb place in header
            file_data_A.write(np.int32(data_block_size / 2).tobytes())
            file_data_A.seek(632)  # Wb place in header
            file_data_A.write(np.int32(data_block_size / 2).tobytes())
            file_data_A.seek(636)  # Navr place in header
            file_data_A.write(
                bytes([np.int32(Navr * no_of_spectra_to_average)]))
            file_data_A.close()

            if Channel == 2:
                file_data_B_name = df_filename + '_Data_chB.dat'
                file_data_B = open(file_data_B_name, 'wb')
                file_data_B.write(file_header)
                file_data_B.seek(574)  # FFT size place in header
                file_data_B.write(np.int32(data_block_size).tobytes())
                file_data_B.seek(624)  # Lb place in header
                file_data_B.write(np.int32(0).tobytes())
                file_data_B.seek(628)  # Hb place in header
                file_data_B.write(np.int32(data_block_size / 2).tobytes())
                file_data_B.seek(632)  # Wb place in header
                file_data_B.write(np.int32(data_block_size / 2).tobytes())
                file_data_B.seek(636)  # Navr place in header
                file_data_B.write(
                    bytes([np.int32(Navr * no_of_spectra_to_average)]))
                file_data_B.close()

            del file_header

        # !!! Make automatic calculations of time and frequency resolutions for waveform mode!!!

        # Manually set frequencies for one channel mode

        if (Channel == 0 and int(CLCfrq / 1000000)
                == 66) or (Channel == 1 and int(CLCfrq / 1000000) == 66):
            freq_points_num = 8192
            frequency = np.linspace(0.0, 33.0, freq_points_num)

        # Manually set frequencies for two channels mode
        if Channel == 2 or (Channel == 0 and int(CLCfrq / 1000000) == 33) or (
                Channel == 1 and int(CLCfrq / 1000000) == 33):
            freq_points_num = 8192
            frequency = np.linspace(16.5, 33.0, freq_points_num)
        # For new receiver (temporary):
        if Channel == 2 and int(CLCfrq / 1000000) == 80:
            freq_points_num = 8192
            frequency = np.linspace(0.0, 40.0, freq_points_num)

        # Calculation of number of blocks and number of spectra in the file
        if Channel == 0 or Channel == 1:  # Single channel mode
            no_of_av_spectra_per_file = (df_filesize - 1024) / (
                2 * data_block_size * no_of_spectra_to_average)
        else:  # Two channels mode
            no_of_av_spectra_per_file = (df_filesize - 1024) / (
                4 * data_block_size * no_of_spectra_to_average)

        no_of_blocks_in_file = (df_filesize - 1024) / data_block_size

        no_of_av_spectra_per_file = int(no_of_av_spectra_per_file)
        fine_CLCfrq = (int(CLCfrq / 1000000.0) * 1000000.0)

        # Real time resolution of averaged spectra
        real_av_spectra_dt = (1 / fine_CLCfrq) * (data_block_size -
                                                  4) * no_of_spectra_to_average

        if fileNo == 0:
            print(' Number of blocks in file:             ',
                  no_of_blocks_in_file)
            print(' Number of spectra to average:         ',
                  no_of_spectra_to_average)
            print(' Number of averaged spectra in file:   ',
                  no_of_av_spectra_per_file)
            print(' Time resolution of averaged spectrum: ',
                  round(real_av_spectra_dt * 1000, 3), ' ms.')
            print('\n  *** Reading data from file *** \n')

        # *******************************************************************************
        #                           R E A D I N G   D A T A                             *
        # *******************************************************************************

        with open(fname, 'rb') as file:
            file.seek(
                1024 + data_block_size * 4 *
                skip_data_blocks)  # Jumping to 1024 byte from file beginning

            # *** DATA READING process ***

            # !!! Fake timing. Real timing to be done!!!
            TimeFigureScaleFig = np.linspace(0, no_of_av_spectra_per_file,
                                             no_of_av_spectra_per_file + 1)
            for i in range(no_of_av_spectra_per_file):
                TimeFigureScaleFig[i] = str(TimeFigureScaleFig[i])

            TimeScaleFig = []
            TimeScaleFull = []

            # Calculation of number of blocks and number of spectra in the file
            if Channel == 0 or Channel == 1:  # Single channel mode
                no_of_spectra_in_file = int(
                    (df_filesize - 1024) / (1 * 2 * data_block_size))
            else:  # Two channels mode
                no_of_spectra_in_file = int(
                    (df_filesize - 1024) / (1 * 4 * data_block_size))

            no_of_av_spectra_per_file = 1

            # Reading and reshaping all data with time data
            if Channel == 0 or Channel == 1:  # Single channel mode
                wf_data = np.fromfile(file,
                                      dtype='i2',
                                      count=no_of_spectra_in_file *
                                      data_block_size)
                wf_data = np.reshape(wf_data,
                                     [data_block_size, no_of_spectra_in_file],
                                     order='F')
            if Channel == 2:  # Two channels mode
                wf_data = np.fromfile(file,
                                      dtype='i2',
                                      count=2 * no_of_spectra_in_file *
                                      data_block_size)
                wf_data = np.reshape(
                    wf_data, [data_block_size, 2 * no_of_spectra_in_file],
                    order='F')

            print('Waveform read, shape: ', wf_data.shape)

            # Timing
            timeline_block_str = jds_waveform_time(wf_data, CLCfrq,
                                                   data_block_size)
            timeline_block_str = timeline_block_str[0::16]
            for j in range(len(timeline_block_str)):
                TimeScaleFig.append(timeline_block_str[j][0:12])
                TimeScaleFull.append(df_creation_timeUTC[0:10] + ' ' +
                                     timeline_block_str[j][0:12])

            # Nulling the time blocks in waveform data
            wf_data[data_block_size - 4:data_block_size, :] = 0

            # Scaling of the data - seems to be wrong in absolute value
            wf_data = wf_data / 32768.0

            if Channel == 0 or Channel == 1:  # Single channel mode
                wf_data_chA = wf_data  # All the data is channel A data
                del wf_data  # Deleting unnecessary array to free the memory

            if Channel == 2:  # Two channels mode

                # Resizing to obtain the matrix for separation of channels
                wf_data_new = np.zeros(
                    (2 * data_block_size, no_of_spectra_in_file))
                for i in range(2 * no_of_spectra_in_file):
                    if i % 2 == 0:
                        wf_data_new[0:data_block_size,
                                    int(i / 2)] = wf_data[:, i]  # Even
                    else:
                        wf_data_new[data_block_size:2 * data_block_size,
                                    int(i / 2)] = wf_data[:, i]  # Odd
                del wf_data  # Deleting unnecessary array to free the memory

                # Separating the data into two channels
                wf_data_chA = np.zeros(
                    (data_block_size,
                     no_of_spectra_in_file))  # Preparing empty array
                wf_data_chB = np.zeros(
                    (data_block_size,
                     no_of_spectra_in_file))  # Preparing empty array
                wf_data_chA[:, :] = wf_data_new[0:(
                    2 * data_block_size):2, :]  # Separation to channel A
                wf_data_chB[:, :] = wf_data_new[1:(
                    2 * data_block_size):2, :]  # Separation to channel B
                del wf_data_new

            print('Before transpose, shape: ', wf_data_chA.shape)

            # preparing matrices for spectra
            wf_data_chA = np.transpose(wf_data_chA)
            spectra_ch_a = np.zeros_like(wf_data_chA)
            if Channel == 2:
                wf_data_chB = np.transpose(wf_data_chB)
                spectra_ch_b = np.zeros_like(wf_data_chB)

            print('After transpose, shape: ', wf_data_chA.shape)

            # Calculation of spectra
            spectra_ch_a[:] = np.power(np.abs(np.fft.fft(wf_data_chA[:])), 2)
            if Channel == 2:  # Two channels mode
                spectra_ch_b[:] = np.power(np.abs(np.fft.fft(wf_data_chB[:])),
                                           2)

            print('After fft, spectrum shape: ', spectra_ch_a.shape)

            # Storing only first (left) mirror part of spectra
            spectra_ch_a = spectra_ch_a[:, :int(data_block_size / 2)]
            if Channel == 2:
                spectra_ch_b = spectra_ch_b[:, :int(data_block_size / 2)]

            print('After fft cut, spectrum shape: ', spectra_ch_a.shape)

            # At 33 MHz the specter is usually upside down, to correct it we use flip up/down
            if int(CLCfrq / 1000000) == 33:
                spectra_ch_a = np.fliplr(spectra_ch_a)
                if Channel == 2:
                    spectra_ch_b = np.fliplr(spectra_ch_b)

            # Deleting the unnecessary matrices
            del wf_data_chA
            if Channel == 2:
                del wf_data_chB

            # Dimensions of [data_block_size / 2, no_of_spectra_in_file]

            # Calculation the averaged spectrum
            print('Shape before averaging: ', spectra_ch_a.shape)
            spectra_ch_a = np.reshape(spectra_ch_a, [
                int(no_of_spectra_in_file / no_of_spectra_to_average),
                no_of_spectra_to_average,
                int(data_block_size / 2)
            ],
                                      order='F')

            spectra_ch_a = spectra_ch_a.mean(axis=1)[:]

            print('Shape after averaging: ', spectra_ch_a.shape)

            if Channel == 2:
                spectra_ch_b = np.reshape(spectra_ch_b, [
                    int(no_of_spectra_in_file / no_of_spectra_to_average),
                    no_of_spectra_to_average,
                    int(data_block_size / 2)
                ],
                                          order='F')

                spectra_ch_b = spectra_ch_b.mean(axis=1)[:]

        file.close()  # Close the data file

        # Saving averaged spectra to a long data files
        if save_long_file_aver == 1:
            temp = spectra_ch_a.copy(order='C')
            file_data_A = open(file_data_A_name, 'ab')
            file_data_A.write(temp)
            file_data_A.close()
            if Channel == 2:
                temp = spectra_ch_b.copy(order='C')
                file_data_B = open(file_data_B_name, 'ab')
                file_data_B.write(temp)
                file_data_B.close()

            # Saving time data to long timeline file
            with open(TLfile_name, 'a') as TLfile:
                for i in range(no_of_av_spectra_per_file):
                    TLfile.write((TimeScaleFull[i][:]) + ' \n')  # str

        # Log data (make dB scale)
        with np.errstate(invalid='ignore', divide='ignore'):
            spectra_ch_a = 10 * np.log10(spectra_ch_a)
            if Channel == 2:
                spectra_ch_b = 10 * np.log10(spectra_ch_b)

        # If the data contains minus infinity values change them to particular values
        spectra_ch_a[np.isinf(spectra_ch_a)] = 40
        if Channel == 2:
            spectra_ch_b[np.isinf(spectra_ch_b)] = 40

        # *******************************************************************************
        #             P L O T T I N G    D Y N A M I C    S P E C T R A                 *
        # *******************************************************************************

        spectra_ch_a = np.transpose(spectra_ch_a)
        if Channel == 2:
            spectra_ch_b = np.transpose(spectra_ch_b)

        no_of_av_spectra_per_file = spectra_ch_a.shape[1]

        if dyn_spectr_save_init == 1:

            # Plot of initial dynamic spectra
            VminA = np.min(spectra_ch_a)
            VmaxA = np.max(spectra_ch_a)
            VminB = VminA
            VmaxB = VmaxA
            if Channel == 2:
                VminB = np.min(spectra_ch_b)
                VmaxB = np.max(spectra_ch_b)

            if Channel == 0 or Channel == 1:  # Single channel mode
                spectra_ch_b = spectra_ch_a

            suptitle = ('Dynamic spectrum (initial) ' + str(df_filename) +
                        ' - Fig. ' + str(1) + ' of ' + str(1) +
                        '\n Initial parameters: dt = ' +
                        str(round(TimeRes * 1000., 3)) + ' ms, df = ' +
                        str(round(df / 1000., 3)) + ' kHz, Receiver: ' +
                        str(df_system_name) + ', Place: ' + str(df_obs_place) +
                        '\n' + ReceiverMode + ', Fclock = ' +
                        str(round(CLCfrq / 1000000, 1)) +
                        ' MHz, Avergaed spectra: ' +
                        str(no_of_spectra_to_average) + ' (' +
                        str(round(no_of_spectra_to_average * TimeRes, 3)) +
                        ' sec.), Description: ' + str(df_description))

            fig_file_name = (initial_spectra_folder + '/' + df_filename[0:14] +
                             ' Initial dynamic spectrum fig.' + str(0 + 1) +
                             '.png')

            if Channel == 0 or Channel == 1:  # Single channel mode
                OneDynSpectraPlot(spectra_ch_a, VminA, VmaxA, suptitle,
                                  'Intensity, dB', no_of_av_spectra_per_file,
                                  TimeScaleFig, frequency, freq_points_num,
                                  colormap, 'UTC Time, HH:MM:SS.msec',
                                  fig_file_name, current_date, current_time,
                                  software_version, custom_dpi)

            if Channel == 2:
                TwoDynSpectraPlot(spectra_ch_a, spectra_ch_b, VminA, VmaxA,
                                  VminB, VmaxB, suptitle, 'Intensity, dB',
                                  'Intensity, dB', no_of_av_spectra_per_file,
                                  TimeScaleFig, TimeScaleFig, frequency,
                                  freq_points_num, colormap, 'Channel A',
                                  'Channel B', fig_file_name, current_date,
                                  current_time, software_version, custom_dpi)

        if dyn_spectr_save_norm == 1:

            # Normalization and cleaning of data

            Normalization_dB(spectra_ch_a.transpose(), freq_points_num,
                             no_of_av_spectra_per_file)
            if Channel == 2:
                Normalization_dB(spectra_ch_b.transpose(), freq_points_num,
                                 no_of_av_spectra_per_file)

            simple_channel_clean(spectra_ch_a, 8)
            if Channel == 2:
                simple_channel_clean(spectra_ch_b, 8)

            # Plot of normalized and cleaned dynamic spectra

            suptitle = ('Normalized and cleaned dynamic spectrum (initial) ' +
                        str(df_filename) + ' - Fig. ' + str(0 + 1) + ' of ' +
                        str(1) + '\n Initial parameters: dt = ' +
                        str(round(TimeRes * 1000, 3)) + ' ms, df = ' +
                        str(round(df / 1000., 3)) + ' kHz, Receiver: ' +
                        str(df_system_name) + ', Place: ' + str(df_obs_place) +
                        '\n' + ReceiverMode + ', Fclock = ' +
                        str(round(CLCfrq / 1000000, 1)) +
                        ' MHz, Avergaed spectra: ' +
                        str(no_of_spectra_to_average) + ' (' +
                        str(round(no_of_spectra_to_average * TimeRes, 3)) +
                        ' sec.), Description: ' + str(df_description))

            fig_file_name = (result_folder + '/' + df_filename[0:14] +
                             ' Normalized and cleaned dynamic spectrum fig.' +
                             str(0 + 1) + '.png')

            if Channel == 0 or Channel == 1:  # Single channel mode
                OneDynSpectraPlot(spectra_ch_a, VminNorm, VmaxNorm, suptitle,
                                  'Intensity, dB', no_of_av_spectra_per_file,
                                  TimeScaleFig, frequency, freq_points_num,
                                  colormap, 'UTC Time, HH:MM:SS.msec',
                                  fig_file_name, current_date, current_time,
                                  software_version, custom_dpi)
            if Channel == 2:
                TwoDynSpectraPlot(spectra_ch_a, spectra_ch_b, VminNorm,
                                  VmaxNorm, VminNorm, VmaxNorm, suptitle,
                                  'Intensity, dB', 'Intensity, dB',
                                  no_of_av_spectra_per_file, TimeScaleFig,
                                  TimeScaleFig, frequency, freq_points_num,
                                  colormap, 'Channel A', 'Channel B',
                                  fig_file_name, current_date, current_time,
                                  software_version, custom_dpi)

    results_files_list = []
    results_files_list.append(file_data_A_name)
    if Channel == 2:
        results_files_list.append(file_data_B_name)

    return results_files_list