def download_chapters(app): downlod_cover(app) bar = IncrementalBar('Downloading chapters', max=len(app.chapters)) bar.start() if os.getenv('debug_mode') == 'yes': bar.next = lambda: None # Hide in debug mode # end if futures_to_check = { app.crawler.executor.submit( download_chapter_body, app, chapter, ): str(chapter['id']) for chapter in app.chapters } app.progress = 0 for future in futures.as_completed(futures_to_check): result = future.result() if result: bar.clearln() logger.error(result) # end if app.progress += 1 bar.next() # end for bar.finish() print('Downloaded %d chapters' % len(app.chapters))
def download_chapters(app): download_cover(app) bar = IncrementalBar('Downloading chapters', max=len(app.chapters)) bar.start() if os.getenv('debug_mode') == 'true': bar.next = lambda: None futures_to_check = { app.crawler.executor.submit( download_chapter_body, app, chapter, ): str(chapter['id']) for chapter in app.chapters } for future in futures.as_completed(futures_to_check): result = future.result() if result: bar.clearln() app.logger.error(result) bar.next() bar.finish()
def search_novels(app): if not app.crawler_links: return bar = IncrementalBar('Searching', max=len(app.crawler_links)) if os.getenv('debug_mode') == 'yes': bar.next = lambda n=1: None # Hide in debug mode else: bar.start() # end if # Add future tasks checked = {} futures_to_check = [] app.progress = 0 for link in app.crawler_links: crawler = crawler_list[link] if crawler in checked: logger.info('A crawler for "%s" already exists', link) bar.next() continue # end if checked[crawler] = True future = executor.submit(get_search_result, app, link, bar) futures_to_check.append(future) # end for # Resolve all futures combined_results = [item for f in futures_to_check for item in f.result()] # Process combined search results app.search_results = process_results(combined_results) bar.clearln() bar.finish() print('Found %d results' % len(app.search_results))
def download_chapters(app): app.progress = 0 bar = IncrementalBar('Downloading chapters', max=len(app.chapters)) if os.getenv('debug_mode') == 'yes': bar.next = lambda: None # Hide in debug mode bar.finish() else: bar.start() # end if if not app.output_formats: app.output_formats = {} # end if futures_to_check = [ app.crawler.executor.submit( download_chapter_body, app, chapter, ) for chapter in app.chapters ] for future in futures_to_check: result = future.result() if result: bar.clearln() logger.error(result) # end if bar.next() # end for bar.finish() print('Processed %d chapters' % len(app.chapters))
def alignChapter(lang, bookid, chapter): """ Align a chapter of a book Args: lang (str): language bookid (str): identifier of a book chapter (int): the chapter to be aligned Returns: list of spacy tokens: the tokens with the added audio alignment information """ bar = IncrementalBar('Processing %s [%s] (%s)' % (bookid, lang, chapter) , max=100) bar.start() outfile = os.path.join(book_manager.chaptersPath(lang, bookid),book_manager.mappingFile(chapter)) audio_file, start_time, stop_time = book_manager.chapterAudio(lang, bookid, chapter) wavfile = os.path.join(config.TEMP_DIR, 'chapter%s.wav' % chapter) gu.removeFile(wavfile) encodeForSphinx(audio_file, start_time, stop_time, wavfile) # encode audio for speech recognition # get spacy models for language processing sp = utils.getSpacy(lang) text = book_manager.bookChapter(lang, bookid, chapter) doc = sp(text) # prepare sentences without punctuation token_count = 0 doc_tokens = [tkn for tkn in doc if tkn.is_alpha and (not tkn.is_punct) and tkn.text.strip()] token_count = len(doc_tokens) audio_segment = AudioSegment.from_wav(wavfile) # read the audio audio_len = len(audio_segment) begin_tkn = 0 begin_audio = 0 startm = time2msec(start_time) stopm = time2msec(stop_time) l = stopm - startm while begin_tkn < token_count: chunk = doc_tokens[begin_tkn:begin_tkn+50] rel_len = 1.25 * len(chunk) / token_count end_audio = begin_audio + int(rel_len * audio_len) last_idx, begin_audio = alignChunk(lang, audio_segment=audio_segment, audio_begin=begin_audio, audio_end=end_audio, chunk=chunk) bar.goto(int(100.0 * begin_audio / l)) if last_idx == -1: # could not map anything break else: begin_tkn += last_idx + 1 gu.removeFile(wavfile) saveAudioMapping(doc_tokens, startm, stopm, outfile)
def search_novels(app): executor = futures.ThreadPoolExecutor(10) # Add future tasks checked = {} futures_to_check = {} for link in app.crawler_links: crawler = crawler_list[link] if crawler in checked: logger.info('A crawler for "%s" already exists', link) continue # end if checked[crawler] = True futures_to_check[ executor.submit( get_search_result, app.user_input, link ) ] = str(crawler) # end for bar = IncrementalBar('Searching', max=len(futures_to_check.keys())) bar.start() if os.getenv('debug_mode') == 'yes': bar.next = lambda: None # Hide in debug mode # end if # Resolve future tasks app.progress = 0 combined_results = [] for future in futures.as_completed(futures_to_check): combined_results += future.result() app.progress += 1 bar.next() # end for # Process combined search results app.search_results = process_results(combined_results) bar.clearln() bar.finish() print('Found %d results' % len(app.search_results)) executor.shutdown()
def download_chapters(app): # download or generate cover app.book_cover = download_cover(app) if not app.book_cover: app.book_cover = generate_cover(app) # end if if not app.book_cover: logger.warn('No cover image') # end if bar = IncrementalBar('Downloading chapters', max=len(app.chapters)) bar.start() if os.getenv('debug_mode') == 'yes': bar.next = lambda: None # Hide in debug mode # end if if not app.output_formats: app.output_formats = {} # end if futures_to_check = { app.crawler.executor.submit( download_chapter_body, app, chapter, ): str(chapter['id']) for chapter in app.chapters } app.progress = 0 for future in futures.as_completed(futures_to_check): result = future.result() if result: bar.clearln() logger.error(result) # end if app.progress += 1 bar.next() # end for bar.finish() print('Downloaded %d chapters' % len(app.chapters))
def main(): """ Description: This function call all others in the right following order. """ path_csv = create_directory('bts_csv') path_images = create_directory('images_bts') step_1 = extract_category_list() # Initialization of Progress Bar bar = IncrementalBar('Progression', max=len(step_1)) bar.start() while step_1 is not []: if not step_1: break else: step_2 = extract_books_url(step_1) step_3 = transform_books_information(step_2, path_images) load_books_information(step_3, path_csv) del step_1[0] # Progress Bar incrementation : bar.next()
def bind(self): logger.debug('Binding %s.pdf', self.file_name) pdf_path = os.path.join(self.app.output_path, 'pdf') os.makedirs(pdf_path, exist_ok=True) all_pages = [] bar = IncrementalBar('Adding chapters to PDF', max=len(self.chapters)) bar.start() if os.getenv('debug_mode') == 'yes': bar.next = lambda: None # Hide in debug mode # end if html = HTML(string=self.create_intro()) all_pages += html.render().pages logger.info('Added intro page') for chapter in self.chapters: html_string = chapter['body'] html = HTML(string=html_string) all_pages += html.render().pages logger.info('Added chapter %d', chapter['id']) bar.next() # end for bar.finish() html = HTML(string=self.make_metadata()) combined = html.render().copy(all_pages) output_file = os.path.join(pdf_path, '%s.pdf' % self.file_name) combined.write_pdf(output_file) print('Created: %s.pdf' % self.file_name) return output_file
sleep() for spin in (Spinner, PieSpinner, MoonSpinner, LineSpinner, PixelSpinner): for i in spin(spin.__name__ + ' ').iter(range(100)): sleep() for singleton in (Counter, Countdown, Stack, Pie): for i in singleton(singleton.__name__ + ' ').iter(range(100)): sleep() bar = IncrementalBar('Random', suffix='%(index)d') for i in range(100): bar.goto(random.randint(0, 100)) sleep() bar.finish() """ import progressbar from time import sleep bar = progressbar.ProgressBar(maxval=20, \ widgets=[progressbar.Bar('=', '[', ']'), ' ', progressbar.Percentage()]) bar.start() for i in xrange(20): bar.update(i+1) sleep(0.1) bar.finish() """ # TQDM # https://notebooks.ai/ernest-galton/tqdm-ef22fcc1/lab # https://github.com/tqdm/tqdm/wiki/How-to-make-a-great-Progress-Bar # https://pypi.org/project/tqdm/
if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( 'path', help='Path to directory containing measurement data' ) args = parser.parse_args() # Use given path as working directory for script so glob works from there os.chdir(args.path) total_lines = get_total_lines(glob.iglob('**/*.CSV', recursive=True)) total_size = int(total_lines / BUFFER_SIZE) bar = IncrementalBar( max=total_size, suffix='%(percent)d%% [ETA: %(eta_td)s]' ) bar.start() csv_glob = peekable(glob.iglob('**/*.CSV', recursive=True)) line_reader = buffered_line_reader( line_reader(csv_glob), buffer_size=BUFFER_SIZE ) with requests.Session() as session: # Iterate through the progress bar to auto update the bar for lines in bar.iter(line_reader): bar.message = csv_glob.peek(bar.message) response = session.post(URL, data=lines) assert response.status_code == 204, response.text
def coherent_wf_to_wf_dedispersion(DM, fname, no_of_points_for_fft_dedisp): """ function reads waveform data in wf32 format, makes FFT, cuts the symmetrical half of the spectra and shifts the lines of complex data to provide coherent dedispersion. Then a symmetrcal part of spectra are made and joined to the shifted one, inverse FFT as applied and data are stored in waveform wf32 format Input parameters: DM - dispersion measure to compensate fname - name of file with initial wf32 data no_of_points_for_fft_dedisp - number of waveform data points to use for FFT Output parameters: file_data_name - name of file with processed data """ # *** Data file header read *** [ df_filename, df_filesize, df_system_name, df_obs_place, df_description, clock_freq, df_creation_timeUTC, Channel, ReceiverMode, Mode, Navr, time_resolution, fmin, fmax, df, frequency_list, freq_points_num, data_block_size ] = FileHeaderReaderJDS(fname, 0, 0) # Manually set frequencies for one channel mode freq_points_num = int(no_of_points_for_fft_dedisp / 2) # Manually set frequencies for 33 MHz clock frequency if int(clock_freq / 1000000) == 33: fmin = 16.5 fmax = 33.0 df = 16500000 / freq_points_num # Create long data files and copy first data file header to them with open(fname, 'rb') as file: # *** Data file header read *** file_header = file.read(1024) # Removing old DM from file name and updating it to current value if fname.startswith('DM_'): prev_dm_str = fname.split('_')[1] prev_dm = np.float32(prev_dm_str) new_dm = prev_dm + DM n = len('DM_' + prev_dm_str + '_') file_data_name = 'DM_' + str(np.round(new_dm, 6)) + '_' + fname[n:] else: file_data_name = 'DM_' + str(np.round(DM, 6)) + '_' + fname # *** Creating a binary file with data for long data storage *** file_data = open(file_data_name, 'wb') file_data.write(file_header) file_data.close() del file_header # *** Creating a new timeline TXT file for results *** new_tl_file_name = file_data_name.split("_Data_ch", 1)[0] + '_Timeline.wtxt' new_tl_file = open( new_tl_file_name, 'w') # Open and close to delete the file with the same name new_tl_file.close() # Calculation of the time shifts shift_vector = DM_full_shift_calc(freq_points_num, fmin, fmax, df / pow(10, 6), time_resolution, DM, 'jds') max_shift = np.abs(shift_vector[0]) # Preparing buffer array buffer_array = np.zeros((freq_points_num, 2 * max_shift), dtype='complex64') print(' Maximal shift is: ', max_shift, ' pixels ') print(' Dispersion measure: ', DM, ' pc / cm3 ') # Calculation of number of blocks and number of spectra in the file no_of_spectra_in_bunch = max_shift.copy() no_of_bunches_per_file = int( np.ceil( (df_filesize - 1024) / (no_of_spectra_in_bunch * no_of_points_for_fft_dedisp * 4))) # Real time resolution of spectra fine_clock_freq = (int(clock_freq / 1000000.0) * 1000000.0) real_spectra_dt = float(no_of_points_for_fft_dedisp / fine_clock_freq) real_spectra_df = float( (fine_clock_freq / 2) / (no_of_points_for_fft_dedisp / 2)) print(' Number of spectra in bunch: ', no_of_spectra_in_bunch) print(' Number of bunches to read in file: ', no_of_bunches_per_file) print(' Time resolution of calculated spectra: ', round(real_spectra_dt * 1000, 3), ' ms') print(' Frequency resolution of calculated spectra: ', round(real_spectra_df / 1000, 3), ' kHz \n') # !!! Fake timing. Real timing to be done!!! # *** Reading timeline file *** old_tl_file_name = fname.split("_Data_ch", 1)[0] + '_Timeline.wtxt' old_tl_file = open(old_tl_file_name, 'r') new_tl_file = open( new_tl_file_name, 'w') # Open and close to delete the file with the same name file.seek(1024) # Jumping to 1024 byte from file beginning # bar = IncrementalBar(' Coherent dispersion delay removing: ', max=no_of_bunches_per_file - 1, bar = IncrementalBar(' Coherent dispersion delay removing: ', max=no_of_bunches_per_file, suffix='%(percent)d%%') bar.start() # for bunch in range(no_of_bunches_per_file - 1): for bunch in range(no_of_bunches_per_file): # Trying to read all the file, not only integer number of bunches if bunch >= no_of_bunches_per_file - 1: no_of_spectra_in_bunch = int( ((df_filesize - 1024) - bunch * max_shift * no_of_points_for_fft_dedisp * 4) / (no_of_points_for_fft_dedisp * 4)) # print('\n Bunch No ', str(bunch+1), ' of ', no_of_bunches_per_file, ' bunches') # print('\n Number of spectra in the last bunch is: ', no_of_spectra_in_bunch) # print('\n Maximal shift is: ', max_shift) # Read time from timeline file for the bunch time_scale_bunch = [] for line in range(no_of_spectra_in_bunch): time_scale_bunch.append(str(old_tl_file.readline())) # Reading and reshaping all data with time data wf_data = np.fromfile(file, dtype='f4', count=no_of_spectra_in_bunch * no_of_points_for_fft_dedisp) ''' fig = plt.figure(figsize=(9, 5)) ax1 = fig.add_subplot(111) ax1.plot(wf_data, linestyle='-', linewidth='1.00', label='Initial waveform') ax1.legend(loc='upper right', fontsize=6) ax1.grid(b=True, which='both', color='silver', linestyle='-') ax1.set_ylabel('Intensity, a.u.', fontsize=6, fontweight='bold') pylab.savefig('00_Initial_waveform_' + str(bunch) + '.png', bbox_inches='tight', dpi=160) plt.close('all') ''' wf_data = np.reshape( wf_data, [no_of_points_for_fft_dedisp, no_of_spectra_in_bunch], order='F') # preparing matrices for spectra spectra = np.zeros( (no_of_points_for_fft_dedisp, no_of_spectra_in_bunch), dtype='complex64') # Calculation of spectra for i in range(no_of_spectra_in_bunch): spectra[:, i] = np.fft.fft(wf_data[:, i]) del wf_data ''' fig = plt.figure(figsize=(9, 5)) ax1 = fig.add_subplot(111) ax1.plot(10 * np.log10(np.power(np.abs(spectra[:, 0]), 2)), linestyle='-', linewidth='1.00', label='Initial spectra before cut') ax1.legend(loc='upper right', fontsize=6) ax1.grid(b=True, which='both', color='silver', linestyle='-') ax1.set_ylabel('Intensity, a.u.', fontsize=6, fontweight='bold') pylab.savefig('00a_Initial_doubled_imm_spectra' + str(bunch) + '.png', bbox_inches='tight', dpi=160) plt.close('all') ''' # Cut half of the spectra spectra = spectra[int(no_of_points_for_fft_dedisp / 2):, :] ''' # making figures fig = plt.figure(figsize=(9, 5)) ax1 = fig.add_subplot(111) ax1.imshow(np.flipud(10*np.log10(np.power(np.abs(spectra), 2))), aspect='auto', cmap='jet') ax1.set_ylabel('Frequency points', fontsize=6, fontweight='bold') ax1.set_xlabel('Time points', fontsize=6, fontweight='bold') pylab.savefig('01_Initial_spectra_' + str(bunch) + '.png', bbox_inches='tight', dpi=160) plt.close('all') fig = plt.figure(figsize=(9, 5)) ax1 = fig.add_subplot(111) ax1.plot(10*np.log10(np.power(np.abs(spectra[:, 0]), 2)), linestyle='-', linewidth='1.00', label='Initial waveform') ax1.legend(loc='upper right', fontsize=6) ax1.grid(b=True, which='both', color='silver', linestyle='-') ax1.set_ylabel('Intensity, a.u.', fontsize=6, fontweight='bold') pylab.savefig('02_Initial_imm_spectra' + str(bunch) + '.png', bbox_inches='tight', dpi=160) plt.close('all') ''' # Dispersion delay removing data_space = np.zeros((freq_points_num, 2 * max_shift), dtype='complex64') # if it is the last bunch - use only availble data if bunch >= no_of_bunches_per_file - 1: data_space[:, max_shift:max_shift + no_of_spectra_in_bunch] = spectra[:, :] else: data_space[:, max_shift:] = spectra[:, :] data_space = pulsar_DM_compensation_with_indices_changes( data_space, shift_vector) del spectra # Adding the next data block buffer_array += data_space # Making and filling the array with fully ready data for plotting and saving to a file if bunch >= no_of_bunches_per_file - 1: array_compensated_dm = buffer_array[:, 0:no_of_spectra_in_bunch] else: array_compensated_dm = buffer_array[:, 0:max_shift] if bunch > 0: # Saving time data to a new file for i in range(len(time_scale_bunch)): new_tl_file.write((time_scale_bunch[i][:]) + '') # Saving data with compensated DM spectra = array_compensated_dm # .copy() ''' # making figures fig = plt.figure(figsize=(9, 5)) ax1 = fig.add_subplot(111) ax1.imshow(np.flipud(10*np.log10(np.power(np.abs(spectra), 2))), aspect='auto', cmap='jet') ax1.set_ylabel('Frequency points', fontsize=6, fontweight='bold') ax1.set_xlabel('Time points', fontsize=6, fontweight='bold') pylab.savefig('03_Compensated_spectra_' + str(bunch) + '.png', bbox_inches='tight', dpi=160) plt.close('all') fig = plt.figure(figsize=(9, 5)) ax1 = fig.add_subplot(111) ax1.plot(10*np.log10(np.power(np.abs(spectra[:,0]), 2)), linestyle='-', linewidth='1.00', label='Initial waveform') ax1.legend(loc='upper right', fontsize=6) ax1.grid(b=True, which='both', color='silver', linestyle='-') ax1.set_ylabel('Intensity, a.u.', fontsize=6, fontweight='bold') pylab.savefig('04_Compensated_imm_spectra' + str(bunch) + '.png', bbox_inches='tight', dpi=160) plt.close('all') ''' wf_data = np.zeros( (no_of_points_for_fft_dedisp, no_of_spectra_in_bunch)) # Add lost half of the spectra second_spectra_half = spectra.copy() second_spectra_half = np.flipud(second_spectra_half) spectra = np.concatenate((second_spectra_half, spectra), axis=0) # Changed places!!! ''' fig = plt.figure(figsize=(9, 5)) ax1 = fig.add_subplot(111) ax1.plot(10*np.log10(np.power(np.abs(spectra[:,0]), 2)), linestyle='-', linewidth='1.00', label='Initial waveform') ax1.legend(loc='upper right', fontsize=6) ax1.grid(b=True, which='both', color='silver', linestyle='-') ax1.set_ylabel('Intensity, a.u.', fontsize=6, fontweight='bold') pylab.savefig('05_Compensated_doubled_imm_spectra' + str(bunch) + '.png', bbox_inches='tight', dpi=160) plt.close('all') ''' # Making IFFT for i in range(no_of_spectra_in_bunch): wf_data[:, i] = np.real(np.fft.ifft(spectra[:, i])) del spectra # Reshaping the waveform to single dimension (real) wf_data = np.reshape( wf_data, [no_of_points_for_fft_dedisp * no_of_spectra_in_bunch, 1], order='F') ''' # making figures fig = plt.figure(figsize=(9, 5)) ax1 = fig.add_subplot(111) ax1.plot(wf_data, linestyle='-', linewidth='1.00', label='Initial waveform') ax1.legend(loc='upper right', fontsize=6) ax1.grid(b=True, which='both', color='silver', linestyle='-') ax1.set_ylabel('Intensity, a.u.', fontsize=6, fontweight='bold') pylab.savefig('06_Compensated_waveform_' + str(bunch) + '.png', bbox_inches='tight', dpi=160) plt.close('all') ''' # Saving waveform data to wf32 file file_data = open(file_data_name, 'ab') file_data.write( np.float32(wf_data).transpose().copy(order='C')) file_data.close() # !!! Saving time data to timeline file !!! # Rolling temp_array to put current data first buffer_array = np.roll(buffer_array, -max_shift) buffer_array[:, max_shift:] = 0 bar.next() bar.finish() old_tl_file.close() new_tl_file.close() return file_data_name
def convert_wf32_to_dat_with_overlap(fname, no_of_points_for_fft_spectr, no_of_spectra_in_bunch, hanning_window): """ function converts waveform data in .wf32 format to spectra in .dat format : fname : name of .wf32 file with waveform data : no_of_points_for_fft : number of points for FFT to provide necessary time-frequency resolution : return : file_data_name - name of .dat file with result spectra """ # *** Data file header read *** [ df_filename, df_filesize, df_system_name, df_obs_place, df_description, clock_freq, df_time_utc, channel, receiver_mode, mode, n_avr, time_resolution, fmin, fmax, df, frequency, freq_points_num, data_block_size ] = FileHeaderReaderJDS(fname, 0, 0) freq_points_num = int(no_of_points_for_fft_spectr / 2) with open(fname, 'rb') as file: # *** Data file header read *** file_header = file.read(1024) # *** Creating a binary file with spectra data for long data storage *** file_data_name = fname[:-5] + '.dat' file_data = open(file_data_name, 'wb') file_data.write(file_header) file_data.seek(574) # FFT size place in header file_data.write(np.int32(no_of_points_for_fft_spectr).tobytes()) file_data.seek(624) # Lb place in header file_data.write(np.int32(0).tobytes()) file_data.seek(628) # Hb place in header file_data.write(np.int32(freq_points_num).tobytes()) file_data.seek(632) # Wb place in header file_data.write(np.int32(freq_points_num).tobytes()) file_data.seek(636) file_data.write(np.int32(1).tobytes()) # Seem to work OK file_data.close() del file_header # Calculation of number of blocks and number of spectra in the file no_of_bunches_per_file = int( (df_filesize - 1024) / ((no_of_spectra_in_bunch + 0.5) * no_of_points_for_fft_spectr * 4)) # Real time resolution of averaged spectra fine_clock_freq = int(clock_freq / 1000000.0) * 1000000.0 real_spectra_dt = float(no_of_points_for_fft_spectr / fine_clock_freq) real_spectra_df = float( (fine_clock_freq / 2) / (no_of_points_for_fft_spectr / 2)) print(' Number of spectra in bunch: ', no_of_spectra_in_bunch) print(' Sampling clock frequency: ', fine_clock_freq, ' Hz') print(' Number of bunches to read in file: ', no_of_bunches_per_file) print(' Time resolution of calculated spectra: ', round(real_spectra_dt * 1000, 3), ' ms') print(' Frequency resolution of calculated spectra: ', round(real_spectra_df / 1000, 3), ' kHz \n') # print('\n Reading data from file \n') file.seek(1024) # Jumping to 1024 byte from file beginning # *** Creating a new timeline TXT file for results *** new_tl_file_name = file_data_name.split('_Data_', 1)[0] + '_Timeline.txt' new_tl_file = open( new_tl_file_name, 'w') # Open and close to delete the file with the same name new_tl_file.close() # *** Reading timeline file *** old_tl_file_name = fname.split("_Data_", 1)[0] + '_Timeline.wtxt' old_tl_file = open(old_tl_file_name, 'r') new_tl_file = open( new_tl_file_name, 'w') # Open and close to delete the file with the same name # Making the variable for half length of the spectrum for convenience half_of_spectrum = int(no_of_points_for_fft_spectr / 2) # Making a small buffer vector to store the last half ot spectrum for the next loop step buffer = np.zeros(half_of_spectrum) bar = IncrementalBar(' Conversion from waveform to spectra: ', max=no_of_bunches_per_file - 1, suffix='%(percent)d%%') bar.start() for bunch in range(no_of_bunches_per_file - 1): # print('Bunch # ', bunch, ' of ', no_of_bunches_per_file - 1) # Read time from timeline file for the bunch time_scale_bunch = [] for line in range(no_of_spectra_in_bunch): tmp = str(old_tl_file.readline()) time_scale_bunch.append(tmp) # append the current value time_scale_bunch.append( tmp ) # append once more the same value for timing of fft with overlap # Saving time data to new file for i in range(len(time_scale_bunch)): new_tl_file.write((time_scale_bunch[i][:]) + '') # Reading and reshaping data of the bunch wf_data = np.fromfile(file, dtype='f4', count=no_of_spectra_in_bunch * no_of_points_for_fft_spectr) wf_data = np.concatenate((buffer, wf_data), axis=0) # Save new data from the end to the buffer buffer = wf_data[-half_of_spectrum:].copy() # Selecting the needed sequence of the data and reshaping to rectangular array wf_data_1 = np.reshape( wf_data[:-half_of_spectrum].copy(), [no_of_points_for_fft_spectr, no_of_spectra_in_bunch], order='F') wf_data_2 = np.reshape( wf_data[half_of_spectrum:].copy(), [no_of_points_for_fft_spectr, no_of_spectra_in_bunch], order='F') wf_data_1 = np.transpose(wf_data_1) wf_data_2 = np.transpose(wf_data_2) del wf_data # Merging 2 arrays into one rectangular array in the one by one order wf_data = np.zeros( (2 * no_of_spectra_in_bunch, no_of_points_for_fft_spectr)) wf_data[0::2, :] = wf_data_1[:, :] wf_data[1::2, :] = wf_data_2[:, :] del wf_data_1, wf_data_2 # Apply window to data for FFT if hanning_window: # window = np.hanning(no_of_points_for_fft_spectr) window = np.hamming(no_of_points_for_fft_spectr) wf_data[:] = wf_data[:] * window[:] del window # Preparing empty array for spectra spectra = np.zeros_like(wf_data, dtype=np.float64) # Calculation of spectra spectra[:] = np.power(np.abs(np.fft.fft(wf_data[:])), 2) # spectra[:] = np.abs(np.fft.fft(np.power(wf_data[:], 2))) # Does not work # spectra[:, i] = np.power(np.abs(np.fft.fft(wf_data[:, i])), 2) del wf_data # Storing only first (left) mirror part of spectra spectra = spectra[:, :int(no_of_points_for_fft_spectr / 2)] # spectra = spectra[: int(no_of_points_for_fft_spectr / 2), :] # At 33 MHz clock frequency the specter is upside down, to correct it we use flip up/down if int(clock_freq / 1000000) == 33: # spectra = np.flipud(spectra) spectra = np.fliplr(spectra) # Saving spectra data to dat file temp = spectra.copy(order='C') file_data = open(file_data_name, 'ab') file_data.write(np.float64(temp)) file_data.close() bar.next() bar.finish() file.close() # Close the data file return file_data_name
def pulsar_period_DM_compensated_pics(common_path, filename, pulsar_name, normalize_response, profile_pic_min, profile_pic_max, spectrum_pic_min, spectrum_pic_max, periods_per_fig, customDPI, colormap, save_strongest, threshold): current_time = time.strftime("%H:%M:%S") current_date = time.strftime("%d.%m.%Y") # Creating a folder where all pictures and results will be stored (if it doesn't exist) result_path = "RESULTS_pulsar_n_periods_pics_" + filename if not os.path.exists(result_path): os.makedirs(result_path) if save_strongest: best_result_path = result_path + '/Strongest_pulses' if not os.path.exists(best_result_path): os.makedirs(best_result_path) # Taking pulsar period from catalogue pulsar_ra, pulsar_dec, DM, p_bar = catalogue_pulsar(pulsar_name) # DAT file to be analyzed: filepath = common_path + filename # Timeline file to be analyzed: timeline_filepath = common_path + filename.split( '_Data_')[0] + '_Timeline.txt' # Opening DAT datafile file = open(filepath, 'rb') # Data file header read df_filesize = os.stat(filepath).st_size # Size of file df_filepath = file.read(32).decode('utf-8').rstrip( '\x00') # Initial data file name file.close() if df_filepath[-4:] == '.adr': [ df_filepath, df_filesize, df_system_name, df_obs_place, df_description, CLCfrq, df_creation_timeUTC, ReceiverMode, Mode, sumDifMode, NAvr, time_resolution, fmin, fmax, df, frequency, FFTsize, SLine, Width, BlockSize ] = FileHeaderReaderADR(filepath, 0, 0) freq_points_num = len(frequency) if df_filepath[-4:] == '.jds': # If data obtained from DSPZ receiver [ df_filepath, df_filesize, df_system_name, df_obs_place, df_description, CLCfrq, df_creation_timeUTC, SpInFile, ReceiverMode, Mode, Navr, time_resolution, fmin, fmax, df, frequency, freq_points_num, dataBlockSize ] = FileHeaderReaderJDS(filepath, 0, 1) # ************************************************************************************ # R E A D I N G D A T A * # ************************************************************************************ # Time line file reading timeline, dt_timeline = time_line_file_reader(timeline_filepath) # Calculation of the dimensions of arrays to read taking into account the pulsar period spectra_in_file = int( (df_filesize - 1024) / (8 * freq_points_num)) # int(df_filesize - 1024)/(2*4*freq_points_num) spectra_to_read = int( np.round((periods_per_fig * p_bar / time_resolution), 0)) num_of_blocks = int(np.floor(spectra_in_file / spectra_to_read)) print(' Pulsar period: ', p_bar, 's.') print(' Time resolution: ', time_resolution, 's.') print(' Number of spectra to read in', periods_per_fig, 'periods: ', spectra_to_read, ' ') print(' Number of spectra in file: ', spectra_in_file, ' ') print(' Number of', periods_per_fig, 'periods blocks in file: ', num_of_blocks, '\n') # Data reading and making figures print('\n\n *** Data reading and making figures *** \n\n') data_file = open(filepath, 'rb') data_file.seek( 1024, os.SEEK_SET ) # Jumping to 1024+number of spectra to skip byte from file beginning bar = IncrementalBar(' Making pictures of n periods: ', max=num_of_blocks, suffix='%(percent)d%%') bar.start() for block in range(num_of_blocks + 1): # Main loop by blocks of data # bar.next() # current_time = time.strftime("%H:%M:%S") # print(' * Data block # ', block + 1, ' of ', num_of_blocks + 1, ' started at: ', current_time) # Reading the last block which is less then 3 periods if block == num_of_blocks: spectra_to_read = spectra_in_file - num_of_blocks * spectra_to_read # Reading and preparing block of data (3 periods) data = np.fromfile(data_file, dtype=np.float64, count=spectra_to_read * len(frequency)) data = np.reshape(data, [len(frequency), spectra_to_read], order='F') data = 10 * np.log10(data) if normalize_response > 0: Normalization_dB(data.transpose(), len(frequency), spectra_to_read) # Preparing single averaged data profile for figure profile = data.mean(axis=0)[:] profile = profile - np.mean(profile) data = data - np.mean(data) # Time line fig_time_scale = timeline[block * spectra_to_read:(block + 1) * spectra_to_read] # Making result picture fig = plt.figure(figsize=(9.2, 4.5)) rc('font', size=5, weight='bold') ax1 = fig.add_subplot(211) ax1.plot(profile, color=u'#1f77b4', linestyle='-', alpha=1.0, linewidth='0.60', label='3 pulses time profile') ax1.legend(loc='upper right', fontsize=5) ax1.grid(b=True, which='both', color='silver', linewidth='0.50', linestyle='-') ax1.axis([0, len(profile), profile_pic_min, profile_pic_max]) ax1.set_ylabel('Amplitude, AU', fontsize=6, fontweight='bold') ax1.set_title('File: ' + filename + ' Description: ' + df_description + ' Resolution: ' + str(np.round(df / 1000, 3)) + ' kHz and ' + str(np.round(time_resolution * 1000, 3)) + ' ms.', fontsize=5, fontweight='bold') ax1.tick_params(axis='x', which='both', bottom=False, top=False, labelbottom=False) ax2 = fig.add_subplot(212) ax2.imshow(np.flipud(data), aspect='auto', cmap=colormap, vmin=spectrum_pic_min, vmax=spectrum_pic_max, extent=[0, len(profile), frequency[0], frequency[-1]]) ax2.set_xlabel('Time UTC (at the lowest frequency), HH:MM:SS.ms', fontsize=6, fontweight='bold') ax2.set_ylabel('Frequency, MHz', fontsize=6, fontweight='bold') text = ax2.get_xticks().tolist() for i in range(len(text) - 1): k = int(text[i]) text[i] = fig_time_scale[k][11:23] ax2.set_xticklabels(text, fontsize=5, fontweight='bold') fig.subplots_adjust(hspace=0.05, top=0.91) fig.suptitle('Single pulses of ' + pulsar_name + ' (DM: ' + str(DM) + r' $\mathrm{pc \cdot cm^{-3}}$' + ', Period: ' + str(p_bar) + ' s.), fig. ' + str(block + 1) + ' of ' + str(num_of_blocks + 1), fontsize=7, fontweight='bold') fig.text(0.80, 0.04, 'Processed ' + current_date + ' at ' + current_time, fontsize=3, transform=plt.gcf().transFigure) fig.text(0.09, 0.04, 'Software version: ' + Software_version + ', [email protected], IRA NASU', fontsize=3, transform=plt.gcf().transFigure) pylab.savefig(result_path + '/' + filename + ' fig. ' + str(block + 1) + ' - Combined picture.png', bbox_inches='tight', dpi=customDPI) # If the profile has points above threshold save picture also into separate folder if save_strongest and np.max(profile) > threshold: pylab.savefig(best_result_path + '/' + filename + ' fig. ' + str(block + 1) + ' - Combined picture.png', bbox_inches='tight', dpi=customDPI) plt.close('all') bar.next() bar.finish() data_file.close()
def laba3(db_file_name, count_range, schema, schema_data): results = { 'linear': [], 'binary': [], 'binary+sort': [], 'multimap': [], 'hashtable_map_good': [], 'hashtable_map_bad': [], 'bad_collisions': [], 'good_collisions': [] } key = 'fio' max_count_iterations = 2 iterations = len(count_range) bar = IncrementalBar('Countdown', max=iterations) bar.start() for count in count_range: bar.next() print('\n') for count_iterations in range(max_count_iterations): generate(db_file_name, count, schema, schema_data) fp_map = defaultdict(list) fp_list = load_fp_from_file(db_file_name) query_obj = random.choice(fp_list) query = getattr(query_obj, key) print('check lin') linear = check_time(linear_search)(fp_list, key, query) print('check sort+bin') sort_and_bin_search = check_time(sort_and_binary_seach)(fp_list, key, query) print('check bin') bin_search = check_time(binary_search)(fp_list, key, query) print('check multimap') map_search = check_time(fp_map.__getitem__)(query) print('check hashtable good') fp_custom_map_good = HashTable() for el in fp_list: el.set_hash_type('good') fp_map[getattr(el, key)].append(el) fp_custom_map_good.add(el) query_obj.set_hash_type('good') custom_map_good_search = check_time(fp_custom_map_good.get)( Hashes.good_hash(query)) print('check hashtable bad') fp_custom_map_bad = HashTable() for el in fp_list: el.set_hash_type('bad') fp_custom_map_bad.add(el) query_obj.set_hash_type('bad') custom_map_bad_search = check_time(fp_custom_map_bad.get)( Hashes.bad_hash(query)) results['linear'].append((count, linear)) results['binary'].append((count, bin_search)) results['binary+sort'].append((count, sort_and_bin_search)) results['multimap'].append((count, map_search)) results['hashtable_map_good'].append( (count, custom_map_good_search)) results['hashtable_map_bad'].append((count, custom_map_bad_search)) results['bad_collisions'].append( (count, fp_custom_map_bad.collision_count)) results['good_collisions'].append( (count, fp_custom_map_good.collision_count)) plot_graph(results, count_range, max_count_iterations) print('bad_collisions: ', results['bad_collisions']) print('good_collisions: ', results['good_collisions']) bar.finish() return results
def convert_jds_wf_to_wf32(source_directory, result_directory, no_of_bunches_per_file): """ function converts jds waveform data to wf32 waveform data for further processing (coherent dedispersion) and saves txt files with time data Input parameters: source_directory - directory where initial jds waveform data are stored result_directory - directory where new wf32 files will be stored no_of_bunches_per_file - number of data bunches per file to process (depends on RAM volume on the PC) Output parameters: result_wf32_files - list of results files """ file_list = find_and_check_files_in_current_folder(source_directory, '.jds') # To print in console the header of first file print('\n First file header parameters: \n') # *** Data file header read *** [df_filename, df_filesize, df_system_name, df_obs_place, df_description, clock_freq, df_creation_timeUTC, channel, receiver_mode, Mode, Navr, time_res, fmin, fmax, df, frequency, freq_points_num, data_block_size] = FileHeaderReaderJDS(source_directory + file_list[0], 0, 1) if Mode > 0: sys.exit(' ERROR!!! Data recorded in wrong mode! Waveform mode needed.\n\n Program stopped!') result_wf32_files = [] # Main loop by files start for file_no in range(len(file_list)): # loop by files fname = source_directory + file_list[file_no] # Create long data files and copy first data file header to them if file_no == 0: with open(fname, 'rb') as file: # *** Data file header read *** file_header = file.read(1024) # *** Creating a name for long timeline TXT file *** tl_file_name = df_filename + '_Timeline.wtxt' tl_file = open(tl_file_name, 'w') # Open and close to delete the file with the same name tl_file.close() # *** Creating a binary file with data for long data storage *** file_data_A_name = df_filename + '_Data_chA.wf32' result_wf32_files.append(file_data_A_name) file_data_A = open(file_data_A_name, 'wb') file_data_A.write(file_header) file_data_A.close() if channel == 2: file_data_B_name = df_filename + '_Data_chB.wf32' result_wf32_files.append(file_data_B_name) file_data_B = open(file_data_B_name, 'wb') file_data_B.write(file_header) file_data_B.close() del file_header # Calculation of number of blocks and number of spectra in the file if channel == 0 or channel == 1: # Single channel mode no_of_spectra_in_bunch = int((df_filesize - 1024) / (no_of_bunches_per_file * 2 * data_block_size)) else: # Two channels mode no_of_spectra_in_bunch = int((df_filesize - 1024) / (no_of_bunches_per_file * 4 * data_block_size)) no_of_blocks_in_file = (df_filesize - 1024) / data_block_size if file_no == 0: print(' Number of blocks in file: ', no_of_blocks_in_file) print(' Number of bunches to read in file: ', no_of_bunches_per_file) print('\n *** Reading data from file *** \n') # ******************************************************************************* # R E A D I N G D A T A * # ******************************************************************************* with open(fname, 'rb') as file: file.seek(1024) # Jumping to 1024 byte from file beginning # !!! Fake timing. Real timing to be done!!! TimeFigureScaleFig = np.linspace(0, no_of_bunches_per_file, no_of_bunches_per_file + 1) for i in range(no_of_bunches_per_file): TimeFigureScaleFig[i] = str(TimeFigureScaleFig[i]) time_scale_bunch = [] bar = IncrementalBar(' File ' + str(file_no + 1) + ' of ' + str(len(file_list)) + ' reading: ', max=no_of_bunches_per_file, suffix='%(percent)d%%') bar.start() for bunch in range(no_of_bunches_per_file): # bar.next() # Reading and reshaping all data with time data if channel == 0 or channel == 1: # Single channel mode wf_data = np.fromfile(file, dtype='i2', count=no_of_spectra_in_bunch * data_block_size) wf_data = np.reshape(wf_data, [data_block_size, no_of_spectra_in_bunch], order='F') if channel == 2: # Two channels mode wf_data = np.fromfile(file, dtype='i2', count=2 * no_of_spectra_in_bunch * data_block_size) wf_data = np.reshape(wf_data, [data_block_size, 2 * no_of_spectra_in_bunch], order='F') # Timing timeline_block_str = jds_waveform_time(wf_data, clock_freq, data_block_size) if channel == 2: # Two channels mode # Cut the timeline of second channel timeline_block_str = timeline_block_str[0:int(len(timeline_block_str) / 2)] for i in range(len(timeline_block_str)): time_scale_bunch.append(df_creation_timeUTC[0:10] + ' ' + timeline_block_str[i]) # [0:12] # Deleting the time blocks from waveform data real_data_block_size = data_block_size - 4 wf_data = wf_data[0: real_data_block_size, :] # Separation data into channels if channel == 0 or channel == 1: # Single channel mode wf_data_chA = np.reshape(wf_data, [real_data_block_size * no_of_spectra_in_bunch, 1], order='F') del wf_data # Deleting unnecessary array name just in case if channel == 2: # Two channels mode # Separating the data into two channels wf_data = np.reshape(wf_data, [2 * real_data_block_size * no_of_spectra_in_bunch, 1], order='F') wf_data_chA = wf_data[0: (2 * real_data_block_size * no_of_spectra_in_bunch): 2] # A wf_data_chB = wf_data[1: (2 * real_data_block_size * no_of_spectra_in_bunch): 2] # B del wf_data # Saving WF data to dat file file_data_A = open(file_data_A_name, 'ab') file_data_A.write(np.float32(wf_data_chA).transpose().copy(order='C')) file_data_A.close() if channel == 2: file_data_B = open(file_data_B_name, 'ab') file_data_B.write(np.float32(wf_data_chB).transpose().copy(order='C')) file_data_B.close() # Saving time data to ling timeline file with open(tl_file_name, 'a') as tl_file: for i in range(no_of_spectra_in_bunch): tl_file.write((str(time_scale_bunch[i][:])) + ' \n') # str bar.next() bar.finish() file.close() # Close the data file del file_data_A if channel == 2: del file_data_B return result_wf32_files
def download_chapter_images(app): app.progress = 0 # download or generate cover app.book_cover = download_cover(app) if not app.book_cover: app.book_cover = generate_cover(app) # end if if not app.book_cover: logger.warn('No cover image') # end if image_count = 0 futures_to_check = {} for chapter in app.chapters: if not chapter.get('body'): continue # end if soup = app.crawler.make_soup(chapter['body']) image_output_path = os.path.join(app.output_path, 'images') for img in soup.select('img'): full_url = app.crawler.absolute_url(img['src'], page_url=chapter['url']) future = app.crawler.executor.submit(download_image, app, full_url, image_output_path) futures_to_check.setdefault(chapter['id'], []) futures_to_check[chapter['id']].append(future) image_count += 1 # end for # end for if not futures_to_check: return # end if bar = IncrementalBar('Downloading images ', max=image_count) if os.getenv('debug_mode') == 'yes': bar.next = lambda: None # Hide in debug mode bar.finish() else: bar.start() # end if for chapter in app.chapters: if chapter['id'] not in futures_to_check: bar.next() continue # end if images = {} for future in futures_to_check[chapter['id']]: url, filename = future.result() bar.next() if filename: images[url] = filename # end if # end for soup = app.crawler.make_soup('<main>' + chapter['body'] + '</main>') for img in soup.select('img'): if img['src'] in images: filename = images[img['src']] img['src'] = 'images/%s' % filename img['style'] = 'float: left; margin: 15px; width: 100%;' # end for chapter['body'] = str(soup.select_one('main')) # end for bar.finish() print('Processed %d images' % image_count)
def normalize_dat_file(directory, filename, no_of_spectra_in_bunch, median_filter_window, show_aver_spectra): """ function calculates the average spectrum in DAT file and normalizes all spectra in file to average spectra Input parameters: directory - name of directory with initial dat file filename - name of initial dat file no_of_spectra_in_bunch - number of spectra in bunch to read median_filter_window - window of median filter to process the average spectrum show_aver_spectra - boolean variable which indicates if the picture of average spectrum to be shown and the script paused till the picture window is closed Output parameters: output_file_name - name of result normalized .dat file """ print( '\n Preparations and calculation of the average spectrum to normalize... \n' ) output_file_name = directory + 'Norm_' + filename filename = directory + filename # Opening DAT datafile file = open(filename, 'rb') # *** Data file header read *** df_filesize = os.stat(filename).st_size # Size of file df_filename = file.read(32).decode('utf-8').rstrip( '\x00') # Initial data file name file.close() if df_filename[-4:] == '.adr': [ df_filename, df_filesize, df_system_name, df_obs_place, df_description, CLCfrq, df_creation_timeUTC, ReceiverMode, Mode, sumDifMode, NAvr, TimeRes, fmin, fmax, df, frequency, FFTsize, SLine, Width, BlockSize ] = FileHeaderReaderADR(filename, 0, 0) if df_filename[-4:] == '.jds': # If data obtained from DSPZ receiver [ df_filename, df_filesize, df_system_name, df_obs_place, df_description, CLCfrq, df_creation_timeUTC, SpInFile, ReceiverMode, Mode, Navr, TimeRes, fmin, fmax, df, frequency, FreqPointsNum, dataBlockSize ] = FileHeaderReaderJDS(filename, 0, 0) # Calculation of the dimensions of arrays to read nx = len(frequency) # the first dimension of the array ny = int(( (df_filesize - 1024) / (nx * 8))) # the second dimension of the array: file size - 1024 bytes # Number of data blocks to read from file num_of_blocks = int(ny // no_of_spectra_in_bunch) # Read data from file by blocks and average it file = open(filename, 'rb') file.seek(1024) average_array = np.empty((nx, 0), float) for block in range(num_of_blocks): if block == (num_of_blocks - 1): spectra_num_in_bunch = ny - (num_of_blocks - 1) * no_of_spectra_in_bunch else: spectra_num_in_bunch = no_of_spectra_in_bunch data = np.fromfile(file, dtype=np.float64, count=nx * spectra_num_in_bunch) data = np.reshape(data, [nx, spectra_num_in_bunch], order='F') tmp = np.empty((nx, 1), float) # tmp[:, 0] = data.mean(axis=1)[:] tmp[:, 0] = data.min(axis=1)[:] average_array = np.append(average_array, tmp, axis=1) # # Average average spectra of all data blocks average_profile = average_array.mean(axis=1) init_average_profile = average_profile.copy() # # Make a figure of average spectrum (profile) # fig = plt.figure(figsize=(9, 5)) # ax1 = fig.add_subplot(111) # ax1.plot(10 * np.log10(average_profile), linestyle='-', linewidth='1.00', label='Average spectra') # ax1.legend(loc='upper right', fontsize=6) # ax1.grid(b=True, which='both', color='silver', linestyle='-') # ax1.set_xlabel('Frequency points, num.', fontsize=6, fontweight='bold') # ax1.set_ylabel('Intensity, dB', fontsize=6, fontweight='bold') # pylab.savefig('Averaged_spectra_'+filename[:-4]+'_before_filtering.png', bbox_inches='tight', dpi=160) # plt.close('all') # Apply median filter to average profile average_profile = median_filter(average_profile, median_filter_window) med_average_profile = average_profile.copy() average_profile = average_filter(average_profile, median_filter_window + 20) # Make a figure of filtered average spectrum (profile) fig = plt.figure(figsize=(12, 8)) ax1 = fig.add_subplot(111) ax1.plot(10 * np.log10(init_average_profile), linestyle='-', linewidth='1.50', label='Initial spectra', color='C0', alpha=0.6) ax1.plot(10 * np.log10(med_average_profile), linestyle='-', linewidth='1.25', label='Median spectra', color='C1', alpha=0.8) ax1.plot(10 * np.log10(average_profile), linestyle='-', linewidth='1.00', label='Median averaged spectra', color='C3') ax1.legend(loc='upper right', fontsize=6) ax1.grid(b=True, which='both', color='silver', linestyle='-') ax1.set_xlabel('Frequency points, num.', fontsize=6, fontweight='bold') ax1.set_ylabel('Intensity, dB', fontsize=6, fontweight='bold') pylab.savefig('Averaged_spectra_' + filename[:-4] + '_after_filtering.png', bbox_inches='tight', dpi=160) if show_aver_spectra: print('\n Close the figure window to continue processing!!!\n') plt.show() plt.close('all') del init_average_profile, med_average_profile # Normalization print(' Spectra normalization... \n') file.seek(0) file_header = file.read(1024) normalized_file = open(output_file_name, 'wb') normalized_file.write(file_header) del file_header bar = IncrementalBar(' Normalizing of the DAT file: ', max=num_of_blocks, suffix='%(percent)d%%') bar.start() for block in range(num_of_blocks): if block == (num_of_blocks - 1): spectra_num_in_bunch = ny - (num_of_blocks - 1) * no_of_spectra_in_bunch else: spectra_num_in_bunch = no_of_spectra_in_bunch data = np.fromfile(file, dtype=np.float64, count=nx * spectra_num_in_bunch) data = np.reshape(data, [nx, spectra_num_in_bunch], order='F') for j in range(spectra_num_in_bunch): data[:, j] = data[:, j] / average_profile[:] temp = data.transpose().copy(order='C') normalized_file.write(np.float64(temp)) bar.next() file.close() normalized_file.close() bar.finish() # *** Creating a new timeline TXT file for results *** new_tl_file_name = output_file_name.split('_Data_', 1)[0] + '_Timeline.txt' new_tl_file = open( new_tl_file_name, 'w') # Open and close to delete the file with the same name new_tl_file.close() # *** Reading timeline file *** old_tl_file_name = filename.split('_Data_', 1)[0] + '_Timeline.txt' old_tl_file = open(old_tl_file_name, 'r') new_tl_file = open(new_tl_file_name, 'w') # Read time from timeline file time_scale_bunch = old_tl_file.readlines() # Saving time data to new file for j in range(len(time_scale_bunch)): new_tl_file.write((time_scale_bunch[j][:]) + '') old_tl_file.close() new_tl_file.close() return output_file_name