def threaded_function(): global memories global done start = timeit.timeit() while not done: #cpus.append(psutil.cpu_percent()) memories.append(psutil.virtual_memory()._asdict()) end = timeit.timeit() if(end - start>5): time.sleep(5); start=time.timeit() print("done")
def ap(): _time_st[s] = timeit() return ()
def Esol_series_bessel(M, e, N=2): E = M for n in xrange(1, N + 1): E += (2. / n) * jn(n, e) * sin(n * M) return E # # ComparaciĆ³n de Soluciones # In[256]: e = 0.6 Ms = linspace(0, 2 * pi, 100) t1 = timeit() # Numeric solution Ens = array([Esol_numeric(M, e) for M in Ms]) t2 = timeit() print "Numeric:", (t2 - t1) * 1E6 t1 = t2 # Analytic solution Eas = array([Esol_analytic(M, e) for M in Ms]) t2 = timeit() print "Analytic:", (t2 - t1) * 1E6 t1 = t2 # Fourier series solution Efs = array([Esol_series_fourier(M, e) for M in Ms])
def Esol_series_bessel(M,e,N=2): E=M for n in xrange(1,N+1): E+=(2./n)*jn(n,e)*sin(n*M) return E # # ComparaciĆ³n de Soluciones # In[256]: e=0.6 Ms=linspace(0,2*pi,100) t1=timeit() # Numeric solution Ens=array([Esol_numeric(M,e) for M in Ms]) t2=timeit();print "Numeric:",(t2-t1)*1E6;t1=t2 # Analytic solution Eas=array([Esol_analytic(M,e) for M in Ms]) t2=timeit();print "Analytic:",(t2-t1)*1E6;t1=t2 # Fourier series solution Efs=array([Esol_series_fourier(M,e) for M in Ms]) t2=timeit();print "Fourier:",(t2-t1)*1E6;t1=t2 # Bessel series solution Ebs=array([Esol_series_bessel(M,e,N=7) for M in Ms])
def wrapper(*args, **kwargs): start = timeit() rv = func(*args, **kwargs) end = timeit() print(start - end) return rv
def convert(data_dir, output_filepath, append_to_existing=False): """ Converts TransXchange formatted schedule data into GTFS feed. data_dir : str Data directory containing one or multiple TransXchange .xml files. output_filepath : str Full filepath to the output GTFS zip-file, e.g. '/home/myuser/data/my_gtfs.zip' append_to_existing : bool (default is False) Flag for appending to existing gtfs-database. This might be useful if you have TransXchange .xml files distributed into multiple directories (e.g. separate files for train data, tube data and bus data) and you want to merge all those datasets into a single GTFS feed. """ # Total start tot_start_t = timeit() # Filepath for temporary gtfs db target_dir = os.path.dirname(output_filepath) gtfs_db = os.path.join(target_dir, "gtfs.db") # If append to database is false remove previous gtfs-database if it exists if append_to_existing == False: if os.path.exists(gtfs_db): os.remove(gtfs_db) # NAPTAN stops naptan_stops_fp = get_path("naptan_stops") # Retrieve all TransXChange files files = glob.glob(os.path.join(data_dir, "*.xml")) # Iterate over files print("Populating database ..") # Limit the processed files by file size (in MB) # Files with lower filesize than below will be processed file_size_limit = 1000 # Create workers workers = create_workers(input_files=files, file_size_limit=file_size_limit, stops_fp=naptan_stops_fp, gtfs_db=gtfs_db) # Create Pool pool = multiprocessing.Pool() # Generate GTFS info to the database in parallel pool.map(process_files, workers) # Print information about the total time tot_end_t = timeit() tot_duration = (tot_end_t - tot_start_t) / 60 print("===========================================================") print("It took %s minutes in total." % round(tot_duration, 1)) # Generate output dictionary gtfs_data = generate_gtfs_export(gtfs_db) # Export to disk save_to_gtfs_zip(output_zip_fp=output_filepath, gtfs_data=gtfs_data)
def process_files(parallel): # Get files from input instance files = parallel.input_files file_size_limit = parallel.file_size_limit naptan_stops_fp = parallel.stops_fp gtfs_db = parallel.gtfs_db for idx, fp in enumerate(files): # Filesize size = round((os.path.getsize(fp) / 1000000), 1) if file_size_limit < size: continue print( "=================================================================" ) print("[%s / %s] Processing TransXChange file: %s" % (idx, len(files), os.path.basename(fp))) print("Size: %s MB" % size) # Log start time start_t = timeit() data = untangle.parse(fp) # Parse stops stop_data = get_stops(data, naptan_stops_fp=naptan_stops_fp) # Parse agency agency = get_agency(data) # Parse GTFS info containing data about trips, calendar, stop_times and calendar_dates gtfs_info = get_gtfs_info(data) # Parse stop_times stop_times = get_stop_times(gtfs_info) # Parse trips trips = get_trips(gtfs_info) # Parse calendar calendar = get_calendar(gtfs_info) # Parse calendar_dates calendar_dates = get_calendar_dates(gtfs_info) # Parse routes routes = get_routes(gtfs_info=gtfs_info, data=data) # Initialize database connection conn = sqlite3.connect(gtfs_db) # Only export data into db if there exists valid stop_times data if len(stop_times) > 0: stop_times.to_sql(name='stop_times', con=conn, index=False, if_exists='append') stop_data.to_sql(name='stops', con=conn, index=False, if_exists='append') routes.to_sql(name='routes', con=conn, index=False, if_exists='append') agency.to_sql(name='agency', con=conn, index=False, if_exists='append') trips.to_sql(name='trips', con=conn, index=False, if_exists='append') calendar.to_sql(name='calendar', con=conn, index=False, if_exists='append') if calendar_dates is not None: calendar_dates.to_sql(name='calendar_dates', con=conn, index=False, if_exists='append') else: print( "UserWarning: File %s did not contain valid stop_sequence data, skipping." % (os.path.basename(fp))) # Close connection conn.close() # Log end time and parse duration end_t = timeit() duration = (end_t - start_t) / 60 print("It took %s minutes." % round(duration, 1))
def process_files(parallel): # Get files from input instance files = parallel.input_files file_size_limit = parallel.file_size_limit gtfs_db = parallel.gtfs_db for idx, path in enumerate(files): # If type is string, it is a direct filepath to XML if isinstance(path, str): data, file_size, xml_name = read_unpacked_xml(path) # If the type is dictionary contents are in a zip elif isinstance(path, dict): # If the type of value is a string the file can be read directly # from the given Zipfile path, with following structure: # {"transxchange_name.xml" : "/home/data/myzipfile.zip"} if isinstance(list(path.values())[0], str): data, file_size, xml_name = read_xml_inside_zip(path) # If the type of value is a dictionary the xml-file # is in a ZipFile which is inside another ZipFile. # In such cases, the path stucture is: # {"outermost_zipfile_path.zip": {"inner_zipfile.zip": "transxchange.xml"}} elif isinstance(list(path.values())[0], dict): data, file_size, xml_name = read_xml_inside_nested_zip(path) else: raise ValueError( "Something is wrong with the input xml-file paths.") else: raise ValueError( "Something is wrong with the input xml-file paths.") # Filesize size = round((file_size / 1000000), 1) if file_size_limit < size: continue print( "=================================================================" ) print("[%s / %s] Processing TransXChange file: %s" % (idx, len(files), xml_name)) print("Size: %s MB" % size) # Log start time start_t = timeit() # Parse stops stop_data = get_stops(data) if stop_data is None: print("Did not found any valid stops. Skipping..") continue # Parse agency agency = get_agency(data) # Parse GTFS info containing data about trips, calendar, stop_times and calendar_dates gtfs_info = get_gtfs_info(data) # Parse stop_times stop_times = get_stop_times(gtfs_info) # Parse trips trips = get_trips(gtfs_info) # Parse calendar calendar = get_calendar(gtfs_info) # Parse calendar_dates calendar_dates = get_calendar_dates(gtfs_info) # Parse routes routes = get_routes(gtfs_info=gtfs_info, data=data) # Initialize database connection conn = sqlite3.connect(gtfs_db) # Only export data into db if there exists valid stop_times data if len(stop_times) > 0: stop_times.to_sql(name='stop_times', con=conn, index=False, if_exists='append') stop_data.to_sql(name='stops', con=conn, index=False, if_exists='append') routes.to_sql(name='routes', con=conn, index=False, if_exists='append') agency.to_sql(name='agency', con=conn, index=False, if_exists='append') trips.to_sql(name='trips', con=conn, index=False, if_exists='append') calendar.to_sql(name='calendar', con=conn, index=False, if_exists='append') if calendar_dates is not None: calendar_dates.to_sql(name='calendar_dates', con=conn, index=False, if_exists='append') else: print( "UserWarning: File %s did not contain valid stop_sequence data, skipping." % (xml_name)) # Close connection conn.close() # Log end time and parse duration end_t = timeit() duration = (end_t - start_t) / 60 print("It took %s minutes." % round(duration, 1))
def convert(input_filepath, output_filepath, append_to_existing=False, worker_cnt=None, file_size_limit=2000): """ Converts TransXchange formatted schedule data into GTFS feed. input_filepath : str File path to data directory or a ZipFile containing one or multiple TransXchange .xml files. Also nested ZipFiles are supported (i.e. a ZipFile with ZipFile(s) containing .xml files.) output_filepath : str Full filepath to the output GTFS zip-file, e.g. '/home/myuser/data/my_gtfs.zip' append_to_existing : bool (default is False) Flag for appending to existing gtfs-database. This might be useful if you have TransXchange .xml files distributed into multiple directories (e.g. separate files for train data, tube data and bus data) and you want to merge all those datasets into a single GTFS feed. worker_cnt : int Number of workers to distribute the conversion process. By default the number of CPUs is used. file_size_limit : int File size limit (in megabytes) can be used to skip larger-than-memory XML-files (should not happen). """ # Total start tot_start_t = timeit() # Filepath for temporary gtfs db target_dir = os.path.dirname(output_filepath) gtfs_db = os.path.join(target_dir, "gtfs.db") # If append to database is false remove previous gtfs-database if it exists if append_to_existing == False: if os.path.exists(gtfs_db): os.remove(gtfs_db) # Retrieve all TransXChange files files = get_xml_paths(input_filepath) # Iterate over files print("Populating database ..") # Create workers workers = create_workers(input_files=files, worker_cnt=worker_cnt, file_size_limit=file_size_limit, gtfs_db=gtfs_db) # Create Pool pool = multiprocessing.Pool() # Generate GTFS info to the database in parallel pool.map(process_files, workers) # Print information about the total time tot_end_t = timeit() tot_duration = (tot_end_t - tot_start_t) / 60 print("===========================================================") print("It took %s minutes in total." % round(tot_duration, 1)) # Generate output dictionary gtfs_data = generate_gtfs_export(gtfs_db) # Export to disk save_to_gtfs_zip(output_zip_fp=output_filepath, gtfs_data=gtfs_data)
def calibrate(args, jones, alphas): # simple calibration to test if simulation went as expected. # Note do not run on large data set # load data ms = table(args.ms) time = ms.getcol('TIME') _, tbin_idx, tbin_counts = chunkify_rows(time, args.utimes_per_chunk) n_time = tbin_idx.size ant1 = ms.getcol('ANTENNA1') ant2 = ms.getcol('ANTENNA2') n_ant = np.maximum(ant1.max(), ant2.max()) + 1 uvw = ms.getcol('UVW').astype(np.float64) data = ms.getcol(args.out_col) # this is where we put the data # we know it is pure Stokes I so we can solve using diagonals only data = data[:, :, (0, 3)].astype(np.complex128) n_row, n_freq, n_corr = data.shape flag = ms.getcol('FLAG') flag = flag[:, :, (0, 3)] # get phase dir radec0 = table(args.ms + '::FIELD').getcol('PHASE_DIR').squeeze().astype( np.float64) # get freqs freq = table(args.ms + '::SPECTRAL_WINDOW').getcol('CHAN_FREQ')[0].astype( np.float64) assert freq.size == n_freq # now get the model # get source coordinates from lsm lsm = Tigger.load(args.sky_model) radec = [] stokes = [] spi = [] ref_freqs = [] for source in lsm.sources: radec.append([source.pos.ra, source.pos.dec]) stokes.append([source.flux.I]) tmp_spec = source.spectrum spi.append([tmp_spec.spi if tmp_spec is not None else 0.0]) ref_freqs.append([tmp_spec.freq0 if tmp_spec is not None else 1.0]) n_dir = len(stokes) radec = np.asarray(radec) lm = radec_to_lm(radec, radec0) # get model visibilities model = np.zeros((n_row, n_freq, n_dir, 2), dtype=np.complex) stokes = np.asarray(stokes) ref_freqs = np.asarray(ref_freqs) spi = np.asarray(spi) for d in range(n_dir): Stokes_I = stokes[d] * (freq / ref_freqs[d])**spi[d] model[:, :, d, 0:1] = im_to_vis(Stokes_I[None, :, None], uvw, lm[d:d + 1], freq) model[:, :, d, 1] = model[:, :, d, 0] # set weights to unity weight = np.ones_like(data, dtype=np.float64) # initialise gains jones0 = np.ones((n_time, n_ant, n_freq, n_dir, n_corr), dtype=np.complex128) # calibrate ti = timeit() jones_hat, jhj, jhr, k = gauss_newton(tbin_idx, tbin_counts, ant1, ant2, jones0, data, flag, model, weight, tol=1e-5, maxiter=100) print("%i iterations took %fs" % (k, timeit() - ti)) # verify result for p in range(2): for q in range(p): diff_true = np.angle(jones[:, p] * jones[:, q].conj()) diff_hat = np.angle(jones_hat[:, p] * jones_hat[:, q].conj()) try: assert_array_almost_equal(diff_true, diff_hat, decimal=2) except Exception as e: print(e)