def time_cut( obs, flare=False, length_minutes=25, flare_margin_minutes=1 ): raster = obs.raster("Mg II k") ts = np.array( raster.get_timestamps( raster_pos=0 ) ) if flare: # find end of preflare in terms of raster_pos=0 raster that happens _before_ the flare (taken care of in find_preflare_end) stop = find_preflare_end( obs, margin_minutes=flare_margin_minutes ) # find closest raster_pos=0 raster to suggested start time start_time = ts[stop] - length_minutes*60 start = np.argmin( np.abs( ts - start_time ) ) else: # set start to first raster sweep start = 0 # find closest raster_pos=0 raster to start time at first raster sweep start_time = ts[0] stop = np.argmin( np.abs( ts - (start_time + length_minutes*60) ) ) start_date = from_Tformat( raster.get_raster_pos_headers( raster_pos=0 )[start]['DATE_OBS'] ) stop_date = from_Tformat( raster.get_raster_pos_headers( raster_pos=raster.n_raster_pos-1 )[stop-1]['DATE_OBS'] ) delta_t = stop_date - start_date return [start, stop, np.round( delta_t.seconds/60, 1 )]
def find_sji_inds( obs ): ts = np.array( obs.sji[0].get_timestamps() ) start_date = obs.raster("Mg II k").time_specific_headers[0]['DATE_OBS'] stop_date = obs.raster("Mg II k").time_specific_headers[-1]['DATE_OBS'] i_start_sji = np.argmin( np.abs( ts - to_epoch( from_Tformat( start_date ) ) ) ) i_stop_sji = np.argmin( np.abs( ts - to_epoch( from_Tformat( stop_date ) ) ) ) return i_start_sji, i_stop_sji
def animate( obs ): ts = np.array( obs.sji[0].get_timestamps() ) start_date = obs.raster("Mg II k").time_specific_headers[0]['DATE_OBS'] stop_date = obs.raster("Mg II k").time_specific_headers[-1]['DATE_OBS'] i_start_sji = np.argmin( np.abs( ts - to_epoch( from_Tformat( start_date ) ) ) ) i_stop_sji = np.argmin( np.abs( ts - to_epoch( from_Tformat( stop_date ) ) ) ) return obs.sji[0].animate( index_start=i_start_sji, index_stop=i_stop_sji, cutoff_percentile=99.0 )
def plot_goes_flux( obs, i_start, i_stop ): raster = obs.raster("Mg II k") start_date = obs.start_date end_date = obs.end_date th = raster.get_raster_pos_headers( raster_pos=0 ) th_n = raster.get_raster_pos_headers( raster_pos=raster.n_raster_pos-1 ) start_cut_date = from_Tformat( th[i_start]['DATE_OBS'] ) stop_cut_date = from_Tformat( th_n[i_stop]['DATE_OBS'] ) # put time cut into green area ax = obs.goes.xrs.data.plot( y='B_FLUX', logy=True, label="GOES X-ray Flux", figsize=(24,5), lw=2 ) ax.axvspan( start_cut_date, stop_cut_date, alpha=0.2, color='green' ) ax.axvline( x=start_cut_date, color='green', linestyle='--', linewidth=3.0 ) ax.axvline( x=stop_cut_date, color='green', linestyle='--', linewidth=3.0 ) ax.set_xlim([start_date, end_date]) plt.text( start_cut_date, 1e-1, i_start, fontsize=14, color="green", ha="center" ) plt.text( stop_cut_date, 1e-1, i_stop+1, fontsize=14, color="green", ha="center" ) ax.axhline( y=1e-4, color='black', linestyle='--', linewidth=1.0 ) ax.axhline( y=1e-5, color='black', linestyle='--', linewidth=1.0 ) ax.axhline( y=1e-6, color='black', linestyle='--', linewidth=1.0 ) ax.axhline( y=1e-7, color='black', linestyle='--', linewidth=1.0 ) ax.axhline( y=1e-8, color='black', linestyle='--', linewidth=1.0 ) steps = obs.raster("Mg II k").get_raster_pos_steps( raster_pos=0 ) if steps<=250: gridstep=1 elif steps<=2500: gridstep=10 else: gridstep=100 for i in range( 0, len(th), gridstep ): ax.axvline( x=th[i]['DATE_OBS'], color='black', linestyle='--', linewidth=1.0 ) ax.set_ylim([1e-9, 1e-2]) ax.set_ylabel(r'Watts / m$^2$') ax.set_xlabel("Universal Time") ax2 = ax.twinx() ax2.set_yscale( 'log' ) ax2.set_ylim( ax.get_ylim() ) ax2.set_yticks([3e-8, 3e-7, 3e-6, 3e-5, 3e-4]) ax2.set_yticklabels(['A', 'B', 'C', 'M', 'X']) ax2.minorticks_off() ax2.tick_params( right=False ) ax3 = ax.twiny() ax3.set_xlim( ax.get_xlim() ) xticks = [from_Tformat(th[i]['DATE_OBS']) for i in np.arange( 0, len(th), 10*gridstep )] ax3.set_xticks( xticks ) ax3.set_xticklabels( np.arange( 0, len(th), 10*gridstep ) ) ax3.set_xlabel("Exposure") plt.show()
def __init__( self, path, keep_null=False ): # find files in directory self.path = path self._sji_files = self._get_files( path, type='sji' ) self._raster_files = self._get_files( path, type='raster' ) self.n_raster = len( self._raster_files ) self.n_sji = len( self._sji_files ) # raise a warning if > 3000 raster files are present if self.n_raster > 3000: warnings.warn( """This observation contains {} raster files - """ """irisreader will abstract them as one raster but """ """this will be very slow.""".format( self.n_raster) ) # create the sji and raster loaders if len( self._sji_files ) > 0: self.sji = sji_loader( self._sji_files, keep_null ) else: self.sji = None if len( self._raster_files ) > 0: self.raster = raster_loader( self._raster_files, keep_null ) else: self.raster = None # Raise an error if no data files are present if self.sji is None and self.raster is None: raise ValueError("This directory contains no data.") # Issue a warning if SJI or rasters are missing if self.sji is None: warnings.warn("No SJI files in this observation.") if self.raster is None: warnings.warn("No raster files in this observation.") # set a few interesting KPIs self.obsid = self.sji[0].obsid self.mode = self.sji[0].mode self.desc = self.sji[0].desc self.start_date = self.sji[0].start_date self.end_date = self.sji[0].end_date self.full_obsid = self.path.strip('/').split("/")[-1] if not re.match(r"[0-9]{8}_[0-9]{6}_[0-9]{10}", self.full_obsid ): self.full_obsid = None # create the goes loader # TODO: read a few xcenix samples and generate xcen with median self.goes = goes_struct() self.goes.xrs = goes_data( from_Tformat(self.start_date), from_Tformat( self.end_date ), path + "/goes_data", lazy_eval=True ) self.goes.events = hek_data( self, instrument="GOES", lazy_eval=True )
def extract_flare( obs, lambda_min, lambda_max, n_bins, length_minutes=25, flare_margin_minutes=1, savedir="level_2A/", data_format="hdf5", verbosity_level=1 ): raster = obs.raster("Mg II k") # get event info of closest flare mx_flares = obs.goes.events.get_flares( classes="MX" ) closest_flare = mx_flares.iloc[0] # extract raster_pos=0 indices of flare start and stop start_time = from_Tformat( closest_flare['event_starttime'] ) stop_time = from_Tformat( closest_flare['event_endtime'] ) ts = np.array( raster.get_timestamps( raster_pos=0 ) ) start = np.argmin( np.abs(ts-to_epoch(start_time) ) ) stop = np.argmin( np.abs(ts-to_epoch(stop_time) ) ) eff_start_time = from_Tformat( raster.get_raster_pos_headers( raster_pos=0 )[start]['DATE_OBS'] ) eff_stop_time = from_Tformat( raster.get_raster_pos_headers( raster_pos=raster.n_raster_pos-1 )[stop]['DATE_OBS'] ) # plot GOES curve if verbosity_level >= 2: plot_goes_flux( obs, start, stop ) if verbosity_level >= 1: print( "Cutting raster from indices {}-{} --> {:.1f} minutes (of {:.1f} minutes total flare duration)".format( start, stop, (eff_stop_time-eff_start_time).seconds/60, (stop_time-start_time).seconds/60 ) ) # cut the raster raster.cut( raster.get_global_raster_step( raster_pos=0, raster_step=start ), raster.get_global_raster_step( raster_pos=0, raster_step=stop+1 ) ) # show animation if verbosity_level >= 2: display( animate( obs ) ) print("HEK URL:") display( obs.get_hek_url() ) # save the data # print( "Saving data.." ) data = get_interpolated_raster_data( raster, lambda_min, lambda_max, n_bins ) # save_data( data, raster.headers, savedir, "{}_{}".format( "FL", obs.full_obsid ) ) if verbosity_level >= 1: full_obs_timedelta = np.round( ( from_Tformat( raster.time_specific_headers[-1]['DATE_OBS'] ) - from_Tformat( raster.time_specific_headers[0]['DATE_OBS'] ) ).seconds/60, 1 ) print("Raster is {} minutes long after cut".format( full_obs_timedelta ) ) print( "data Shape: {}, raster shape: {}, n_raster_pos: {}".format( data.shape, raster.shape, raster.n_raster_pos ) ) try: sji = obs.sji("Mg II h/k 2796") except: sji = obs.sji("Si") return data, raster, sji
def get_goes_flux( self ): """ Interpolates GOES X-ray flux to time steps of the data cube. Returns ------- float : List of X-ray fluxes """ if self.n_steps > 0: g = goes_data( from_Tformat( self.start_date ), from_Tformat( self.end_date ), os.path.dirname( self._files[0] ) + "/goes_data", lazy_eval=True ) return g.interpolate( self.get_timestamps() ) else: return np.array([])
def _load_time_specific_header_file( self, file_no ): if ir.config.verbosity_level >= 2: print("[iris_data_cube] Lazy loading time specific headers for file {}".format(file_no)) # request file from file hub f = ir.file_hub.open( self._files[file_no] ) # read headers from data array file_time_specific_headers = array2dict( f[self._n_ext-2].header, f[self._n_ext-2].data ) # apply some corrections to the individual headers startobs = from_Tformat( self.primary_headers['STARTOBS'] ) for i in range( len( file_time_specific_headers ) ): # set a DATE_OBS header in each frame as DATE_OBS = STARTOBS + TIME file_time_specific_headers[i]['DATE_OBS'] = to_Tformat( startobs + timedelta( seconds=file_time_specific_headers[i]['TIME'] ) ) # if key 'DSRCNIX' exists: rename it to 'DSRCRCNIX' if 'DSRCNIX' in file_time_specific_headers[i].keys(): file_time_specific_headers[i]['DSRCRCNIX'] = file_time_specific_headers[i].pop('DSRCNIX') # remove some keys (as IDL does it, currently disabled) # for key_to_remove in ['PC1_1IX', 'PC1_2IX', 'PC2_1IX', 'PC2_2IX', 'PC2_3IX', 'PC3_1IX', 'PC3_2IX', 'PC3_3IX', 'OPHASEIX', 'OBS_VRIX']: # if key_to_remove in file_time_specific_headers[i].keys(): # del file_time_specific_headers[i][ key_to_remove ] # return headers return file_time_specific_headers
def _load(self): """ Download HEK data and add IRIS position information. """ # download data from lmsal API self.data = download_hek_data(self.start_date, self.end_date, self.instrument) # add iris position information if len(self.data) > 0: iris_x = [] iris_y = [] for t in self.data.event_starttime: x, y = self.get_iris_coordinates(from_Tformat(t)) iris_x.append(x) iris_y.append(y) self.data['iris_xcenix'] = iris_x self.data['iris_ycenix'] = iris_y # add euclidean distances to IRIS FOV center self.data['dist_arcsec'] = np.sqrt( (self.data.hpc_x - self.data.iris_xcenix)**2 + (self.data.hpc_y - self.data.iris_ycenix)**2) self.data.sort_values(by="dist_arcsec", inplace=True)
def __init__(self, caller, instrument="GOES", lazy_eval=False): self._caller = caller # not every observation has rasters if caller.n_raster > 0: self._caller_cube = caller.raster[0] else: self._caller_cube = caller.sji[0] self.start_date = from_Tformat(self._caller_cube.start_date) self.end_date = from_Tformat(self._caller_cube.end_date) self.instrument = instrument self.data = None if not lazy_eval: self._load()
def get_timestamps( self, raster_pos=None ): """ Converts DATE_OBS to milliseconds since 1970 with the aim to make timestamp comparisons easier. Parameters ---------- raster_pos : int raster position (between 0 and n_raster_pos) Returns ------- float : List of millisecond timestamps. """ if raster_pos is None: headers = self.time_specific_headers else: headers = self.get_raster_pos_headers( raster_pos ) return [to_epoch( from_Tformat( h['DATE_OBS'] ) ) for h in headers]
def animate(self, clr_dic, labels, gamma=0.2, cutoff_percentile=99.9, figsize=(15, 15), transparency=.5, marker_size=5, x_range=None, y_range=None, save_path=None): ''' Setup the first step in the animation. Input - clr_dic: dictionary of colrs associated to each centroid - labels: labels from k-means indicating which centroid each spectra is best represented by. Lables could also be a vector of feature values. - gamma: exponent for gamma correction that adjusts the plot color scale - cutoff_percentile: "often the maximum pixels shine out everything else, even after gamma correction. In order to reduce this effect, the percentile at which to cut the intensity off can be specified with cutoff_percentile in a range between 0 and 100". - transparency: adjust the transparency of the centroid mask (0=transparent, 1=opaque) - marker_size: adjust the size of each marker denoting the spectra position on the SJI - save_path: specify a path to save the data Returns - centroid or feature masked SJI animation of the specified observation. ''' step = 0 fig = plt.figure(figsize=figsize) sji_raster_pos = self.sji_raster(self.steps) times = [ to_epoch(from_Tformat(self.hdrs[sji_raster_pos][i]['DATE_OBS'])) for i in range(self.n_rasters) ] sjiind = np.argmin( np.abs(np.array(self.sji.get_timestamps()) - times[step])) image = self.sji.get_image_step(sjiind).clip(min=0.01)**gamma vmax = np.percentile(image, cutoff_percentile) # constant im = plt.imshow(image, cmap="Greys", vmax=vmax, origin='lower') centroid_mask = self.get_raster_centroid_mask(step, clr_dic, labels) # plot slits slits = np.vstack( (np.asarray(centroid_mask['x']), np.asarray(centroid_mask['y']))) slit_mask = plt.scatter(centroid_mask['x'], centroid_mask['y'], c='white', s=.1) # plot centroid mask clean_centroid_mask = self.clean_mask(step, centroid_mask['x'], centroid_mask['y'], centroid_mask['c']) if self.mode == 'cluster': clrs = clean_centroid_mask['c'] if self.mode == 'features': cmap = plt.cm.get_cmap(self.hmap) clrs = cmap(clean_centroid_mask['c']) scat = plt.scatter(clean_centroid_mask['x'], clean_centroid_mask['y'], c=clrs, s=marker_size, alpha=transparency) date_obs = self.sji.headers[sjiind]['DATE_OBS'] im.axes.set_title("Frame {}: {}".format(step, date_obs), fontsize=18, alpha=.8) plt.xlim(x_range[0], x_range[1]) plt.ylim(y_range[0], y_range[1]) plt.close(fig) # do nothing in the initialization function def init(): return im, # animation function def update(i): ''' Update the data for each successive raster. ''' sjiind = np.argmin( np.abs(np.array(self.sji.get_timestamps()) - times[i])) date_obs = self.sji.headers[sjiind]['DATE_OBS'] im.axes.set_title("Frame {}: {}".format(i, date_obs), fontsize=18, alpha=.8) im.set_data(self.sji.get_image_step(sjiind).clip(min=0.01)**gamma) centroid_mask = self.get_raster_centroid_mask( 0, clr_dic, labels) # make 0->i but doesnt seem to work slits = np.vstack((np.asarray(centroid_mask['x']), np.asarray(centroid_mask['y']))) slit_mask.set_offsets(slits.T) clean_centroid_mask = self.clean_mask(i, centroid_mask['x'], centroid_mask['y'], centroid_mask['c']) dd = np.vstack((np.asarray(clean_centroid_mask['x']), np.asarray(clean_centroid_mask['y']))) scat.set_offsets(dd.T) if self.mode == 'cluster': scat.set_color(clean_centroid_mask['c']) if self.mode == 'features': cmap = plt.cm.get_cmap(self.hmap) scat.set_color(cmap(clean_centroid_mask['c'])) return im, # Call the animator. blit=True means only re-draw the parts that have changed. anim = animation.FuncAnimation(fig, lambda i: update(i), init_func=init, frames=self.n_rasters, interval=200, blit=True) # Save animation if requested if save_path is not None: anim.save(save_path) # Show animation in notebook return HTML(anim.to_html5_video())
def get_mg2k_centroid_table(obs, centroids=None, lambda_min=LAMBDA_MIN, lambda_max=LAMBDA_MAX, crop_raster=False): """ Returns a data frame with centroid counts for each raster image of a given observation. Parameters ---------- obs_path : str Path to observation centroids : numpy.array if None, the centroids defined in the above study will be used, otherwise an array of shape (n_centroids, n_bins) should be passed lambda_min : float wavelength value where interpolation should start lambda_max : float wavelength value where interpolation should stop crop_raster : bool Whether to crop raster before assigning centroids. If set to False, spectra which are -200 everywhere will be assigned to centroid 51 and spectra that are for some part -200 will be assigned to the nearest centroid. Returns ------- centroids_df : pd.DataFrame Data frame with image ids and assigned centroids assigned_centroids : list List with array of assigned centroids for every raster image """ # open raster and crop it if desired if not obs.raster.has_line("Mg II k"): raise Exception("This observation contains no Mg II k line") raster = obs.raster("Mg II k") if crop_raster: raster.crop() # infer number of centroids and number of bins if centroids is None: n_centroids = 53 bins = 216 else: n_centroids = centroids.shape[0] bins = centroids.shape[1] # get goes flux try: goes_flux = raster.get_goes_flux() except Exception as e: print("Could not get GOES flux - setting to None") goes_flux = [None] * raster.n_steps # empty list for assigned centroids assigned_centroids = [] # empty data frame for centroid counts df = pd.DataFrame(columns=[ "full_obsid", "date", "image_no", "goes_flux", "centroid", "count" ]) # assign centroids for each image and create aggregated data frame for step in range(raster.n_steps): # fetch assigned centroids img = interpolate(raster, step, bins=bins, lambda_min=lambda_min, lambda_max=lambda_max) img_assigned_centroids = assign_mg2k_centroids(img, centroids=centroids) assigned_centroids.append(img_assigned_centroids) # count centroids recovered_centroids, counts = np.unique(img_assigned_centroids, return_counts=True) # append to data frame df = df.append(pd.DataFrame({ 'full_obsid': obs.full_obsid, 'date': date.from_Tformat(raster.headers[step]['DATE_OBS']), 'image_no': step, 'goes_flux': goes_flux[step], 'centroid': recovered_centroids, 'count': counts }), sort=False) # create pivot table df = df.pivot_table(index=['full_obsid', 'date', 'image_no', 'goes_flux'], columns='centroid', values='count', aggfunc='first', fill_value=0) df.reset_index(inplace=True) # make sure all centroids are represented for i in range(n_centroids): if i not in df: df[i] = 0 # make sure columns are sorted df = df.reindex(["full_obsid", "date", "image_no", "goes_flux"] + [i for i in range(n_centroids)], axis=1) df.columns.name = '' # rename number columns into string columns cols = df.columns.values cols[4:] = ["c" + str(col) for col in df.columns[4:]] df.columns = cols # close observation obs.close() # return data frame return df, assigned_centroids
if not r: raise Exception( "Connection error. Please check HEK: https://www.lmsal.com/isolsearch" ) events = r.json()["result"] if len(events) == 0: break hek_events += events hek_params['page'] += 1 # convert results to a data frame and filter out events that stopped before the observation time or started after the observation time df = pd.DataFrame(hek_events) if len(df) > 0: df = df[np.logical_and(df.event_starttime < to_Tformat(end_date), df.event_endtime > to_Tformat(start_date))] return df if __name__ == "__main__": from irisreader import observation s = from_Tformat("2014-01-01T21:30:11") e = from_Tformat("2014-01-03T1:30:17") obs = observation( "/home/chuwyler/Desktop/FITS/20140329_140938_3860258481/") h = hek_data(instrument="GOES", caller=obs)
def preprocess_obs( obs, obs_type, lambda_min, lambda_max, n_bins, start_stop_inds=None, length_minutes=25, flare_margin_minutes=1, savedir="level_2A/", data_format="hdf5", verbosity_level=1 ): """ verbosity level : int 0: no output 1: print shapes 2: print flare diagnostics """ raster = obs.raster("Mg II k") if verbosity_level == 1: print("\n---------------------------------------------------------------- Flare Events -------------------------------------------------------------------------------------------------------------------\n") # plot flare locations obs.goes.events.plot_flares() plt.show() # get closest flare flares = obs.goes.events.get_flares() mx_flares = obs.goes.events.get_flares( classes="MX" ) if len( flares ) > 0: display(HTML(flares.to_html())) if len( mx_flares ) > 0: closest_flare = mx_flares.iloc[0] print( "Start time of closest M/X flare: {} (distance {} arcsec)".format( closest_flare['event_starttime'], np.round( closest_flare['dist_arcsec'], 2 ) ) ) else: print( "No M/X flares within range." ) else: print( "No flares within range." ) print("\n---------------------------------------------------------------- Proposed time cut -------------------------------------------------------------------------------------------------------------\n") if start_stop_inds is None: start, stop, delta_t = time_cut( obs, flare=(obs_type=="PF"), length_minutes=length_minutes, flare_margin_minutes=flare_margin_minutes ) else: start = start_stop_inds[0] stop = start_stop_inds[1] delta_t = ( from_Tformat( raster.get_raster_pos_headers( raster_pos=raster.n_raster_pos-1 )[stop-1]['DATE_OBS'] ) - from_Tformat( raster.get_raster_pos_headers( raster_pos=0 )[start]['DATE_OBS'] ) ).seconds/60 if verbosity_level >= 2: plot_goes_flux( obs, start, stop ) if verbosity_level >= 1: print("index interval for raster position 0: {}-{} (totalling {} exposures, {} minutes)".format(start, stop, stop-start, np.round(delta_t, 1 ) ) ) raster.cut( raster.get_global_raster_step( raster_pos=0, raster_step=start ), raster.get_global_raster_step( raster_pos=0, raster_step=stop ) ) if verbosity_level >= 2: display( animate( obs ) ) print("HEK URL:") display( obs.get_hek_url() ) data = get_interpolated_raster_data( raster, lambda_min, lambda_max, n_bins ) if verbosity_level >= 1: full_obs_timedelta = np.round( ( from_Tformat( raster.time_specific_headers[-1]['DATE_OBS'] ) - from_Tformat( raster.time_specific_headers[0]['DATE_OBS'] ) ).seconds/60, 1 ) print("Raster is {} minutes long after cut (should be equal to {} minutes estimated above)".format( full_obs_timedelta, delta_t, 2 ) ) print( "data Shape: {}, raster shape: {}, n_raster_pos: {}".format( data.shape, raster.shape, raster.n_raster_pos ) ) try: sji = obs.sji("Mg II h/k 2796") except: sji = obs.sji("Si") return data, raster, sji
def unique_identifiers(fits_object): """ Creates unique identifiers for the images in the data cube that can be used to refer to an image in a proper, robust way. Format: tllyyyyMMddhhmmssfff t: type of the FITS file (1: sji, 2: raster) ll: line window id yyyy: year MM: month dd: day hh: hour mm: minute ss: seconds fff: milliseconds SJI line window ids: 0 C II 1330 1 Mg II h/k 2796 2 Mg II wing 2832 3 Si IV 1400 raster line window ids: 0 1343 1 2786 2 2787 3 2814 4 2826 5 2830 6 2831 7 2832 8 2833 9 C I 1354 10 C II 1336 11 Cl I 1352 12 Fe XII 1349 13 Mg II h 2803 14 Mg II k 2796 15 O I 1356 16 Si IV 1394 17 Si IV 1403 """ ids = [] dates = [ from_Tformat(h['DATE_OBS']) for h in fits_object.time_specific_headers ] for d in dates: if fits_object.type == 'sji': line_id = sji_linelist.index( fits_object.line_info ) if fits_object.line_info in sji_linelist else 0 fits_type = 1 else: fits_type = 2 line_id = raster_linelist.index( fits_object.line_info ) if fits_object.line_info in raster_linelist else 0 millisec = "{:06d}".format(d.microsecond)[:-3] ids.append("{}{:02d}{}{:02d}{:02d}{:02d}{:02d}{:02d}{}".format( fits_type, line_id, d.year, d.month, d.day, d.hour, d.minute, d.second, millisec)) assert (np.all(np.array(list(map(len, np.array(ids)))) == 20)) return ids