def get_lightcurve_ds_from_events_ds(destination, axis, dt): try: if len(axis) != 2: logging.warn("Wrong number of axis") return "" dataset = DaveReader.get_file_dataset(destination) lc = get_lightcurve_from_dataset(dataset, axis, "", [], "", dt) if lc: #Changes lc format to stingray_addons format tmp_lc = {} tmp_lc['lc'] = lc.countrate tmp_lc['elc'] = [] # TODO: Get error from lightcurve tmp_lc['time'] = lc.time tmp_lc['GTI'] = lc.gti lc_dataset = DataSet.get_lightcurve_dataset_from_stingray_lcurve(tmp_lc, dataset.tables["EVENTS"].header, dataset.tables["EVENTS"].header_comments, "RATE", "TIME") dataset = None # Dispose memory lc = None # Dispose memory new_cache_key = DsCache.get_key(destination + "|ligthcurve") DsCache.add(new_cache_key, dataset) # Adds new cached dataset for new key return new_cache_key except: logging.error(str(sys.exc_info())) return ""
def append_file_to_dataset(destination, next_destination): dataset = DaveReader.get_file_dataset(destination) if DsHelper.is_events_dataset(dataset): next_dataset = DaveReader.get_file_dataset(next_destination) if DsHelper.is_events_dataset(next_dataset): # Looks what dataset is earliest ds_start_time = DsHelper.get_events_dataset_start(dataset) next_ds_start_time = DsHelper.get_events_dataset_start(next_dataset) if next_ds_start_time < ds_start_time: #Swap datasets tmp_ds = dataset dataset = next_dataset next_dataset = tmp_ds #Join and cache joined dataset dataset.tables["EVENTS"] = dataset.tables["EVENTS"].join(next_dataset.tables["EVENTS"]) dataset.tables["GTI"] = DsHelper.join_gti_tables(dataset.tables["GTI"], next_dataset.tables["GTI"]) DsCache.remove(destination) # Removes previous cached dataset for prev key new_cache_key = DsCache.get_key(destination + "|" + next_destination) DsCache.add(new_cache_key, dataset) # Adds new cached dataset for new key return new_cache_key return ""
def get_divided_lightcurve_ds(lc0_destination, lc1_destination): try: lc0_ds = DaveReader.get_file_dataset(lc0_destination) if not DsHelper.is_lightcurve_dataset(lc0_ds): logging.warn("Wrong dataset type for lc0") return "" count_rate_0 = np.array(lc0_ds.tables["RATE"].columns["RATE"].values) lc1_ds = DaveReader.get_file_dataset(lc1_destination) if not DsHelper.is_lightcurve_dataset(lc1_ds): logging.warn("Wrong dataset type for lc1") return "" count_rate_1 = np.array(lc1_ds.tables["RATE"].columns["RATE"].values) if count_rate_0.shape == count_rate_1.shape: ret_lc_ds = lc0_ds.clone(True) with np.errstate(all='ignore'): # Ignore divisions by 0 and others count_rate = np.nan_to_num(count_rate_0 / count_rate_1) count_rate[count_rate > BIG_NUMBER]=0 ret_lc_ds.tables["RATE"].columns["RATE"].clear() ret_lc_ds.tables["RATE"].columns["RATE"].add_values(count_rate) # TODO: Set error from lightcurve lc0_ds = None # Dispose memory lc1_ds = None # Dispose memory count_rate_1 = None # Dispose memory count_rate_0 = None # Dispose memory count_rate = None # Dispose memory new_cache_key = DsCache.get_key(lc0_destination + "|" + lc1_destination + "|ligthcurve") DsCache.add(new_cache_key, ret_lc_ds) # Adds new cached dataset for new key return new_cache_key else: logging.warn("Lightcurves have different shapes.") return None except: logging.error(str(sys.exc_info())) return ""
def get_file_dataset(destination, time_offset=0): dataset = None cache_key = "" try: if destination: cache_key = get_cache_key_for_destination(destination, time_offset) if DsCache.contains(cache_key): logging.debug( "get_file_dataset: returned cached dataset, cache_key: " + str(cache_key)) return DsCache.get(cache_key), cache_key logging.debug("get_file_dataset: reading destination: " + str(destination)) filename = os.path.splitext(destination)[0] file_extension_from_file = os.path.splitext(destination)[1] file_extension = magic.from_file(destination) logging.debug("File extension: %s" % file_extension) if file_extension.find("ASCII") == 0: table_id = "EVENTS" header_names = [CONFIG.TIME_COLUMN, "PHA", "Color1", "Color2"] dataset = get_txt_dataset(destination, table_id, header_names) table = dataset.tables[table_id] table.add_columns(["AMPLITUDE"]) numValues = len(table.columns[CONFIG.TIME_COLUMN].values) random_values = np.random.uniform(-1, 1, size=numValues) table.columns["AMPLITUDE"].values = random_values elif file_extension.find("FITS") == 0 \ or file_extension.find("gzip") > -1: # Opening Fits hdulist = fits.open(destination, memmap=True) if get_hdu_string_from_hdulist(CONFIG.EVENTS_STRING, hdulist) != "": # If EVENTS extension found, consider the Fits as EVENTS Fits dataset = get_events_fits_dataset_with_stingray( destination, hdulist, dsId='FITS', hduname=get_hdu_string_from_hdulist( CONFIG.EVENTS_STRING, hdulist), column=CONFIG.TIME_COLUMN, gtistring=CONFIG.GTI_STRING, extra_colums=['PI', "PHA"], time_offset=time_offset) elif 'RATE' in hdulist: # If RATE extension found, consider the Fits as LIGHTCURVE Fits dataset = get_lightcurve_fits_dataset_with_stingray( destination, hdulist, hduname='RATE', column=CONFIG.TIME_COLUMN, gtistring=CONFIG.GTI_STRING, time_offset=time_offset) elif 'EBOUNDS' in hdulist: # If EBOUNDS extension found, consider the Fits as RMF Fits dataset = get_fits_dataset(hdulist, "RMF", ["EBOUNDS"]) elif get_hdu_string_from_hdulist(CONFIG.GTI_STRING, hdulist) != "": # If not EVENTS or RATE extension found, check if is GTI Fits dataset = get_gti_fits_dataset_with_stingray( hdulist, gtistring=CONFIG.GTI_STRING, time_offset=time_offset) else: logging.warn("Unsupported FITS type! Any table found: " + CONFIG.EVENTS_STRING + ", RATE, EBOUNDS or " + CONFIG.GTI_STRING) elif file_extension == "data" and (file_extension_from_file in [".p", ".nc"]): # If file is pickle object, tries to parse it as dataset dataset = load_dataset_from_intermediate_file(destination) else: logging.warn("Unknown file extension: " + str(file_extension) + " , " + str(file_extension_from_file)) if dataset: DsCache.add(cache_key, dataset) logging.debug( "get_file_dataset, dataset added to cache, cache_key: " + str(cache_key)) else: logging.error("get_file_dataset: Destination is empty") except: logging.error(ExHelper.getException('get_file_dataset')) return dataset, cache_key
def get_file_dataset(destination, time_offset=0): dataset = None cache_key = "" try: if destination: cache_key = get_cache_key_for_destination(destination, time_offset) if DsCache.contains(cache_key): logging.debug("get_file_dataset: returned cached dataset, cache_key: " + str(cache_key)) return DsCache.get(cache_key), cache_key logging.debug("get_file_dataset: reading destination: " + str(destination)) filename = os.path.splitext(destination)[0] file_extension_from_file = os.path.splitext(destination)[1] file_extension = magic.from_file(destination) logging.debug("File extension: %s" % file_extension) if file_extension.find("ASCII") == 0: table_id = "EVENTS" header_names = [CONFIG.TIME_COLUMN, "PHA", "Color1", "Color2"] dataset = get_txt_dataset(destination, table_id, header_names) table = dataset.tables[table_id] table.add_columns(["AMPLITUDE"]) numValues = len(table.columns[CONFIG.TIME_COLUMN].values) random_values = np.random.uniform(-1, 1, size=numValues) table.columns["AMPLITUDE"].values = random_values elif file_extension.find("FITS") == 0 \ or file_extension.find("gzip") > -1: # Opening Fits hdulist = fits.open(destination, memmap=True) if get_hdu_string_from_hdulist(CONFIG.EVENTS_STRING, hdulist) != "": # If EVENTS extension found, consider the Fits as EVENTS Fits dataset = get_events_fits_dataset_with_stingray(destination, hdulist, dsId='FITS', hduname=get_hdu_string_from_hdulist(CONFIG.EVENTS_STRING, hdulist), column=CONFIG.TIME_COLUMN, gtistring=CONFIG.GTI_STRING, extra_colums=['PI', "PHA"], time_offset=time_offset) elif 'RATE' in hdulist: # If RATE extension found, consider the Fits as LIGHTCURVE Fits dataset = get_lightcurve_fits_dataset_with_stingray(destination, hdulist, hduname='RATE', column=CONFIG.TIME_COLUMN, gtistring=CONFIG.GTI_STRING, time_offset=time_offset) elif 'EBOUNDS' in hdulist: # If EBOUNDS extension found, consider the Fits as RMF Fits dataset = get_fits_dataset(hdulist, "RMF", ["EBOUNDS"]) elif get_hdu_string_from_hdulist(CONFIG.GTI_STRING, hdulist) != "": # If not EVENTS or RATE extension found, check if is GTI Fits dataset = get_gti_fits_dataset_with_stingray(hdulist,gtistring=CONFIG.GTI_STRING, time_offset=time_offset) else: logging.warn("Unsupported FITS type! Any table found: " + CONFIG.EVENTS_STRING + ", RATE, EBOUNDS or " + CONFIG.GTI_STRING) elif file_extension == "data" and (file_extension_from_file in [".p", ".nc"]): # If file is pickle object, tries to parse it as dataset dataset = load_dataset_from_intermediate_file(destination) else: logging.warn("Unknown file extension: " + str(file_extension) + " , " + str(file_extension_from_file)) if dataset: DsCache.add(cache_key, dataset) logging.debug("get_file_dataset, dataset added to cache, cache_key: " + str(cache_key)) else: logging.error("get_file_dataset: Destination is empty") except: logging.error(ExHelper.getException('get_file_dataset')) return dataset, cache_key
def get_file_dataset(destination): if not destination: return None if DsCache.contains(destination): logging.debug("Returned cached dataset") return DsCache.get(destination) filename = os.path.splitext(destination)[0] file_extension = magic.from_file(destination) logging.debug("File extension: %s" % file_extension) if file_extension.find("ASCII") == 0: table_id = "txt_table" header_names = ["Time", "Rate", "color1", "color2"] dataset = get_txt_dataset(destination, table_id, header_names) table = dataset.tables[table_id] table.add_columns(["Amplitude"]) numValues = len(table.columns["Time"].values) random_values = np.random.uniform(-1, 1, size=numValues) table.columns["Amplitude"].values = random_values DsCache.add(destination, dataset) return dataset elif file_extension.find("FITS") == 0: # ds_id = "fits_table" # table_ids = ["Primary", "EVENTS", "GTI"] # dataset = get_fits_dataset(destination, ds_id, table_ids) # return dataset # Opening Fits hdulist = fits.open(destination) if 'EVENTS' in hdulist: # If EVENTS extension found, consider the Fits as EVENTS Fits dataset = get_events_fits_dataset_with_stingray( destination, hdulist, dsId='FITS', hduname='EVENTS', column='TIME', gtistring=gtistring) elif 'RATE' in hdulist: # If RATE extension found, consider the Fits as LIGHTCURVE Fits dataset = get_lightcurve_fits_dataset_with_stingray( destination, hdulist, hduname='RATE', column='TIME', gtistring=gtistring) else: # If not EVENTS or RATE extension found, consider the Fits as GTI Fits dataset = get_gti_fits_dataset_with_stingray(hdulist, gtistring=gtistring) if dataset: DsCache.add(destination, dataset) return dataset else: return None