def apply_time_filter(self, filter, hduname='EVENTS', column=CONFIG.TIME_COLUMN): if "GTI" not in self.tables: logging.warn("dataset.apply_time_filter: Dataset GTIs missed") return self if len(self.tables["GTI"].columns["START"].values) == 0: logging.warn("dataset.apply_time_filter: Dataset no valid GTIs") return self columns_values = dict() columns_error_values = dict() for column_name in self.tables[hduname].columns: if column_name != column: columns_values[column_name] = self.tables[hduname].columns[column_name].values columns_error_values[column_name] = self.tables[hduname].columns[column_name].error_values ev_list = self.tables[hduname].columns[column].values ev_list_err = self.tables[hduname].columns[column].error_values gti_start = self.tables["GTI"].columns["START"].values gti_end = self.tables["GTI"].columns["STOP"].values dataset = get_dataset_applying_gtis(self.id, self.tables[hduname].header, self.tables[hduname].header_comments, columns_values, columns_error_values, ev_list, ev_list_err, gti_start, gti_end, filter["from"], filter["to"], hduname, column) return dataset
def apply_time_filter(self, filter, hduname='EVENTS', column=CONFIG.TIME_COLUMN): if "GTI" not in self.tables: logging.warn("dataset.apply_time_filter: Dataset GTIs missed") return self if len(self.tables["GTI"].columns["START"].values) == 0: logging.warn("dataset.apply_time_filter: Dataset no valid GTIs") return self columns_values = dict() columns_error_values = dict() for column_name in self.tables[hduname].columns: if column_name != column: columns_values[column_name] = self.tables[hduname].columns[ column_name].values columns_error_values[column_name] = self.tables[ hduname].columns[column_name].error_values ev_list = self.tables[hduname].columns[column].values ev_list_err = self.tables[hduname].columns[column].error_values gti_start = self.tables["GTI"].columns["START"].values gti_end = self.tables["GTI"].columns["STOP"].values dataset = get_dataset_applying_gtis( self.id, self.tables[hduname].header, self.tables[hduname].header_comments, columns_values, columns_error_values, ev_list, ev_list_err, gti_start, gti_end, filter["from"], filter["to"], hduname, column) return dataset
def get_lightcurve_dataset_from_stingray_Lightcurve(lcurve, header=None, header_comments=None, hduname='RATE', column=CONFIG.TIME_COLUMN): from astropy.io.fits import Header dataset = get_hdu_type_dataset("LIGHTCURVE", [column, hduname], hduname) hdu_table = dataset.tables[hduname] if header is None: if not hasattr(lcurve, 'header'): logging.warn("Light curve has no header") lcurve.header = Header() header = Header.fromstring(lcurve.header) header = dict() for header_column in header: header[header_column] = str(header[header_column]) header_comments[header_column] = \ str(header.comments[header_column]) hdu_table.set_header_info(header, header_comments) hdu_table.columns[column].add_values(lcurve.time) hdu_table.columns[hduname].add_values(lcurve.counts, lcurve.counts_err) dataset.tables["GTI"] = \ DsHelper.get_gti_table_from_stingray_gti(lcurve.gti) return dataset
def get_dataset_start_time(dataset): hdutable = get_hdutable_from_dataset(dataset) if hdutable: return get_column_start_time(hdutable.columns[CONFIG.TIME_COLUMN]) else: logging.warn("get_dataset_start_time: hdutable is None") return 0
def get_dataset_start_time(dataset): hdutable = get_hdutable_from_dataset(dataset) if hdutable: return get_column_start_time(hdutable.columns[CONFIG.TIME_COLUMN]) else: logging.warn("get_dataset_start_time: hdutable is None") return 0
def get_lightcurve_ds_from_events_ds(destination, axis, dt): try: if len(axis) != 2: logging.warn("Wrong number of axis") return "" dataset = DaveReader.get_file_dataset(destination) lc = get_lightcurve_from_dataset(dataset, axis, "", [], "", dt) if lc: #Changes lc format to stingray_addons format tmp_lc = {} tmp_lc['lc'] = lc.countrate tmp_lc['elc'] = [] # TODO: Get error from lightcurve tmp_lc['time'] = lc.time tmp_lc['GTI'] = lc.gti lc_dataset = DataSet.get_lightcurve_dataset_from_stingray_lcurve(tmp_lc, dataset.tables["EVENTS"].header, dataset.tables["EVENTS"].header_comments, "RATE", "TIME") dataset = None # Dispose memory lc = None # Dispose memory new_cache_key = DsCache.get_key(destination + "|ligthcurve") DsCache.add(new_cache_key, dataset) # Adds new cached dataset for new key return new_cache_key except: logging.error(str(sys.exc_info())) return ""
def get_lightcurve_dataset_from_stingray_Lightcurve(lcurve, header=None, header_comments=None, hduname='RATE', column=CONFIG.TIME_COLUMN): from astropy.io.fits import Header dataset = get_hdu_type_dataset("LIGHTCURVE", [column, hduname], hduname) hdu_table = dataset.tables[hduname] if header is None: if not hasattr(lcurve, 'header'): logging.warn("Light curve has no header") lcurve.header = Header() header = Header.fromstring(lcurve.header) header = dict() for header_column in header: header[header_column] = str(header[header_column]) header_comments[header_column] = \ str(header.comments[header_column]) hdu_table.set_header_info(header, header_comments) hdu_table.columns[column].add_values(lcurve.time) hdu_table.columns[hduname].add_values(lcurve.counts, lcurve.counts_err) dataset.tables["GTI"] = \ DsHelper.get_gti_table_from_stingray_gti(lcurve.gti) return dataset
def get_colors_lightcurve(src_destination, bck_destination, gti_destination, filters, axis, dt): if len(axis) != 2: logging.warn("Wrong number of axis") return None try: filters = FltHelper.apply_bin_size_to_filters(filters, dt) count_column_name = "PI" color_keys = FltHelper.get_color_keys_from_filters(filters) filtered_datasets = split_dataset_with_color_filters(src_destination, filters, color_keys, count_column_name, gti_destination) color_axis = get_color_axis_for_ds() # Creates lightcurves array applying bck and gtis from each color logging.debug("Create color lightcurves ....") lightcurves = get_lightcurves_from_datasets_array(filtered_datasets, color_keys, count_column_name, color_axis, bck_destination, filters, gti_destination, dt) filtered_datasets = None # Dispose memory # Preapares the result logging.debug("Result color lightcurves ....") if len(lightcurves) == 4: if lightcurves[0]: result = push_to_results_array([], lightcurves[0].time) result = push_divided_values_to_results_array(result, lightcurves[0].countrate, lightcurves[1].countrate) result = push_divided_values_to_results_array(result, lightcurves[2].countrate, lightcurves[3].countrate) return result except: logging.error(str(sys.exc_info())) return None
def get_fits_dataset(destination, dsId, table_ids): hdulist = fits.open(destination) dataset = DataSet.get_empty_dataset(dsId) for t in range(len(hdulist)): if isinstance(hdulist[t], fits.hdu.table.BinTableHDU): table_id = table_ids[t] header_names = hdulist[t].columns.names tbdata = hdulist[t].data dataset.add_table(table_id, header_names) for i in range(len(header_names)): header_name = header_names[i] dataset.tables[table_id].columns[header_name].values.append( tbdata.field(i)) else: logging.warn("No valid data on: %s" % t) logging.warn("Type of Data: %s" % type(hdulist[t])) hdulist.close() logging.debug("Read fits file successfully: %s" % destination) return dataset
def get_dataset_applying_gti_dataset(src_dataset, gti_dataset, hduname="EVENTS", column=CONFIG.TIME_COLUMN): if not is_events_dataset(src_dataset): logging.warn("get_dataset_applying_gti_dataset: src_dataset is not a events dataset instance") return None if not is_gti_dataset(gti_dataset): logging.warn("get_dataset_applying_gti_dataset: gti_dataset is not a gti dataset instance") return None # Creates the new dataset dataset = src_dataset.clone(False) additional_columns = get_additional_column_names(dataset.tables[hduname].columns, column) hdu_table = src_dataset.tables[hduname] st_gtis = get_stingray_gti_from_gti_table(gti_dataset.tables["GTI"]) ev_list = hdu_table.columns[column].values ev_list_err = hdu_table.columns[column].error_values ds_columns = get_columns_as_dict (src_dataset.tables[hduname].columns, column) ds_columns_errors = get_columns_errors_as_dict (src_dataset.tables[hduname].columns, column) # Gets start time of observation events_start_time = 0 if "TSTART" in hdu_table.header: events_start_time = float(hdu_table.header["TSTART"]) gti_start = st_gtis[:, 0] - events_start_time gti_end = st_gtis[:, 1] - events_start_time update_dataset_filtering_by_gti (dataset.tables[hduname], dataset.tables["GTI"], ev_list, ev_list_err, ds_columns, ds_columns_errors, gti_start, gti_end, additional_columns, column) return dataset
def get_lightcurve_from_lc_dataset(dataset, gti=None): if not is_lightcurve_dataset(dataset): logging.warn( "get_eventlist_from_evt_dataset: dataset is not a events dataset instance" ) return None # Extract axis values time_data = np.array(dataset.tables["RATE"].columns["TIME"].values) counts = np.array(dataset.tables["RATE"].columns["RATE"].values) err_counts = np.array(dataset.tables["RATE"].columns["RATE"].error_values) # Extract GTIs if not gti: gti = get_stingray_gti_from_gti_table(dataset.tables["GTI"]) # Returns the EventList if len(gti) > 0: return Lightcurve(time_data, counts, err=err_counts, input_counts=True, gti=gti) else: return Lightcurve(time_data, counts, err=err_counts, input_counts=True)
def get_astropy_priors(dave_priors): priors = {} num_models = len(dave_priors) for i in range(num_models): model_params = dave_priors[i] for paramName in model_params.keys(): if num_models > 1: prior_key = str(paramName) + "_" + str(i) else: prior_key = str(paramName) model_param = model_params[paramName] if "type" in model_param: if model_param["type"] == "uniform": if ("min" in model_param) and ("max" in model_param): priors[prior_key] = lambda value, min=float( model_param["min"]), max=float(model_param[ "max"]): ((min <= value) & (value <= max)) else: logging.warn( "get_astropy_priors: Wrong uniform prior parameters, prior_key: " + prior_key) elif model_param["type"] == "normal": if ("mean" in model_param) and ("sigma" in model_param): priors[prior_key] = lambda value, mean=float( model_param["mean"]), sigma=float( model_param["sigma"]): scipy.stats.norm( mean, sigma).pdf(value) else: logging.warn( "get_astropy_priors: Wrong normal prior parameters, prior_key: " + prior_key) elif model_param["type"] == "lognormal": if ("mean" in model_param) and ("sigma" in model_param): priors[prior_key] = lambda value, mean=float( model_param["mean"]), sigma=float( model_param["sigma"]): scipy.stats.lognorm( mean, sigma).pdf(value) else: logging.warn( "get_astropy_priors: Wrong lognormal prior parameters, prior_key: " + prior_key) else: logging.warn( "get_astropy_priors: Unknown prior 'type', prior_key: " + prior_key + ", type: " + model_param["type"]) else: logging.warn( "get_astropy_priors: can't find 'type' key on dave_priors, prior_key: " + prior_key) return priors
def get_hdutable_from_dataset(dataset): if is_events_dataset(dataset): return dataset.tables["EVENTS"] elif is_lightcurve_dataset(dataset): return dataset.tables["RATE"] else: logging.warn("get_hdutable_from_dataset: dataset is no Events or Lightcurve type") return None
def get_column_start_time(column): if column.has_extra("TSTART"): return column.get_extra("TSTART") else: logging.warn("get_column_start_time: Couldn't read TSTART from extras, using column[0]") if len(column.values) > 0: return column.values[0] return 0
def split_dataset_with_color_filters(src_destination, filters, color_keys, count_column_name, gti_destination): filtered_datasets = [] for color_key in color_keys: filtered_ds = get_color_filtered_dataset(src_destination, filters, color_key, count_column_name, gti_destination) if not DsHelper.is_events_dataset(filtered_ds): logging.warn("Can't create filtered_ds for " + str(color_key)) return None filtered_datasets.append(filtered_ds) return filtered_datasets
def get_hdutable_from_dataset(dataset): if is_events_dataset(dataset): return dataset.tables["EVENTS"] elif is_lightcurve_dataset(dataset): return dataset.tables["RATE"] else: logging.warn( "get_hdutable_from_dataset: dataset is no Events or Lightcurve type" ) return None
def get_binsize_from_lightcurve_ds(dataset): if is_lightcurve_dataset(dataset): table = dataset.tables["RATE"] if "TIMEDEL" in table.header: return float(table.header["TIMEDEL"]) elif "FRMTIME" in table.header: return float(table.header["FRMTIME"]) / 1000 else: logging.warn("get_binsize_from_lightcurve_ds: Couldn't read TIMEDEL or FRMTIME from RATE HEADER!") return 0
def get_column_start_time(column): if column.has_extra("TSTART"): return column.get_extra("TSTART") else: logging.warn( "get_column_start_time: Couldn't read TSTART from extras, using column[0]" ) if len(column.values) > 0: return column.values[0] return 0
def get_lightcurve_from_dataset(filtered_ds, axis, bck_destination, filters, gti_destination, dt): eventlist = DsHelper.get_eventlist_from_dataset(filtered_ds, axis) if not eventlist or len(eventlist.time) == 0: logging.warn("Wrong lightcurve counts for eventlist from ds.id -> " + str(filtered_ds.id)) return None filtered_ds = None # Dispose memory lc = eventlist.to_lc(dt) if bck_destination: lc = apply_background_to_lc(lc, bck_destination, filters, axis, gti_destination, dt) eventlist = None # Dispose memory return lc
def get_binsize_from_lightcurve_ds(dataset): if is_lightcurve_dataset(dataset): table = dataset.tables["RATE"] if "TIMEDEL" in table.header: return float(table.header["TIMEDEL"]) elif "FRMTIME" in table.header: return float(table.header["FRMTIME"]) / 1000 else: logging.warn( "get_binsize_from_lightcurve_ds: Couldn't read TIMEDEL or FRMTIME from RATE HEADER!" ) return 0
def get_lightcurve_fits_dataset_with_stingray(destination, hdulist, hduname='RATE', column=CONFIG.TIME_COLUMN, gtistring=CONFIG.GTI_STRING, time_offset=0): supported_rate_columns = set(['RATE', 'RATE1', 'COUNTS']) found_rate_columns = set(hdulist[hduname].data.names) intersection_columns = supported_rate_columns.intersection(found_rate_columns) #Check if HDUCLAS1 = LIGHTCURVE column exists logging.debug("Reading Lightcurve Fits columns") if "HDUCLAS1" not in hdulist[hduname].header: logging.warn("HDUCLAS1 not found in header: " + hduname) return None elif hdulist[hduname].header["HDUCLAS1"] != "LIGHTCURVE": logging.warn("HDUCLAS1 is not LIGHTCURVE") return None elif len(intersection_columns) == 0: logging.warn("RATE, RATE1 or COUNTS columns not found in " + str(hduname) + " HDU, found columns: " + str(hdulist[hduname].data.names)) return None elif len(intersection_columns) > 1: logging.warn("RATE, RATE1 or COUNTS ambiguous columns found in " + str(hduname) + " HDU, found columns: " + str(hdulist[hduname].data.names)) return None ratecolumn = list(intersection_columns)[0] if len(hdulist[hduname].data[ratecolumn].shape) != 1 \ or not (isinstance(hdulist[hduname].data[ratecolumn][0], int) \ or isinstance(hdulist[hduname].data[ratecolumn][0], np.integer) \ or isinstance(hdulist[hduname].data[ratecolumn][0], float) \ or isinstance(hdulist[hduname].data[ratecolumn][0], np.floating)): logging.warn("Wrong data type found for column: " + str(ratecolumn) + " in " + str(hduname) + " HDU, expected Integer or Float.") return None header, header_comments = get_header(hdulist, hduname) # Reads the lightcurve with HENDRICS outfile = lcurve_from_fits(destination, gtistring=get_hdu_string_from_hdulist(gtistring, hdulist), timecolumn=column, ratecolumn=ratecolumn, ratehdu=1, fracexp_limit=CONFIG.FRACEXP_LIMIT)[0] lcurve, events_start_time = substract_tstart_from_lcurve(load_data(outfile), time_offset) dataset = DataSet.get_lightcurve_dataset_from_stingray_lcurve(lcurve, header, header_comments, hduname, column) # Stores the events_start_time in time column extra dataset.tables[hduname].columns[column].set_extra("TSTART", events_start_time) logging.debug("Read Lightcurve fits with stingray file successfully: " + str(destination) + ", tstart: " + str(events_start_time) + ", rate: " + str(len(lcurve["counts"]))) return dataset
def join_gti_tables(gti_table_0, gti_table_1): if not gti_table_0: logging.warn("join_gti_tables: gti_table_0 is None, returned gti_table_1") return gti_table_1 if not gti_table_1: logging.warn("join_gti_tables: gti_table_1 is None, returned gti_table_0") return gti_table_0 gti_0 = get_stingray_gti_from_gti_table (gti_table_0) gti_1 = get_stingray_gti_from_gti_table (gti_table_1) joined_gti = join_gtis(gti_0, gti_1) return get_gti_table_from_stingray_gti(joined_gti)
def get_eventlist_from_evt_dataset(dataset): if not is_events_dataset(dataset): logging.warn("get_eventlist_from_evt_dataset: dataset is not a events dataset instance") return None # TODO: Probably all this check can be moved to dave_reader for doing it only once # instead with every call to get_eventlist_from_evt_dataset if not "PHA" in dataset.tables["EVENTS"].columns: logging.warn("get_eventlist_from_evt_dataset: PHA column not found in dataset") dataset.tables["EVENTS"].add_columns(["PHA"]) dataset.tables["EVENTS"].columns["PHA"].set_extra("FAKE_COLUMN", True); try: dataset.tables["EVENTS"].columns["PHA"].values = \ dataset.tables["EVENTS"].columns["PI"].values logging.warn("Using PI instead of PHA") except: dataset.tables["EVENTS"].columns["PHA"].values = \ np.zeros(len(dataset.tables["EVENTS"].columns[CONFIG.TIME_COLUMN].values), dtype=int) logging.warn("PHA column will be empty") # Extract axis values time_data = np.array(dataset.tables["EVENTS"].columns[CONFIG.TIME_COLUMN].values) pha_data = np.array(dataset.tables["EVENTS"].columns["PHA"].values) # Extract GTIs gti = get_stingray_gti_from_gti_table (dataset.tables["GTI"]) # Returns the EventList if len(gti) > 0: return EventList(time_data, gti=gti, pi=pha_data) else: return EventList(time_data, pi=pha_data)
def get_lightcurve(src_destination, bck_destination, gti_destination, filters, axis, dt): time_vals = [] count_rate = [] error_values = [] try: if len(axis) != 2: logging.warn("Wrong number of axis") return None filters = FltHelper.get_filters_clean_color_filters(filters) filters = FltHelper.apply_bin_size_to_filters(filters, dt) filtered_ds = get_filtered_dataset(src_destination, filters, gti_destination) if not DsHelper.is_events_dataset(filtered_ds) \ and not DsHelper.is_lightcurve_dataset(filtered_ds): logging.warn("Wrong dataset type") return None if DsHelper.is_events_dataset(filtered_ds): # Creates lightcurves by gti and joins in one logging.debug("Create lightcurve ....Event count: " + str(len(filtered_ds.tables["EVENTS"].columns["TIME"].values))) lc = get_lightcurve_from_dataset(filtered_ds, axis, bck_destination, filters, gti_destination, dt) filtered_ds = None # Dispose memory if lc: logging.debug("Result time: " + str(len(lc.time))) time_vals = lc.time count_rate = lc.countrate error_values = [] # TODO: Implement error values on Stingray #lc = None # Dispose memory elif DsHelper.is_lightcurve_dataset(filtered_ds): #If dataset is LIGHTCURVE type time_vals = filtered_ds.tables["RATE"].columns["TIME"].values count_rate = filtered_ds.tables["RATE"].columns["RATE"].values error_values = filtered_ds.tables["RATE"].columns["ERROR"].values except: logging.error(str(sys.exc_info())) # Preapares the result logging.debug("Result lightcurve .... " + str(len(time_vals))) result = push_to_results_array([], time_vals) result = push_to_results_array(result, count_rate) result = push_to_results_array(result, error_values) return result
def join_gti_tables(gti_table_0, gti_table_1): if not gti_table_0: logging.warn( "join_gti_tables: gti_table_0 is None, returned gti_table_1") return gti_table_1 if not gti_table_1: logging.warn( "join_gti_tables: gti_table_1 is None, returned gti_table_0") return gti_table_0 gti_0 = get_stingray_gti_from_gti_table(gti_table_0) gti_1 = get_stingray_gti_from_gti_table(gti_table_1) joined_gti = join_gtis(gti_0, gti_1) return get_gti_table_from_stingray_gti(joined_gti)
def get_splited_gti(gti, ti): if ti <= (gti[1] - gti[0])/2: #If the gti is splitable by ti start = gti[0] num_gtis = int((gti[1] - gti[0]) / ti) new_gtis = [] for i in range(num_gtis): end = start + ti new_gtis.append([start, end]) start = end return np.array(new_gtis) else: logging.warn("get_splited_gti: gti is not splitable by time_interval") return None
def get_splited_gti(gti, ti): if ti <= (gti[1] - gti[0]) / 2: #If the gti is splitable by ti start = gti[0] num_gtis = int((gti[1] - gti[0]) / ti) new_gtis = [] for i in range(num_gtis): end = start + ti new_gtis.append([start, end]) start = end return np.array(new_gtis) else: logging.warn("get_splited_gti: gti is not splitable by time_interval") return None
def get_dataset_applying_gti_dataset(src_dataset, gti_dataset, hduname="EVENTS", column='TIME'): if not is_events_dataset(src_dataset): logging.warn( "get_dataset_applying_gti_dataset: src_dataset is not a events dataset instance" ) return None if not is_gti_dataset(gti_dataset): logging.warn( "get_dataset_applying_gti_dataset: gti_dataset is not a gti dataset instance" ) return None # Creates the new dataset dataset = src_dataset.clone(False) additional_columns = get_additional_column_names( dataset.tables[hduname].columns, column) hdu_table = src_dataset.tables[hduname] st_gtis = get_stingray_gti_from_gti_table(gti_dataset.tables["GTI"]) ev_list = hdu_table.columns[column].values ev_list_err = hdu_table.columns[column].error_values ds_columns = get_columns_as_dict(src_dataset.tables[hduname].columns, column) ds_columns_errors = get_columns_errors_as_dict( src_dataset.tables[hduname].columns, column) # Gets start time of observation events_start_time = 0 if "TSTART" in hdu_table.header: events_start_time = float(hdu_table.header["TSTART"]) gti_start = st_gtis[:, 0] - events_start_time gti_end = st_gtis[:, 1] - events_start_time update_dataset_filtering_by_gti(dataset.tables[hduname], dataset.tables["GTI"], ev_list, ev_list_err, ds_columns, ds_columns_errors, gti_start, gti_end, additional_columns, column) return dataset
def get_joined_lightcurves_from_colors(src_destination, bck_destination, gti_destination, filters, axis, dt): if len(axis) != 2: logging.warn("Wrong number of axis") return None try: filters = FltHelper.apply_bin_size_to_filters(filters, dt) # Prepares SRC_LC clean_filters = FltHelper.get_filters_clean_color_filters(filters) filtered_ds = get_filtered_dataset(src_destination, clean_filters, gti_destination) # Creates src lightcurve applying bck and gtis src_lc = get_lightcurve_from_dataset(filtered_ds, axis, bck_destination, clean_filters, gti_destination, dt) if not src_lc: logging.warn("Cant create lc_src") return None count_column_name = "PI" color_keys = FltHelper.get_color_keys_from_filters(filters) filtered_datasets = split_dataset_with_color_filters(src_destination, filters, color_keys, count_column_name, gti_destination) color_axis = get_color_axis_for_ds() # Creates lightcurves array applying bck and gtis from each color logging.debug("Create color lightcurves ....") lightcurves = get_lightcurves_from_datasets_array(filtered_datasets, color_keys, count_column_name, color_axis, bck_destination, filters, gti_destination, dt) filtered_datasets = None # Dispose memory if len(lightcurves) == 2: # Preapares the result logging.debug("Result joined lightcurves ....") result = push_to_results_array([], src_lc.countrate) result = push_divided_values_to_results_array(result, lightcurves[0].countrate, lightcurves[1].countrate) result = push_to_results_array(result, src_lc.time) return result except: logging.error(str(sys.exc_info())) return None
def get_lightcurve_from_lc_dataset(dataset, gti=None): if not is_lightcurve_dataset(dataset): logging.warn("get_eventlist_from_evt_dataset: dataset is not a events dataset instance") return None # Extract axis values time_data = np.array(dataset.tables["RATE"].columns[CONFIG.TIME_COLUMN].values) counts = np.array(dataset.tables["RATE"].columns["RATE"].values) err_counts = np.array(dataset.tables["RATE"].columns["RATE"].error_values) # Extract GTIs if not gti: gti = get_stingray_gti_from_gti_table (dataset.tables["GTI"]) # Returns the EventList if len(gti) > 0: return Lightcurve(time_data, counts, err=err_counts, input_counts=True, gti=gti) else: return Lightcurve(time_data, counts, err=err_counts, input_counts=True)
def substract_tstart_from_lcurve(lcurve, time_offset=0): # Gets start time of observation and substract it from all time data, # sure this can be done on lcurve_from_fits, but I consider this is cleaner events_start_time = 0 real_start_time = 0 if "tstart" in lcurve: real_start_time = lcurve["tstart"] if time_offset == 0: events_start_time = real_start_time else: events_start_time = real_start_time - (real_start_time - time_offset) lcurve["time"] = lcurve["time"] - events_start_time lcurve["gti"][:, 0] = lcurve["gti"][:, 0] - events_start_time lcurve["gti"][:, 1] = lcurve["gti"][:, 1] - events_start_time else: logging.warn("TSTART not readed from lightcurve Fits") return lcurve, real_start_time
def get_astropy_priors(dave_priors): priors = {} num_models = len(dave_priors) for i in range(num_models): model_params = dave_priors[i] for paramName in model_params.keys(): if num_models > 1: prior_key = str(paramName) + "_" + str(i) else: prior_key = str(paramName) model_param = model_params[paramName] if "type" in model_param: if model_param["type"] == "uniform": if ("min" in model_param) and ("max" in model_param): priors[prior_key] = lambda value, min=float(model_param["min"]), max=float(model_param["max"]): ((min <= value) & (value <= max)) else: logging.warn("get_astropy_priors: Wrong uniform prior parameters, prior_key: " + prior_key) elif model_param["type"] == "normal": if ("mean" in model_param) and ("sigma" in model_param): priors[prior_key] = lambda value, mean=float(model_param["mean"]), sigma=float(model_param["sigma"]): scipy.stats.norm(mean, sigma).pdf(value) else: logging.warn("get_astropy_priors: Wrong normal prior parameters, prior_key: " + prior_key) elif model_param["type"] == "lognormal": if ("mean" in model_param) and ("sigma" in model_param): priors[prior_key] = lambda value, mean=float(model_param["mean"]), sigma=float(model_param["sigma"]): scipy.stats.lognorm(mean, sigma).pdf(value) else: logging.warn("get_astropy_priors: Wrong lognormal prior parameters, prior_key: " + prior_key) else: logging.warn("get_astropy_priors: Unknown prior 'type', prior_key: " + prior_key + ", type: " + model_param["type"]) else: logging.warn("get_astropy_priors: can't find 'type' key on dave_priors, prior_key: " + prior_key) return priors
def substract_tstart_from_lcurve(lcurve, time_offset=0): # Gets start time of observation and substract it from all time data, # sure this can be done on lcurve_from_fits, but I consider this is cleaner events_start_time = 0 real_start_time = 0 if "tstart" in lcurve: real_start_time = lcurve["tstart"] if time_offset == 0: events_start_time = real_start_time else: events_start_time = real_start_time - (real_start_time - time_offset) lcurve["time"] = lcurve["time"] - events_start_time lcurve["gti"][:, 0] = lcurve["gti"][:, 0] - events_start_time lcurve["gti"][:, 1] = lcurve["gti"][:, 1] - events_start_time else: logging.warn("TSTART not readed from lightcurve Fits") return lcurve, real_start_time
def get_fits_dataset(hdulist, dsId, table_ids): dataset = DataSet.get_empty_dataset(dsId) for t in range(len(hdulist)): if isinstance(hdulist[t], fits.hdu.table.BinTableHDU): if hdulist[t].name in table_ids: table_id = hdulist[t].name header_names = hdulist[t].columns.names tbdata = hdulist[t].data dataset.add_table(table_id, header_names) header, header_comments = get_header(hdulist, table_id) dataset.tables[table_id].set_header_info(header, header_comments) for i in range(len(header_names)): header_name = header_names[i] dataset.tables[table_id].columns[header_name].add_values(np.nan_to_num(tbdata.field(i))) else: logging.warn("Ignored table data: %s" % hdulist[t].name) else: logging.warn("No valid data on: %s" % t) logging.warn("Type of Data: %s" % type(hdulist[t])) hdulist.close() logging.debug("Read fits file successfully: %s" % dsId) return dataset
def apply_background_to_lc(lc, bck_destination, filters, axis, gti_destination, dt): filtered_bck_ds = get_filtered_dataset(bck_destination, filters, gti_destination) if DsHelper.is_events_dataset(filtered_bck_ds): logging.debug("Create background lightcurve ....") bck_eventlist = DsHelper.get_eventlist_from_dataset(filtered_bck_ds, axis) if bck_eventlist and len(bck_eventlist.time) > 0: bck_lc = bck_eventlist.to_lc(dt) if lc.countrate.shape == bck_lc.countrate.shape: lc.countrate -= bck_lc.countrate else: logging.warn("Background counts differs from lc counts, omiting Bck data.") bck_lc = None else: logging.warn("Wrong lightcurve counts for background data...") bck_eventlist = None # Dispose memory filtered_bck_ds = None else: logging.warn("Background dataset is None!, omiting Bck data.") return lc
def get_fits_dataset(hdulist, dsId, table_ids): dataset = DataSet.get_empty_dataset(dsId) for t in range(len(hdulist)): if isinstance(hdulist[t], fits.hdu.table.BinTableHDU): if hdulist[t].name in table_ids: table_id = hdulist[t].name header_names = hdulist[t].columns.names tbdata = hdulist[t].data dataset.add_table(table_id, header_names) header, header_comments = get_header(hdulist, table_id) dataset.tables[table_id].set_header_info( header, header_comments) for i in range(len(header_names)): header_name = header_names[i] dataset.tables[table_id].columns[header_name].add_values( np.nan_to_num(tbdata.field(i))) else: logging.warn("Ignored table data: %s" % hdulist[t].name) else: logging.warn("No valid data on: %s" % t) logging.warn("Type of Data: %s" % type(hdulist[t])) hdulist.close() logging.debug("Read fits file successfully: %s" % dsId) return dataset
def get_eventlist_from_dataset(dataset, axis): if not is_events_dataset(dataset): logging.warn( "get_eventlist_from_dataset: dataset is not a events dataset instance" ) return None # Extract axis values time_data = np.array( dataset.tables[axis[0]["table"]].columns[axis[0]["column"]].values) pi_data = np.array( dataset.tables[axis[1]["table"]].columns[axis[1]["column"]].values) # Extract GTIs gti = get_stingray_gti_from_gti_table(dataset.tables["GTI"]) # Returns the EventList if len(gti) > 0: return EventList(time_data, gti=gti, pi=pi_data) else: return EventList(time_data, pi=pi_data)
def get_eventlist_dataset_from_stingray_Eventlist(evlist, header=None, header_comments=None, hduname='EVENTS', column='TIME'): from astropy.io.fits import Header lc_columns = [column, "PI", "ENERGY"] dataset = get_hdu_type_dataset("EVENTS", lc_columns, hduname) hdu_table = dataset.tables[hduname] if header is None: if not hasattr(evlist, 'header'): logging.warn("Event list has no header") evlist.header = Header() header = Header.fromstring(evlist.header) header = dict() for header_column in header: header[header_column] = str(header[header_column]) header_comments[header_column] = \ str(header.comments[header_column]) hdu_table.set_header_info(header, header_comments) hdu_table.columns[lc_columns[0]].add_values(evlist.time) if hasattr(evlist, 'energy'): hdu_table.columns['ENERGY'].add_values(evlist.energy) else: hdu_table.columns['ENERGY'].add_values(np.zeros_like(evlist.time)) if hasattr(evlist, 'pi'): hdu_table.columns['PI'].add_values(evlist.pi) else: hdu_table.columns['ENERGY'].add_values(np.zeros_like(evlist.time)) dataset.tables["GTI"] = \ DsHelper.get_gti_table_from_stingray_gti(evlist.gti) return dataset
def get_lightcurve_fits_dataset_with_stingray(destination, hdulist, hduname='RATE', column='TIME', gtistring='GTI,STDGTI'): #Check if HDUCLAS1 = LIGHTCURVE column exists logging.debug("Reading Lightcurve Fits columns") if "HDUCLAS1" not in hdulist[hduname].header: logging.warn("HDUCLAS1 not found in header: " + hduname) return None if hdulist[hduname].header["HDUCLAS1"] != "LIGHTCURVE": logging.warn("HDUCLAS1 is not LIGHTCURVE") return None # Gets FITS header properties header = dict() header_comments = dict() for header_column in hdulist[hduname].header: header[header_column] = str(hdulist[hduname].header[header_column]) header_comments[header_column] = str( hdulist[hduname].header.comments[header_column]) lcurve = lcurve_from_fits(destination, gtistring=gtistring, timecolumn=column, ratecolumn=None, ratehdu=1, fracexp_limit=0.9) dataset = DataSet.get_lightcurve_dataset_from_stingray_lcurve( lcurve, header, header_comments, hduname, column) logging.debug("Read Lightcurve fits with stingray file successfully: %s" % destination) return dataset
def get_eventlist_dataset_from_stingray_Eventlist(evlist, header=None, header_comments=None, hduname='EVENTS', column=CONFIG.TIME_COLUMN): from astropy.io.fits import Header evt_columns = [column, "PI"] if hasattr(evlist, 'energy'): evt_columns = [column, "PI", "E"] dataset = get_hdu_type_dataset("EVENTS", evt_columns, hduname) hdu_table = dataset.tables[hduname] if header is None: if not hasattr(evlist, 'header'): logging.warn("Event list has no header") evlist.header = Header() header = Header.fromstring(evlist.header) header = dict() for header_column in header: header[header_column] = str(header[header_column]) header_comments[header_column] = \ str(header.comments[header_column]) hdu_table.set_header_info(header, header_comments) hdu_table.columns[column].add_values(evlist.time) if hasattr(evlist, 'energy'): if evlist.energy is not None and len(evlist.energy) == len( evlist.time): hdu_table.columns['E'].add_values(evlist.energy) else: logging.warn( "Event list energies differs from event counts, setted all energies as 0" ) hdu_table.columns['E'].add_values(np.zeros_like(evlist.time)) if hasattr(evlist, 'pi') and evlist.pi is not None and len( evlist.pi) == len(evlist.time): hdu_table.columns['PI'].add_values(evlist.pi) else: logging.warn("Event list has no PI values, using np.zeros_like") hdu_table.columns['PI'].add_values(np.zeros_like(evlist.time)) dataset.tables["GTI"] = \ DsHelper.get_gti_table_from_stingray_gti(evlist.gti) return dataset
def get_joined_lightcurves(lc0_destination, lc1_destination, filters, axis, dt): try: if len(axis) != 2: logging.warn("Wrong number of axis") return None filters = FltHelper.get_filters_clean_color_filters(filters) filters = FltHelper.apply_bin_size_to_filters(filters, dt) lc0_ds = get_filtered_dataset(lc0_destination, filters) if not DsHelper.is_lightcurve_dataset(lc0_ds): logging.warn("Wrong dataset type for lc0") return None lc1_ds = get_filtered_dataset(lc1_destination, filters) if not DsHelper.is_lightcurve_dataset(lc1_ds): logging.warn("Wrong dataset type for lc1") return None # Problaby here we can use a stronger checking if len(lc0_ds.tables["RATE"].columns["TIME"].values) == len(lc1_ds.tables["RATE"].columns["TIME"].values): # Preapares the result logging.debug("Result joined lightcurves ....") result = push_to_results_array([], lc0_ds.tables["RATE"].columns["RATE"].values) result = push_to_results_array(result, lc1_ds.tables["RATE"].columns["RATE"].values) result = push_to_results_array(result, lc0_ds.tables["RATE"].columns["TIME"].values) return result else: logging.warn("Lightcurves have different durations.") return None except: logging.error(str(sys.exc_info())) return None
def get_eventlist_from_evt_dataset(dataset): if not is_events_dataset(dataset): logging.warn( "get_eventlist_from_evt_dataset: dataset is not a events dataset instance" ) return None # TODO: Probably all this check can be moved to dave_reader for doing it only once # instead with every call to get_eventlist_from_evt_dataset if not "PHA" in dataset.tables["EVENTS"].columns: logging.warn( "get_eventlist_from_evt_dataset: PHA column not found in dataset") dataset.tables["EVENTS"].add_columns(["PHA"]) dataset.tables["EVENTS"].columns["PHA"].set_extra("FAKE_COLUMN", True) try: dataset.tables["EVENTS"].columns["PHA"].values = \ dataset.tables["EVENTS"].columns["PI"].values logging.warn("Using PI instead of PHA") except: dataset.tables["EVENTS"].columns["PHA"].values = \ np.zeros(len(dataset.tables["EVENTS"].columns[CONFIG.TIME_COLUMN].values), dtype=int) logging.warn("PHA column will be empty") # Extract axis values time_data = np.array( dataset.tables["EVENTS"].columns[CONFIG.TIME_COLUMN].values) pha_data = np.array(dataset.tables["EVENTS"].columns["PHA"].values) # Extract GTIs gti = get_stingray_gti_from_gti_table(dataset.tables["GTI"]) # Returns the EventList if len(gti) > 0: return EventList(time_data, gti=gti, pi=pha_data) else: return EventList(time_data, pi=pha_data)
def get_divided_lightcurve_ds(lc0_destination, lc1_destination): try: lc0_ds = DaveReader.get_file_dataset(lc0_destination) if not DsHelper.is_lightcurve_dataset(lc0_ds): logging.warn("Wrong dataset type for lc0") return "" count_rate_0 = np.array(lc0_ds.tables["RATE"].columns["RATE"].values) lc1_ds = DaveReader.get_file_dataset(lc1_destination) if not DsHelper.is_lightcurve_dataset(lc1_ds): logging.warn("Wrong dataset type for lc1") return "" count_rate_1 = np.array(lc1_ds.tables["RATE"].columns["RATE"].values) if count_rate_0.shape == count_rate_1.shape: ret_lc_ds = lc0_ds.clone(True) with np.errstate(all='ignore'): # Ignore divisions by 0 and others count_rate = np.nan_to_num(count_rate_0 / count_rate_1) count_rate[count_rate > BIG_NUMBER]=0 ret_lc_ds.tables["RATE"].columns["RATE"].clear() ret_lc_ds.tables["RATE"].columns["RATE"].add_values(count_rate) # TODO: Set error from lightcurve lc0_ds = None # Dispose memory lc1_ds = None # Dispose memory count_rate_1 = None # Dispose memory count_rate_0 = None # Dispose memory count_rate = None # Dispose memory new_cache_key = DsCache.get_key(lc0_destination + "|" + lc1_destination + "|ligthcurve") DsCache.add(new_cache_key, ret_lc_ds) # Adds new cached dataset for new key return new_cache_key else: logging.warn("Lightcurves have different shapes.") return None except: logging.error(str(sys.exc_info())) return ""
def get_filtered_dataset(destination, filters, gti_destination=""): dataset = DaveReader.get_file_dataset(destination) if not dataset: logging.warn("get_filtered_dataset: destination specified but not loadable.") return None if gti_destination: gti_dataset = DaveReader.get_file_dataset(gti_destination) if gti_dataset: dataset = DsHelper.get_dataset_applying_gti_dataset(dataset, gti_dataset) if not dataset: logging.warn("get_filtered_dataset: dataset is none after applying gti_dataset.") return None else: logging.warn("get_filtered_dataset: Gti_destination specified but not loadable.") return dataset.apply_filters(filters)
def get_eventlist_dataset_from_stingray_Eventlist(evlist, header=None, header_comments=None, hduname='EVENTS', column=CONFIG.TIME_COLUMN): from astropy.io.fits import Header evt_columns = [column, "PI"] if hasattr(evlist, 'energy'): evt_columns = [column, "PI", "E"] dataset = get_hdu_type_dataset("EVENTS", evt_columns, hduname) hdu_table = dataset.tables[hduname] if header is None: if not hasattr(evlist, 'header'): logging.warn("Event list has no header") evlist.header = Header() header = Header.fromstring(evlist.header) header = dict() for header_column in header: header[header_column] = str(header[header_column]) header_comments[header_column] = \ str(header.comments[header_column]) hdu_table.set_header_info(header, header_comments) hdu_table.columns[column].add_values(evlist.time) if hasattr(evlist, 'energy'): if evlist.energy is not None and len(evlist.energy) == len(evlist.time): hdu_table.columns['E'].add_values(evlist.energy) else: logging.warn("Event list energies differs from event counts, setted all energies as 0") hdu_table.columns['E'].add_values(np.zeros_like(evlist.time)) if hasattr(evlist, 'pi') and evlist.pi is not None and len(evlist.pi) == len(evlist.time): hdu_table.columns['PI'].add_values(evlist.pi) else: logging.warn("Event list has no PI values, using np.zeros_like") hdu_table.columns['PI'].add_values(np.zeros_like(evlist.time)) dataset.tables["GTI"] = \ DsHelper.get_gti_table_from_stingray_gti(evlist.gti) return dataset
def get_file_dataset(destination, time_offset=0): dataset = None cache_key = "" try: if destination: cache_key = get_cache_key_for_destination(destination, time_offset) if DsCache.contains(cache_key): logging.debug("get_file_dataset: returned cached dataset, cache_key: " + str(cache_key)) return DsCache.get(cache_key), cache_key logging.debug("get_file_dataset: reading destination: " + str(destination)) filename = os.path.splitext(destination)[0] file_extension_from_file = os.path.splitext(destination)[1] file_extension = magic.from_file(destination) logging.debug("File extension: %s" % file_extension) if file_extension.find("ASCII") == 0: table_id = "EVENTS" header_names = [CONFIG.TIME_COLUMN, "PHA", "Color1", "Color2"] dataset = get_txt_dataset(destination, table_id, header_names) table = dataset.tables[table_id] table.add_columns(["AMPLITUDE"]) numValues = len(table.columns[CONFIG.TIME_COLUMN].values) random_values = np.random.uniform(-1, 1, size=numValues) table.columns["AMPLITUDE"].values = random_values elif file_extension.find("FITS") == 0 \ or file_extension.find("gzip") > -1: # Opening Fits hdulist = fits.open(destination, memmap=True) if get_hdu_string_from_hdulist(CONFIG.EVENTS_STRING, hdulist) != "": # If EVENTS extension found, consider the Fits as EVENTS Fits dataset = get_events_fits_dataset_with_stingray(destination, hdulist, dsId='FITS', hduname=get_hdu_string_from_hdulist(CONFIG.EVENTS_STRING, hdulist), column=CONFIG.TIME_COLUMN, gtistring=CONFIG.GTI_STRING, extra_colums=['PI', "PHA"], time_offset=time_offset) elif 'RATE' in hdulist: # If RATE extension found, consider the Fits as LIGHTCURVE Fits dataset = get_lightcurve_fits_dataset_with_stingray(destination, hdulist, hduname='RATE', column=CONFIG.TIME_COLUMN, gtistring=CONFIG.GTI_STRING, time_offset=time_offset) elif 'EBOUNDS' in hdulist: # If EBOUNDS extension found, consider the Fits as RMF Fits dataset = get_fits_dataset(hdulist, "RMF", ["EBOUNDS"]) elif get_hdu_string_from_hdulist(CONFIG.GTI_STRING, hdulist) != "": # If not EVENTS or RATE extension found, check if is GTI Fits dataset = get_gti_fits_dataset_with_stingray(hdulist,gtistring=CONFIG.GTI_STRING, time_offset=time_offset) else: logging.warn("Unsupported FITS type! Any table found: " + CONFIG.EVENTS_STRING + ", RATE, EBOUNDS or " + CONFIG.GTI_STRING) elif file_extension == "data" and (file_extension_from_file in [".p", ".nc"]): # If file is pickle object, tries to parse it as dataset dataset = load_dataset_from_intermediate_file(destination) else: logging.warn("Unknown file extension: " + str(file_extension) + " , " + str(file_extension_from_file)) if dataset: DsCache.add(cache_key, dataset) logging.debug("get_file_dataset, dataset added to cache, cache_key: " + str(cache_key)) else: logging.error("get_file_dataset: Destination is empty") except: logging.error(ExHelper.getException('get_file_dataset')) return dataset, cache_key
def get_file_dataset(destination, time_offset=0): dataset = None cache_key = "" try: if destination: cache_key = get_cache_key_for_destination(destination, time_offset) if DsCache.contains(cache_key): logging.debug( "get_file_dataset: returned cached dataset, cache_key: " + str(cache_key)) return DsCache.get(cache_key), cache_key logging.debug("get_file_dataset: reading destination: " + str(destination)) filename = os.path.splitext(destination)[0] file_extension_from_file = os.path.splitext(destination)[1] file_extension = magic.from_file(destination) logging.debug("File extension: %s" % file_extension) if file_extension.find("ASCII") == 0: table_id = "EVENTS" header_names = [CONFIG.TIME_COLUMN, "PHA", "Color1", "Color2"] dataset = get_txt_dataset(destination, table_id, header_names) table = dataset.tables[table_id] table.add_columns(["AMPLITUDE"]) numValues = len(table.columns[CONFIG.TIME_COLUMN].values) random_values = np.random.uniform(-1, 1, size=numValues) table.columns["AMPLITUDE"].values = random_values elif file_extension.find("FITS") == 0 \ or file_extension.find("gzip") > -1: # Opening Fits hdulist = fits.open(destination, memmap=True) if get_hdu_string_from_hdulist(CONFIG.EVENTS_STRING, hdulist) != "": # If EVENTS extension found, consider the Fits as EVENTS Fits dataset = get_events_fits_dataset_with_stingray( destination, hdulist, dsId='FITS', hduname=get_hdu_string_from_hdulist( CONFIG.EVENTS_STRING, hdulist), column=CONFIG.TIME_COLUMN, gtistring=CONFIG.GTI_STRING, extra_colums=['PI', "PHA"], time_offset=time_offset) elif 'RATE' in hdulist: # If RATE extension found, consider the Fits as LIGHTCURVE Fits dataset = get_lightcurve_fits_dataset_with_stingray( destination, hdulist, hduname='RATE', column=CONFIG.TIME_COLUMN, gtistring=CONFIG.GTI_STRING, time_offset=time_offset) elif 'EBOUNDS' in hdulist: # If EBOUNDS extension found, consider the Fits as RMF Fits dataset = get_fits_dataset(hdulist, "RMF", ["EBOUNDS"]) elif get_hdu_string_from_hdulist(CONFIG.GTI_STRING, hdulist) != "": # If not EVENTS or RATE extension found, check if is GTI Fits dataset = get_gti_fits_dataset_with_stingray( hdulist, gtistring=CONFIG.GTI_STRING, time_offset=time_offset) else: logging.warn("Unsupported FITS type! Any table found: " + CONFIG.EVENTS_STRING + ", RATE, EBOUNDS or " + CONFIG.GTI_STRING) elif file_extension == "data" and (file_extension_from_file in [".p", ".nc"]): # If file is pickle object, tries to parse it as dataset dataset = load_dataset_from_intermediate_file(destination) else: logging.warn("Unknown file extension: " + str(file_extension) + " , " + str(file_extension_from_file)) if dataset: DsCache.add(cache_key, dataset) logging.debug( "get_file_dataset, dataset added to cache, cache_key: " + str(cache_key)) else: logging.error("get_file_dataset: Destination is empty") except: logging.error(ExHelper.getException('get_file_dataset')) return dataset, cache_key