def apply_filters(self, filters): if not filters or not len(filters): return self filtered_dataset = self.clone() time_filter = FltHelper.get_time_filter( filters) # Firts filter by time for reducing arrays length if time_filter: filtered_dataset = self.apply_time_filter(time_filter, time_filter["table"]) for filter in filters: table_id = filter["table"] if table_id not in [ "EVENTS", "RATE" ] or filter["column"] != CONFIG.TIME_COLUMN: # Exclude time filter if table_id in filtered_dataset.tables: filtered_dataset.tables[ table_id] = filtered_dataset.tables[ table_id].apply_filter(filter) else: logging.error("dataset.apply_filters wrong table_id: %s" % table_id) return filtered_dataset
def load_dataset_from_intermediate_file(fname): """Save Stingray object to intermediate file.""" from stingray.lightcurve import Lightcurve from stingray.events import EventList from stingray.crossspectrum import Crossspectrum from hendrics.io import get_file_type from stingray.io import _retrieve_pickle_object # This will return an EventList, a light curve, a Powerspectrum, ... # depending on the contents of the file try: ftype, contents = get_file_type(fname) except: contents = _retrieve_pickle_object(fname) if isinstance(contents, Lightcurve): return DataSet.get_lightcurve_dataset_from_stingray_Lightcurve( contents) elif isinstance(contents, EventList): return DataSet.get_eventlist_dataset_from_stingray_Eventlist(contents) # This also work for Powerspectrum and AveragedCrosspowerspectrum, clearly elif isinstance(contents, Crossspectrum): logging.error("Unsupported intermediate file type: Crossspectrum") else: logging.error("Unsupported intermediate file type: %s" % type(stingray_object).__name__) return None
def append_file_to_dataset(filename, nextfile, target): destination = get_destination(filename, target) if not destination: return common_error("Invalid file or cache key") if not nextfile: return common_error("No nextfile setted") if not SessionHelper.is_file_uploaded(nextfile): if not FileUtils.file_exist(target, nextfile): logging.error("Filename not uploaded for nextfile %s" % nextfile) return common_error("Nextfile not uploaded") next_destination = FileUtils.get_destination(target, nextfile) if not FileUtils.is_valid_file(next_destination): return common_error("Invalid next file") logging.debug("append_file_to_dataset, destination: %s" % destination) logging.debug("append_file_to_dataset, next_destination: %s" % next_destination) new_filename = DaveEngine.append_file_to_dataset(destination, next_destination) logging.debug("append_file_to_dataset, cache_key: %s" % new_filename) return json.dumps(new_filename)
def default(self, obj): try: if isinstance(obj, int): if obj > CONFIG.BIG_NUMBER: return CONFIG.BIG_NUMBER if obj < -CONFIG.BIG_NUMBER: return -CONFIG.BIG_NUMBER return int(obj) elif isinstance(obj, float): if obj > CONFIG.BIG_NUMBER: return CONFIG.BIG_NUMBER if obj < -CONFIG.BIG_NUMBER: return -CONFIG.BIG_NUMBER return float(obj) if isinstance(obj, numpy.integer): if obj > CONFIG.BIG_NUMBER: return CONFIG.BIG_NUMBER if obj < -CONFIG.BIG_NUMBER: return -CONFIG.BIG_NUMBER return int(obj) elif isinstance(obj, numpy.floating): if obj > CONFIG.BIG_NUMBER: return CONFIG.BIG_NUMBER if obj < -CONFIG.BIG_NUMBER: return -CONFIG.BIG_NUMBER return float(obj) elif isinstance(obj, complex): return self.default(numpy.real(obj)) elif isinstance(obj, numpy.ndarray): return obj.tolist() else: return super(NPEncoder, self).default(obj) except: logging.error(ExHelper.getException('NPEncoder')) return None
def apply_filter(self, filter): column_name = filter["column"] if column_name not in self.columns: logging.error("table.apply_filter wrong column: %s" % column_name) return self if filter["from"] > filter["to"]: logging.error("table.apply_filter wrong from-to: %s" % column_name) return self filtered_table = Table(self.id) for tmp_column_name in self.columns: filtered_table.columns[tmp_column_name] = Column(tmp_column_name) column = self.columns[column_name] values = np.array(column.values) filtered_indexes = np.where((values >= filter["from"]) & (values <= filter["to"]))[0] for column_name in self.columns: col_values, col_error_values = self.columns[ column_name].get_values(filtered_indexes) filtered_table.columns[column_name].add_values( col_values, col_error_values) return filtered_table
def get_colors_lightcurve(src_destination, bck_destination, gti_destination, filters, axis, dt): if len(axis) != 2: logging.warn("Wrong number of axis") return None try: filters = FltHelper.apply_bin_size_to_filters(filters, dt) count_column_name = "PI" color_keys = FltHelper.get_color_keys_from_filters(filters) filtered_datasets = split_dataset_with_color_filters(src_destination, filters, color_keys, count_column_name, gti_destination) color_axis = get_color_axis_for_ds() # Creates lightcurves array applying bck and gtis from each color logging.debug("Create color lightcurves ....") lightcurves = get_lightcurves_from_datasets_array(filtered_datasets, color_keys, count_column_name, color_axis, bck_destination, filters, gti_destination, dt) filtered_datasets = None # Dispose memory # Preapares the result logging.debug("Result color lightcurves ....") if len(lightcurves) == 4: if lightcurves[0]: result = push_to_results_array([], lightcurves[0].time) result = push_divided_values_to_results_array(result, lightcurves[0].countrate, lightcurves[1].countrate) result = push_divided_values_to_results_array(result, lightcurves[2].countrate, lightcurves[3].countrate) return result except: logging.error(str(sys.exc_info())) return None
def load_dataset_from_intermediate_file(fname): """Save Stingray object to intermediate file.""" from stingray.lightcurve import Lightcurve from stingray.events import EventList from stingray.crossspectrum import Crossspectrum from hendrics.io import get_file_type from stingray.io import _retrieve_pickle_object # This will return an EventList, a light curve, a Powerspectrum, ... # depending on the contents of the file try: ftype, contents = get_file_type(fname) except: contents = _retrieve_pickle_object(fname) if isinstance(contents, Lightcurve): return DataSet.get_lightcurve_dataset_from_stingray_Lightcurve(contents) elif isinstance(contents, EventList): return DataSet.get_eventlist_dataset_from_stingray_Eventlist(contents) # This also work for Powerspectrum and AveragedCrosspowerspectrum, clearly elif isinstance(contents, Crossspectrum): logging.error("Unsupported intermediate file type: Crossspectrum") else: logging.error("Unsupported intermediate file type: %s" % type(stingray_object).__name__) return None
def get_lightcurve_ds_from_events_ds(destination, axis, dt): try: if len(axis) != 2: logging.warn("Wrong number of axis") return "" dataset = DaveReader.get_file_dataset(destination) lc = get_lightcurve_from_dataset(dataset, axis, "", [], "", dt) if lc: #Changes lc format to stingray_addons format tmp_lc = {} tmp_lc['lc'] = lc.countrate tmp_lc['elc'] = [] # TODO: Get error from lightcurve tmp_lc['time'] = lc.time tmp_lc['GTI'] = lc.gti lc_dataset = DataSet.get_lightcurve_dataset_from_stingray_lcurve(tmp_lc, dataset.tables["EVENTS"].header, dataset.tables["EVENTS"].header_comments, "RATE", "TIME") dataset = None # Dispose memory lc = None # Dispose memory new_cache_key = DsCache.get_key(destination + "|ligthcurve") DsCache.add(new_cache_key, dataset) # Adds new cached dataset for new key return new_cache_key except: logging.error(str(sys.exc_info())) return ""
def apply_filter(self, filter): column_name = filter["column"] if column_name not in self.columns: logging.error("table.apply_filter wrong column: %s" % column_name) return self if filter["from"] > filter["to"]: logging.error("table.apply_filter wrong from-to: %s" % column_name) return self filtered_table = Table(self.id) for tmp_column_name in self.columns: filtered_table.columns[tmp_column_name] = Column(tmp_column_name) column = self.columns[column_name] filtered_indexes = np.array([ i for i in range(len(column.values)) if ((column.values[i] >= filter["from"]) and ( column.values[i] <= filter["to"])) ]) for i in filtered_indexes: filtered_table.add_row(self.get_row(i)) return filtered_table
def get_intermediate_file(filepath, target): try: stingray_object = DaveReader.get_stingray_object(filepath) if stingray_object: filename = FileUtils.get_intermediate_filename( target, filepath, HEN_FILE_EXTENSION) if DaveReader.save_to_intermediate_file(stingray_object, filename): return filename except: logging.error(ExHelper.getException('get_intermediate_file')) return None
def get_intermediate_files(filepaths, target): filenames = [] for filepath in filepaths: if not FileUtils.is_valid_file(filepath): logging.error("Filepath not found or invalid: %s" % filepath) else: filename = DaveBulk.get_intermediate_file(filepath, target) logging.debug("get_intermediate_files filename: %s" % filename) if filename: filenames.append(filename) return json.dumps(filenames, cls=NPEncoder)
def build_data_list(dataset, axis): data = [] for i in range(len(axis)): table_name = axis[i]["table"] if table_name in dataset.tables: column = dataset.tables[table_name].columns[axis[i]["column"]] column_data = dict() column_data["values"] = column.values column_data["error_values"] = column.error_values data = np.append(data, [column_data]) else: logging.error("Accessing unknown table: %s" % table_name) return data
def get_destination(target, filename): try: if CONFIG.IS_LOCAL_SERVER: if filename.startswith('/') and os.path.isfile(filename): # This is supposed to be an absolute path return filename else: # Relative path return "/".join([target, filename]) else: return "/".join([target, secure_filename(filename)]) except: logging.error(ExHelper.getException('get_destination')) return ""
def get_lightcurve(src_destination, bck_destination, gti_destination, filters, axis, dt): time_vals = [] count_rate = [] error_values = [] try: if len(axis) != 2: logging.warn("Wrong number of axis") return None filters = FltHelper.get_filters_clean_color_filters(filters) filters = FltHelper.apply_bin_size_to_filters(filters, dt) filtered_ds = get_filtered_dataset(src_destination, filters, gti_destination) if not DsHelper.is_events_dataset(filtered_ds) \ and not DsHelper.is_lightcurve_dataset(filtered_ds): logging.warn("Wrong dataset type") return None if DsHelper.is_events_dataset(filtered_ds): # Creates lightcurves by gti and joins in one logging.debug("Create lightcurve ....Event count: " + str(len(filtered_ds.tables["EVENTS"].columns["TIME"].values))) lc = get_lightcurve_from_dataset(filtered_ds, axis, bck_destination, filters, gti_destination, dt) filtered_ds = None # Dispose memory if lc: logging.debug("Result time: " + str(len(lc.time))) time_vals = lc.time count_rate = lc.countrate error_values = [] # TODO: Implement error values on Stingray #lc = None # Dispose memory elif DsHelper.is_lightcurve_dataset(filtered_ds): #If dataset is LIGHTCURVE type time_vals = filtered_ds.tables["RATE"].columns["TIME"].values count_rate = filtered_ds.tables["RATE"].columns["RATE"].values error_values = filtered_ds.tables["RATE"].columns["ERROR"].values except: logging.error(str(sys.exc_info())) # Preapares the result logging.debug("Result lightcurve .... " + str(len(time_vals))) result = push_to_results_array([], time_vals) result = push_to_results_array(result, count_rate) result = push_to_results_array(result, error_values) return result
def get_divided_lightcurve_ds(lc0_destination, lc1_destination): try: lc0_ds = DaveReader.get_file_dataset(lc0_destination) if not DsHelper.is_lightcurve_dataset(lc0_ds): logging.warn("Wrong dataset type for lc0") return "" count_rate_0 = np.array(lc0_ds.tables["RATE"].columns["RATE"].values) lc1_ds = DaveReader.get_file_dataset(lc1_destination) if not DsHelper.is_lightcurve_dataset(lc1_ds): logging.warn("Wrong dataset type for lc1") return "" count_rate_1 = np.array(lc1_ds.tables["RATE"].columns["RATE"].values) if count_rate_0.shape == count_rate_1.shape: ret_lc_ds = lc0_ds.clone(True) with np.errstate(all='ignore'): # Ignore divisions by 0 and others count_rate = np.nan_to_num(count_rate_0 / count_rate_1) count_rate[count_rate > BIG_NUMBER]=0 ret_lc_ds.tables["RATE"].columns["RATE"].clear() ret_lc_ds.tables["RATE"].columns["RATE"].add_values(count_rate) # TODO: Set error from lightcurve lc0_ds = None # Dispose memory lc1_ds = None # Dispose memory count_rate_1 = None # Dispose memory count_rate_0 = None # Dispose memory count_rate = None # Dispose memory new_cache_key = DsCache.get_key(lc0_destination + "|" + lc1_destination + "|ligthcurve") DsCache.add(new_cache_key, ret_lc_ds) # Adds new cached dataset for new key return new_cache_key else: logging.warn("Lightcurves have different shapes.") return None except: logging.error(str(sys.exc_info())) return ""
def save_to_intermediate_file(stingray_object, fname): """Save Stingray object to intermediate file.""" from stingray.lightcurve import Lightcurve from stingray.events import EventList from stingray.crossspectrum import Crossspectrum from hendrics.io import save_lcurve, save_events, save_pds if isinstance(stingray_object, Lightcurve): save_lcurve(stingray_object, fname) elif isinstance(stingray_object, EventList): save_events(stingray_object, fname) # This also work for Powerspectrum and AveragedCrosspowerspectrum, clearly elif isinstance(stingray_object, Crossspectrum): save_pds(stingray_object, fname) else: logging.error("save_to_intermediate_file: Unknown object type: %s" % type(stingray_object).__name__) return False return True
def get_joined_lightcurves_from_colors(src_destination, bck_destination, gti_destination, filters, axis, dt): if len(axis) != 2: logging.warn("Wrong number of axis") return None try: filters = FltHelper.apply_bin_size_to_filters(filters, dt) # Prepares SRC_LC clean_filters = FltHelper.get_filters_clean_color_filters(filters) filtered_ds = get_filtered_dataset(src_destination, clean_filters, gti_destination) # Creates src lightcurve applying bck and gtis src_lc = get_lightcurve_from_dataset(filtered_ds, axis, bck_destination, clean_filters, gti_destination, dt) if not src_lc: logging.warn("Cant create lc_src") return None count_column_name = "PI" color_keys = FltHelper.get_color_keys_from_filters(filters) filtered_datasets = split_dataset_with_color_filters(src_destination, filters, color_keys, count_column_name, gti_destination) color_axis = get_color_axis_for_ds() # Creates lightcurves array applying bck and gtis from each color logging.debug("Create color lightcurves ....") lightcurves = get_lightcurves_from_datasets_array(filtered_datasets, color_keys, count_column_name, color_axis, bck_destination, filters, gti_destination, dt) filtered_datasets = None # Dispose memory if len(lightcurves) == 2: # Preapares the result logging.debug("Result joined lightcurves ....") result = push_to_results_array([], src_lc.countrate) result = push_divided_values_to_results_array(result, lightcurves[0].countrate, lightcurves[1].countrate) result = push_to_results_array(result, src_lc.time) return result except: logging.error(str(sys.exc_info())) return None
def get_stingray_object(destination, time_offset=0): if not destination: return None filename = os.path.splitext(destination)[0] file_extension = magic.from_file(destination) logging.debug("File extension: %s" % file_extension) if file_extension.find("FITS") == 0: # Opening Fits hdulist = fits.open(destination, memmap=True) if 'EVENTS' in hdulist: # If EVENTS extension found, consider the Fits as EVENTS Fits fits_data = load_events_and_gtis(destination, additional_columns=['PI', "PHA"], gtistring=CONFIG.GTI_STRING, hduname='EVENTS', column=CONFIG.TIME_COLUMN) return substract_tstart_from_events(fits_data, time_offset) elif 'RATE' in hdulist: # If RATE extension found, consider the Fits as LIGHTCURVE Fits # Reads the lightcurve with hendrics outfile = lcurve_from_fits(destination, gtistring=get_hdu_string_from_hdulist( CONFIG.GTI_STRING, hdulist), timecolumn=CONFIG.TIME_COLUMN, ratecolumn=None, ratehdu=1, fracexp_limit=CONFIG.FRACEXP_LIMIT)[0] return substract_tstart_from_lcurve(load_lcurve(outfile), time_offset) else: logging.error("Unsupported FITS type!") else: logging.error("Unknown file extension: %s" % file_extension) return None
def apply_filters(self, filters): if not filters or not len(filters): return self filtered_dataset = self.clone() time_filter = FltHelper.get_time_filter(filters) # Firts filter by time for reducing arrays length if time_filter: filtered_dataset = self.apply_time_filter(time_filter, time_filter["table"]) for filter in filters: table_id = filter["table"] if table_id not in ["EVENTS", "RATE"] or filter["column"] != CONFIG.TIME_COLUMN: # Exclude time filter if table_id in filtered_dataset.tables: filtered_dataset.tables[table_id] = filtered_dataset.tables[table_id].apply_filter(filter) else: logging.error("dataset.apply_filters wrong table_id: %s" % table_id) return filtered_dataset
def apply_filters(self, filters): if not filters or not len(filters): return self filtered_dataset = self.clone() for filter in filters: table_id = filter["table"] if table_id in filtered_dataset.tables: if table_id == "EVENTS" and filter["column"] == "TIME": filtered_dataset = self.apply_time_filter(filter) else: filtered_dataset.tables[ table_id] = filtered_dataset.tables[ table_id].apply_filter(filter) else: logging.error("dataset.apply_filters wrong table_id: %s" % table_id) return filtered_dataset
def get_joined_lightcurves(lc0_destination, lc1_destination, filters, axis, dt): try: if len(axis) != 2: logging.warn("Wrong number of axis") return None filters = FltHelper.get_filters_clean_color_filters(filters) filters = FltHelper.apply_bin_size_to_filters(filters, dt) lc0_ds = get_filtered_dataset(lc0_destination, filters) if not DsHelper.is_lightcurve_dataset(lc0_ds): logging.warn("Wrong dataset type for lc0") return None lc1_ds = get_filtered_dataset(lc1_destination, filters) if not DsHelper.is_lightcurve_dataset(lc1_ds): logging.warn("Wrong dataset type for lc1") return None # Problaby here we can use a stronger checking if len(lc0_ds.tables["RATE"].columns["TIME"].values) == len(lc1_ds.tables["RATE"].columns["TIME"].values): # Preapares the result logging.debug("Result joined lightcurves ....") result = push_to_results_array([], lc0_ds.tables["RATE"].columns["RATE"].values) result = push_to_results_array(result, lc1_ds.tables["RATE"].columns["RATE"].values) result = push_to_results_array(result, lc0_ds.tables["RATE"].columns["TIME"].values) return result else: logging.warn("Lightcurves have different durations.") return None except: logging.error(str(sys.exc_info())) return None
def apply_filter(self, filter): column_name = filter["column"] if column_name not in self.columns: logging.error("table.apply_filter wrong column: %s" % column_name) return self if filter["from"] > filter["to"]: logging.error("table.apply_filter wrong from-to: %s" % column_name) return self filtered_table = Table(self.id) for tmp_column_name in self.columns: filtered_table.columns[tmp_column_name] = Column(tmp_column_name) column = self.columns[column_name] values = np.array(column.values) filtered_indexes = np.where((values >= filter["from"]) & (values <= filter["to"]))[0] for column_name in self.columns: col_values, col_error_values = self.columns[column_name].get_values(filtered_indexes) filtered_table.columns[column_name].add_values(col_values, col_error_values) return filtered_table
def get_stingray_object(destination, time_offset=0): if not destination: return None filename = os.path.splitext(destination)[0] file_extension = magic.from_file(destination) logging.debug("File extension: %s" % file_extension) if file_extension.find("FITS") == 0: # Opening Fits hdulist = fits.open(destination, memmap=True) if 'EVENTS' in hdulist: # If EVENTS extension found, consider the Fits as EVENTS Fits fits_data = load_events_and_gtis(destination, additional_columns=['PI', "PHA"], gtistring=CONFIG.GTI_STRING, hduname='EVENTS', column=CONFIG.TIME_COLUMN) return substract_tstart_from_events(fits_data, time_offset) elif 'RATE' in hdulist: # If RATE extension found, consider the Fits as LIGHTCURVE Fits # Reads the lightcurve with hendrics outfile = lcurve_from_fits(destination, gtistring=get_hdu_string_from_hdulist(CONFIG.GTI_STRING, hdulist), timecolumn=CONFIG.TIME_COLUMN, ratecolumn=None, ratehdu=1, fracexp_limit=CONFIG.FRACEXP_LIMIT)[0] return substract_tstart_from_lcurve(load_lcurve(outfile), time_offset) else: logging.error("Unsupported FITS type!") else: logging.error("Unknown file extension: %s" % file_extension) return None
def get_destination(filename, target): if not filename: logging.error("No filename or cache key setted for filename %s" % filename) return None if not SessionHelper.is_file_uploaded(filename): if not DsCache.contains(filename): if not FileUtils.file_exist(target, filename): logging.error("Filename not uploaded or not found in cache for filename %s" % filename) return None destination = FileUtils.get_destination(target, filename) if not FileUtils.is_valid_file(destination): if not DsCache.contains(filename): logging.error("Invalid file or not found in cache filename %s" % filename) return None else: destination = filename # Filename represents only a joined dataset key, not a real file return destination
def get_destination(filename, target): if not filename: logging.error("No filename or cache key setted for filename %s" % filename) return None if not SessionHelper.is_file_uploaded(filename): if not DsCache.contains(filename): logging.error( "Filename not uploaded or not found in cache for filename %s" % filename) return None destination = FileUtils.get_destination(target, filename) if not FileUtils.is_valid_file(destination): if not DsCache.contains(filename): logging.error("Invalid file or not found in cache filename %s" % filename) return None else: destination = filename # Filename represents only a joined dataset key, not a real file return destination
def bulk_analisys(filenames, plot_configs, outdir): try: results = dict() results["outdir"] = outdir results["plot_configs"] = [] # For each plot config do a bulk analisys for plot_config in plot_configs: plot_config_outdir = "/".join([outdir, plot_config["id"]]) dt = plot_config["dt"] filters = FltHelper.get_filters_clean_color_filters( plot_config["filters"]) filters = FltHelper.apply_bin_size_to_filters(filters, dt) if "class" in plot_config: if plot_config["class"] == "LcPlot": args = ['--outdir', plot_config_outdir] args.extend(['--bintime', str(dt)]) args = add_filter_to_args(args, filters, CONFIG.TIME_COLUMN, '--safe-interval') args = add_filter_to_args(args, filters, "PI", '--pi-interval') args = add_filter_to_args(args, filters, "E", '--e-interval') #args = add_filter_to_args(args, filters, "PHA", '--pha-interval') #args = add_filter_to_args(args, filters, "RATE", '--rate-interval') args.extend(filenames) logging.debug("Calling MPlcurve, args: " + str(args)) MPlcurve(args) push_plotconfig_results(results["plot_configs"], plot_config["id"], plot_config_outdir) elif plot_config["class"] == "PDSPlot": logging.error( "PDSPlot not supported yet, still work in progress!!") ''' TO MANY QUESTIONS OPENED FOR IMPLEMENT PDS ON BULK ANALYSIS args = ['--outdir', plot_config_outdir] args.extend(['--kind', "PDS"]) args.extend(['--bintime', str(dt)]) # Normalization: MaltPyn supports Leahy or rms, now DAVE sends leahy, frac, abs or none args.extend(['--norm', str(plot_config["norm"])]) # Rebin: is this args.extend(['--rebin', str(plot_config["norm"])]) args.extend(filenames) MPfspec(args) push_plotconfig_results(results["plot_configs"], plot_config["id"], plot_config_outdir) ''' else: logging.error("PlotConfig.class not supported!!") else: logging.error("PlotConfig has no class key!!") return results except: logging.error(ExHelper.getException('bulk_analisys')) return None
def get_file_dataset(destination, time_offset=0): dataset = None cache_key = "" try: if destination: cache_key = get_cache_key_for_destination(destination, time_offset) if DsCache.contains(cache_key): logging.debug("get_file_dataset: returned cached dataset, cache_key: " + str(cache_key)) return DsCache.get(cache_key), cache_key logging.debug("get_file_dataset: reading destination: " + str(destination)) filename = os.path.splitext(destination)[0] file_extension_from_file = os.path.splitext(destination)[1] file_extension = magic.from_file(destination) logging.debug("File extension: %s" % file_extension) if file_extension.find("ASCII") == 0: table_id = "EVENTS" header_names = [CONFIG.TIME_COLUMN, "PHA", "Color1", "Color2"] dataset = get_txt_dataset(destination, table_id, header_names) table = dataset.tables[table_id] table.add_columns(["AMPLITUDE"]) numValues = len(table.columns[CONFIG.TIME_COLUMN].values) random_values = np.random.uniform(-1, 1, size=numValues) table.columns["AMPLITUDE"].values = random_values elif file_extension.find("FITS") == 0 \ or file_extension.find("gzip") > -1: # Opening Fits hdulist = fits.open(destination, memmap=True) if get_hdu_string_from_hdulist(CONFIG.EVENTS_STRING, hdulist) != "": # If EVENTS extension found, consider the Fits as EVENTS Fits dataset = get_events_fits_dataset_with_stingray(destination, hdulist, dsId='FITS', hduname=get_hdu_string_from_hdulist(CONFIG.EVENTS_STRING, hdulist), column=CONFIG.TIME_COLUMN, gtistring=CONFIG.GTI_STRING, extra_colums=['PI', "PHA"], time_offset=time_offset) elif 'RATE' in hdulist: # If RATE extension found, consider the Fits as LIGHTCURVE Fits dataset = get_lightcurve_fits_dataset_with_stingray(destination, hdulist, hduname='RATE', column=CONFIG.TIME_COLUMN, gtistring=CONFIG.GTI_STRING, time_offset=time_offset) elif 'EBOUNDS' in hdulist: # If EBOUNDS extension found, consider the Fits as RMF Fits dataset = get_fits_dataset(hdulist, "RMF", ["EBOUNDS"]) elif get_hdu_string_from_hdulist(CONFIG.GTI_STRING, hdulist) != "": # If not EVENTS or RATE extension found, check if is GTI Fits dataset = get_gti_fits_dataset_with_stingray(hdulist,gtistring=CONFIG.GTI_STRING, time_offset=time_offset) else: logging.warn("Unsupported FITS type! Any table found: " + CONFIG.EVENTS_STRING + ", RATE, EBOUNDS or " + CONFIG.GTI_STRING) elif file_extension == "data" and (file_extension_from_file in [".p", ".nc"]): # If file is pickle object, tries to parse it as dataset dataset = load_dataset_from_intermediate_file(destination) else: logging.warn("Unknown file extension: " + str(file_extension) + " , " + str(file_extension_from_file)) if dataset: DsCache.add(cache_key, dataset) logging.debug("get_file_dataset, dataset added to cache, cache_key: " + str(cache_key)) else: logging.error("get_file_dataset: Destination is empty") except: logging.error(ExHelper.getException('get_file_dataset')) return dataset, cache_key
def get_file_dataset(destination, time_offset=0): dataset = None cache_key = "" try: if destination: cache_key = get_cache_key_for_destination(destination, time_offset) if DsCache.contains(cache_key): logging.debug( "get_file_dataset: returned cached dataset, cache_key: " + str(cache_key)) return DsCache.get(cache_key), cache_key logging.debug("get_file_dataset: reading destination: " + str(destination)) filename = os.path.splitext(destination)[0] file_extension_from_file = os.path.splitext(destination)[1] file_extension = magic.from_file(destination) logging.debug("File extension: %s" % file_extension) if file_extension.find("ASCII") == 0: table_id = "EVENTS" header_names = [CONFIG.TIME_COLUMN, "PHA", "Color1", "Color2"] dataset = get_txt_dataset(destination, table_id, header_names) table = dataset.tables[table_id] table.add_columns(["AMPLITUDE"]) numValues = len(table.columns[CONFIG.TIME_COLUMN].values) random_values = np.random.uniform(-1, 1, size=numValues) table.columns["AMPLITUDE"].values = random_values elif file_extension.find("FITS") == 0 \ or file_extension.find("gzip") > -1: # Opening Fits hdulist = fits.open(destination, memmap=True) if get_hdu_string_from_hdulist(CONFIG.EVENTS_STRING, hdulist) != "": # If EVENTS extension found, consider the Fits as EVENTS Fits dataset = get_events_fits_dataset_with_stingray( destination, hdulist, dsId='FITS', hduname=get_hdu_string_from_hdulist( CONFIG.EVENTS_STRING, hdulist), column=CONFIG.TIME_COLUMN, gtistring=CONFIG.GTI_STRING, extra_colums=['PI', "PHA"], time_offset=time_offset) elif 'RATE' in hdulist: # If RATE extension found, consider the Fits as LIGHTCURVE Fits dataset = get_lightcurve_fits_dataset_with_stingray( destination, hdulist, hduname='RATE', column=CONFIG.TIME_COLUMN, gtistring=CONFIG.GTI_STRING, time_offset=time_offset) elif 'EBOUNDS' in hdulist: # If EBOUNDS extension found, consider the Fits as RMF Fits dataset = get_fits_dataset(hdulist, "RMF", ["EBOUNDS"]) elif get_hdu_string_from_hdulist(CONFIG.GTI_STRING, hdulist) != "": # If not EVENTS or RATE extension found, check if is GTI Fits dataset = get_gti_fits_dataset_with_stingray( hdulist, gtistring=CONFIG.GTI_STRING, time_offset=time_offset) else: logging.warn("Unsupported FITS type! Any table found: " + CONFIG.EVENTS_STRING + ", RATE, EBOUNDS or " + CONFIG.GTI_STRING) elif file_extension == "data" and (file_extension_from_file in [".p", ".nc"]): # If file is pickle object, tries to parse it as dataset dataset = load_dataset_from_intermediate_file(destination) else: logging.warn("Unknown file extension: " + str(file_extension) + " , " + str(file_extension_from_file)) if dataset: DsCache.add(cache_key, dataset) logging.debug( "get_file_dataset, dataset added to cache, cache_key: " + str(cache_key)) else: logging.error("get_file_dataset: Destination is empty") except: logging.error(ExHelper.getException('get_file_dataset')) return dataset, cache_key