def perform_settlement_clustering(self): self.logger.debug("Performing Level 1 clustering") settlement_clustering = Regions() self.regions = settlement_clustering.detect_regions(self.dataset) self.base_stations = settlement_clustering.locate_base_stations_proximity( )
def _process_partial_data(self, partial_requested_data): ''' This function is always executed after managing the response of each URL of this Data Source. The data should be processed from the external structure to the form of a DataFrame. Parameters ---------- partial_requested_data : csv it is the requested CSV of one URL. Returns ------- pd.DataFrame a DataFrame with daily [Date] as row indexer and [Region, Data Item] as column multiindexer. ''' region_codes_ine = Regions._get_property(self.regions, self.__class__.REGION_REPRESENTATION) codesine_regions_dict = dict(zip(region_codes_ine,self.regions)) df = partial_requested_data[(partial_requested_data['cod_ine_ambito'].isin(region_codes_ine)) & (partial_requested_data['nombre_sexo'] == 'todos') & (partial_requested_data['nombre_gedad'] == 'todos')] df.cod_ine_ambito = df.cod_ine_ambito.astype(int).astype(str).apply(lambda x: x.zfill(2)).replace(codesine_regions_dict) df.rename(columns={'cod_ine_ambito':'Region'},inplace=True) df.drop(['ambito','cod_ambito','nombre_ambito','cod_sexo','nombre_sexo','cod_gedad','nombre_gedad'],axis='columns',errors='ignore',inplace=True) df = df[['Region','fecha_defuncion']+self.data_items] df.set_index(['Region'],inplace=True) df = df.pivot_table(index='fecha_defuncion',columns='Region').swaplevel(i=0,j=1,axis='columns') df.columns.rename("Item",level=1,inplace=True) df.set_index(pd.to_datetime(df.index, format="%Y-%m-%d"),inplace=True) return df
def __init__(self, elements_layout, bounds_domain, curvature=0., info=None, name=None, pre_No=None): SW = (bounds_domain[0][0], bounds_domain[1][0]) SE = (bounds_domain[0][1], bounds_domain[1][0]) NW = (bounds_domain[0][0], bounds_domain[1][1]) NE = (bounds_domain[0][1], bounds_domain[1][1]) regions = { 'R_crazy': (SW, SE, NW, NE), } region_type = {} boundaries_dict = { 'South': ('0S', ), 'North': ('0N', ), 'West': ('0W', ), 'East': ('0E', ) } REGIONS = (regions, region_type, boundaries_dict) R = Regions(REGIONS) super().__init__(2, R, elements_layout, info, name, pre_No) if curvature > 0.316: warnings.warn( " <MESH> <CRAZY> : curvature={} > 0.316 (normally curvature in [0, 0.3]), grid may go outside domain somewhere!" .format(curvature)) self._curvature = curvature (self._x_bounds, self._y_bounds) = bounds_domain self._bounds_domain = bounds_domain
def __get_stations_by_regions(self): """ Gets the aemet stations per region Returns ------- dict {str : str} a dictionary with instance regions as keys, and a string with the aemet stations separated by commas as values. Notes ----- * It is used for completing 'idema' argument in the AEMET API function (https://opendata.aemet.es/dist/index.html?#!/valores-climatologicos/Climatolog%C3%ADas_diarias) """ aemet_stations = Regions._get_property( self.regions, self.__class__.REGION_REPRESENTATION) str_aemet_stations = {} pos_region = 0 for stations in aemet_stations: str_aemet_stations[self.regions[pos_region]] = '' for station in stations: str_aemet_stations[ self.regions[pos_region]] = str_aemet_stations[ self.regions[pos_region]] + "," + station str_aemet_stations[self.regions[pos_region]] = str_aemet_stations[ self.regions[pos_region]][1:] pos_region = pos_region + 1 return str_aemet_stations
def from_regions(cls, regions, **kwargs): """Create region geom from list of regions The regions are combined with union to a compound region. Parameters ---------- regions : list of `~regions.SkyRegion` or str Regions **kwargs: dict Keyword arguments forwarded to `RegionGeom` Returns ------- geom : `RegionGeom` Region map geometry """ if isinstance(regions, str): regions = Regions.parse(data=regions, format="ds9") elif isinstance(regions, SkyRegion): regions = [regions] elif isinstance(regions, SkyCoord): regions = [PointSkyRegion(center=regions)] if regions: regions = regions_to_compound_region(regions) return cls(region=regions, **kwargs)
def compound_region_to_regions(region): """Create list of regions from compound regions. Parameters ---------- region : `~regions.CompoundSkyRegion` or `~regions.SkyRegion` Compound sky region Returns ------- regions : `~regions.Regions` List of regions. """ regions = Regions([]) if isinstance(region, CompoundSkyRegion): if region.operator is operator.or_: regions_1 = compound_region_to_regions(region.region1) regions.extend(regions_1) regions_2 = compound_region_to_regions(region.region2) regions.extend(regions_2) else: raise ValueError("Only union operator supported") else: return Regions([region]) return regions
def get_dataframe_by_definition(def_dict): """Return dataframe corresponding to definition dict.""" file_path = os.path.join(def_dict['folder'], def_dict['filename']) if 'anchor' in def_dict.keys(): gen = read_sheet(file_path, def_dict['sheet'], def_dict['anchor']) else: gen = read_sheet(file_path, def_dict['sheet']) try: df = get_dataframe(gen)[Regions.names()] except: raise ValueError(file_path) df.insert(0, 'varname', def_dict['varname']) return df
def get_dataframe_by_definition(def_dict): """Return dataframe corresponding to definition dict.""" file_path = os.path.join(def_dict['folder'], def_dict['filename']) if 'anchor' in def_dict.keys(): gen = read_sheet(file_path, def_dict['sheet'], def_dict['anchor']) else: gen = read_sheet(file_path, def_dict['sheet']) try: df = get_dataframe(gen)[Regions.names()] except: raise ValueError(file_path) df.insert(0, 'varname', def_dict['varname']) return df
def test_compound_region_center(): regions_ds9 = ("galactic;" "circle(1,1,0.1);" "circle(-1,1,0.1);" "circle(1,-1,0.1);" "circle(-1,-1,0.1);") regions = Regions.parse(regions_ds9, format="ds9") region = regions_to_compound_region(regions) center = compound_region_center(region) assert_allclose(center.galactic.l.wrap_at("180d"), 0 * u.deg, atol=1e-6) assert_allclose(center.galactic.b, 0 * u.deg, atol=1e-6)
def _to_region_table(self): """Export region to a FITS region table.""" if self.region is None: raise ValueError("Region definition required.") # TODO: make this a to_hdulist() method region_list = compound_region_to_regions(self.region) pixel_region_list = [] for reg in region_list: pixel_region_list.append(reg.to_pixel(self.wcs)) table = Regions(pixel_region_list).serialize(format="fits") header = WcsGeom(wcs=self.wcs, npix=self.wcs.array_shape).to_header() table.meta.update(header) return table
def from_hdulist(cls, hdulist, format="ogip", hdu=None): """Read region table and convert it to region list. Parameters ---------- hdulist : `~astropy.io.fits.HDUList` HDU list format : {"ogip", "ogip-arf", "gadf"} HDU format Returns ------- geom : `RegionGeom` Region map geometry """ region_hdu = "REGION" if format == "gadf" and hdu: region_hdu = hdu + "_" + region_hdu if region_hdu in hdulist: region_table = Table.read(hdulist[region_hdu]) wcs = WcsGeom.from_header(region_table.meta).wcs regions = [] for reg in Regions.parse(data=region_table, format="fits"): # TODO: remove workaround once regions issue with fits serialization is sorted out # see https://github.com/astropy/regions/issues/400 reg.meta["include"] = True regions.append(reg.to_sky(wcs)) region = regions_to_compound_region(regions) else: region, wcs = None, None if format == "ogip": hdu_bands = "EBOUNDS" elif format == "ogip-arf": hdu_bands = "SPECRESP" elif format == "gadf": hdu_bands = hdu + "_BANDS" else: raise ValueError(f"Unknown format {format}") axes = MapAxes.from_table_hdu(hdulist[hdu_bands], format=format) return cls(region=region, wcs=wcs, axes=axes)
def catalog_to_regions(catalog, ra='RA', dec='DEC', majax='Maj', minax='Min', PA='PA'): ''' Convert catalog to a list of regions Keyword arguments: catalog -- Input catalog ra, dec, majax, minax, PA -- Column names of containing required variables ''' regions = Regions([ EllipseSkyRegion(center=SkyCoord(source[ra], source[dec], unit='deg'), height=source[majax] * u.deg, width=source[minax] * u.deg, angle=source[PA] * u.deg) for source in catalog ]) return regions
def __get_regions_by_stations(self): """ Gets the region per aemet station. Returns ------- dict {str : str} a dictionary with aemet stations as keys, and instance regions as values. """ aemet_stations = Regions._get_property( self.regions, self.__class__.REGION_REPRESENTATION) region_by_aemet_station = {} pos_region = 0 for stations in aemet_stations: for station in stations: region_by_aemet_station[station] = self.regions[pos_region] pos_region = pos_region + 1 return region_by_aemet_station
def inject_references(obj): if isinstance(obj, dict): if 'file' in obj: filename = pjoin(src_path, obj['file']) if filename.endswith('csv'): df = pd.read_csv(filename) obj.pop('file') obj.update(df.to_dict(orient='list')) elif filename.endswith('.png'): return base64.b64encode(open(filename, 'rb').read()) elif filename.endswith('xml'): with open(filename, 'rb') as fid: payload = fid.read() return payload elif filename.endswith('reg'): return Regions.read(filename).serialize(format='ds9') elif filename.endswith('h5') or filename.endswith('hdf5'): payload = Table.read(filename).to_pandas().to_dict( orient='list') return payload else: raise NotImplementedError( f'{filename}: Only CSV, PNG, xml, reg, and hdf5 files currently supported for extending individual objects' ) for k, v in obj.items(): obj[k] = inject_references(v) return obj elif isinstance(obj, str) and obj.startswith('='): try: return references[obj[1:]] except KeyError: print( f'\nReference {obj[1:]} not found while posting to {endpoint}; skipping' ) raise elif isinstance(obj, list): return [inject_references(item) for item in obj] else: return obj
def _process_partial_data(self, partial_requested_data): ''' This function is always executed after managing the response of each URL of this Data Source. The data should be processed from the external structure to the form of a DataFrame. Parameters ---------- partial_requested_data : json it is the requested JSON of one URL. Returns ------- pd.DataFrame a DataFrame with [Region] as row indexer and [Data Item] as column indexer. ''' if self.__class__.DATA_ITEMS_INFO[self.data_items[ self.processed_urls]]['funcion'] == 'DATOS_TABLA': df = pd.json_normalize(partial_requested_data, 'Data', ['Nombre']) df.Nombre = df.Nombre.replace( dict( zip( Regions._get_property( self.regions, self.__class__.REGION_REPRESENTATION), self.regions)), regex=True ) # some regions contains commas, fix it through configuration file # json field 'Nombre' is splitted into Region, SubItem and Item columns # SubItem is a subdivision of the seeked Item df[['Region', 'SubItem', 'Item']] = df['Nombre'].str.split(", ", n=2, expand=True) # if in Region we find the word 'sexo', we should interchange Region and Subitem Columns if any(df.Region.str.contains(pat='sexo', case=False, regex=True)): df[['Region', 'SubItem']] = df[['SubItem', 'Region']]
def load_static_regions_from_file(self, region_file, region_format='ds9', prefix='region', max_num_regions=20, **kwargs): """Load regions defined in the given file. See :ref:`regions:regions_io` for supported file formats. Parameters ---------- region_file : str Path to region file. region_format : {'crtf', 'ds9', 'fits'} See :meth:`regions.Regions.get_formats`. prefix : str Prefix for the Subset names generated by loaded regions. Names will have the format of ``<prefix>_<i>``, where ``i`` is the index in the load order. max_num_regions : int Maximum number of regions to read from the file, starting from top of the file, invalid regions included. kwargs : dict See :meth:`load_static_regions`. Returns ------- bad_regions : dict See :meth:`load_static_regions`. """ from regions import Regions raw_regs = Regions.read(region_file, format=region_format) my_regions = dict([(f'{prefix}_{i}', reg) for i, reg in enumerate(raw_regs[:max_num_regions])]) return self.load_static_regions(my_regions, **kwargs)
def test_gcnevents_observations(driver, user, super_admin_token, upload_data_token, view_only_token, ztf_camera): datafile = f'{os.path.dirname(__file__)}/../data/GW190425_initial.xml' with open(datafile, 'rb') as fid: payload = fid.read() data = {'xml': payload} status, data = api('POST', 'gcn_event', data=data, token=super_admin_token) assert status == 200 assert data['status'] == 'success' telescope_name = str(uuid.uuid4()) status, data = api( 'POST', 'telescope', data={ 'name': telescope_name, 'nickname': telescope_name, 'lat': 0.0, 'lon': 0.0, 'elevation': 0.0, 'diameter': 10.0, }, token=super_admin_token, ) assert status == 200 assert data['status'] == 'success' telescope_id = data['data']['id'] fielddatafile = f'{os.path.dirname(__file__)}/../../../data/ZTF_Fields.csv' regionsdatafile = f'{os.path.dirname(__file__)}/../../../data/ZTF_Region.reg' instrument_name = str(uuid.uuid4()) status, data = api( 'POST', 'instrument', data={ 'name': instrument_name, 'type': 'imager', 'band': 'Optical', 'filters': ['ztfr'], 'telescope_id': telescope_id, 'field_data': pd.read_csv(fielddatafile)[:5].to_dict(orient='list'), 'field_region': Regions.read(regionsdatafile).serialize(format='ds9'), }, token=super_admin_token, ) assert status == 200 assert data['status'] == 'success' instrument_id = data['data']['id'] params = {'includeGeoJSON': True} # wait for the fields to populate nretries = 0 fields_loaded = False while not fields_loaded and nretries < 5: try: status, data = api( 'GET', f'instrument/{instrument_id}', params=params, token=super_admin_token, ) assert status == 200 assert data['status'] == 'success' assert data['data']['band'] == 'NIR' assert len(data['data']['fields']) == 5 fields_loaded = True except AssertionError: nretries = nretries + 1 time.sleep(3) datafile = f'{os.path.dirname(__file__)}/../../../data/sample_observation_data.csv' data = { 'telescopeName': telescope_name, 'instrumentName': instrument_name, 'observationData': pd.read_csv(datafile).to_dict(orient='list'), } status, data = api('POST', 'observation', data=data, token=super_admin_token) assert status == 200 assert data['status'] == 'success' # wait for the executed observations to populate nretries = 0 observations_loaded = False while not observations_loaded and nretries < 5: try: status, data = api('GET', 'observation', params=data, token=super_admin_token) assert status == 200 data = data["data"] assert len(data['observations']) == 10 observations_loaded = True except AssertionError: nretries = nretries + 1 time.sleep(3) driver.get(f'/become_user/{user.id}') driver.get('/gcn_events/2019-04-25T08:18:05') driver.wait_for_xpath('//*[text()="190425 08:18:05"]') driver.wait_for_xpath('//*[text()="LVC"]') driver.wait_for_xpath('//*[text()="BNS"]') # test modify sources form driver.wait_for_xpath('//*[@id="root_startDate"]').send_keys('04/24/2019') driver.wait_for_xpath('//*[@id="root_startDate"]').send_keys(Keys.TAB) driver.wait_for_xpath('//*[@id="root_startDate"]').send_keys('01:01') driver.wait_for_xpath('//*[@id="root_startDate"]').send_keys('P') driver.wait_for_xpath('//*[@id="root_endDate"]').send_keys('04/30/2019') driver.wait_for_xpath('//*[@id="root_endDate"]').send_keys(Keys.TAB) driver.wait_for_xpath('//*[@id="root_endDate"]').send_keys('01:01') driver.wait_for_xpath('//*[@id="root_endDate"]').send_keys('P') driver.wait_for_xpath('//*[@id="root_localizationCumprob"]').clear() driver.wait_for_xpath('//*[@id="root_localizationCumprob"]').send_keys( '1.01') driver.wait_for_xpath('//*[@id="root_localizationName"]') driver.click_xpath('//*[@id="root_localizationName"]') driver.wait_for_xpath('//li[contains(text(), "bayestar.fits.gz")]') driver.click_xpath('//li[contains(text(), "bayestar.fits.gz")]') submit_button_xpath = ( '//div[@data-testid="gcnsource-selection-form"]//button[@type="submit"]' ) driver.wait_for_xpath(submit_button_xpath) driver.click_xpath(submit_button_xpath) # check that the executed observation table appears driver.wait_for_xpath('//*[text()="84434604"]') driver.wait_for_xpath('//*[text()="ztfr"]') driver.wait_for_xpath('//*[text()="1.57415"]') driver.wait_for_xpath('//*[text()="20.40705"]')
def test_token_user_post_get_instrument(super_admin_token): name = str(uuid.uuid4()) status, data = api( 'POST', 'telescope', data={ 'name': name, 'nickname': name, 'lat': 0.0, 'lon': 0.0, 'elevation': 0.0, 'diameter': 10.0, }, token=super_admin_token, ) assert status == 200 assert data['status'] == 'success' telescope_id = data['data']['id'] fielddatafile = f'{os.path.dirname(__file__)}/../../../data/ZTF_Fields.csv' regionsdatafile = f'{os.path.dirname(__file__)}/../../../data/ZTF_Region.reg' instrument_name = str(uuid.uuid4()) status, data = api( 'POST', 'instrument', data={ 'name': instrument_name, 'type': 'imager', 'band': 'NIR', 'filters': ['f110w'], 'telescope_id': telescope_id, 'field_data': pd.read_csv(fielddatafile)[:5].to_dict(orient='list'), 'field_region': Regions.read(regionsdatafile).serialize(format='ds9'), }, token=super_admin_token, ) assert status == 200 assert data['status'] == 'success' instrument_id = data['data']['id'] params = {'includeGeoJSON': True} # wait for the fields to populate nretries = 0 fields_loaded = False while not fields_loaded and nretries < 5: try: status, data = api( 'GET', f'instrument/{instrument_id}', params=params, token=super_admin_token, ) assert status == 200 assert data['status'] == 'success' assert data['data']['band'] == 'NIR' assert len(data['data']['fields']) == 5 fields_loaded = True except AssertionError: nretries = nretries + 1 time.sleep(3) params = {'includeGeoJSON': True} instrument_id = data['data']['id'] status, data = api( 'GET', f'instrument/{instrument_id}', params=params, token=super_admin_token ) assert status == 200 assert data['status'] == 'success' assert data['data']['band'] == 'NIR' assert len(data['data']['fields']) == 5 assert any( [ d['field_id'] == 1 and d['contour']['features'][0]['geometry']['coordinates'][0][0] == [110.84791974982103, -87.01522999646508] for d in data['data']['fields'] ] ) params = {'includeGeoJSONSummary': True} instrument_id = data['data']['id'] status, data = api( 'GET', f'instrument/{instrument_id}', params=params, token=super_admin_token ) assert status == 200 assert data['status'] == 'success' assert data['data']['band'] == 'NIR' assert len(data['data']['fields']) == 5 assert any( [ d['field_id'] == 1 and d['contour_summary']['features'][0]['geometry']['coordinates'][0] == [1.0238351746164418, -89.93777511600825] for d in data['data']['fields'] ] )
from datetime import date import pandas as pd import os from xls_read import read_sheet, read_by_definition, yearmon from regions import Regions filter_region_name = Regions.filter_region_name reference_region_names = Regions.names() rf_name = Regions.rf_name() district_names = Regions.district_names() summable_regions = Regions.summable_regions() def get_dataframe(datapoints_stream): """Return dataframe corresponding to datapoints stream.""" list_of_dicts = [{'val':x[0], 'region':x[1], 'dates':x[2]} for x in datapoints_stream] df = pd.DataFrame(list_of_dicts) df = df.pivot(columns='region', values='val', index='dates')[reference_region_names] df.index = pd.DatetimeIndex(df.index) return df def get_dataframe_by_definition(def_dict): """Return dataframe corresponding to definition dict.""" file_path = os.path.join(def_dict['folder'], def_dict['filename']) if 'anchor' in def_dict.keys(): gen = read_sheet(file_path, def_dict['sheet'], def_dict['anchor']) else: gen = read_sheet(file_path, def_dict['sheet']) try: df = get_dataframe(gen)[Regions.names()] except:
dfs = import_xl_data() # this import is faster, but series is alphabetic # dfs = import_csv_data() # output 1: save all dataframes to xls by sheet - one df per sheet if '1' in jobs: to_xl_book(dfs, tag = 'by_sheet') # output 2: concat all dataframes to one xls sheet if '2' in jobs: r = pd.concat(dfs) to_xl_sheet(r, tag = 'one_page', sheet = "regions") # output 3: make Russia file (1 sheet) rf = Regions.rf_name() if '3' in jobs: # note: must have pandas 17 or higher for 'rename' df_rf = pd.concat([d[rf].rename(d['varname'][0]) for d in dfs], axis = 1) to_xl_sheet(df_rf, tag = 'rf', sheet = 'rf') # output 4: make fed districts file (num_var sheets) if '4' in jobs: cols = ['varname'] + [Regions.rf_name()] + Regions.district_names() dfs2 = [d.reindex(columns=cols) for d in dfs] to_xl_book(dfs2, tag = 'districts') # output 5: make regions only file (num_var sheets) if '5' in jobs: cols = ['varname'] + [Regions.rf_name()] + Regions.summable_regions() dfs3 = [d.reindex(columns=cols) for d in dfs]
def post(self): # See bottom of this file for redoc docstring -- moved it there so that # it could be made an f-string. data = self.get_json() telescope_id = data.get('telescope_id') telescope = Telescope.get_if_accessible_by( telescope_id, self.current_user, raise_if_none=True, mode="read" ) sensitivity_data = data.get("sensitivity_data", None) if sensitivity_data: filters = data.get("filters", []) if not set(sensitivity_data.keys()).issubset(filters): return self.error( 'Sensitivity_data filters must be a subset of the instrument filters' ) field_data = data.pop("field_data", None) field_region = data.pop("field_region", None) field_fov_type = data.pop("field_fov_type", None) field_fov_attributes = data.pop("field_fov_attributes", None) if (field_region is not None) and (field_fov_type is not None): return self.error('must supply only one of field_region or field_fov_type') if field_region is not None: regions = Regions.parse(field_region, format='ds9') data['region'] = regions.serialize(format='ds9') if field_fov_type is not None: if field_fov_attributes is None: return self.error( 'field_fov_attributes required if field_fov_type supplied' ) if not field_fov_type.lower() in ["circle", "rectangle"]: return self.error('field_fov_type must be circle or rectangle') if isinstance(field_fov_attributes, list): field_fov_attributes = [float(x) for x in field_fov_attributes] else: field_fov_attributes = [float(field_fov_attributes)] center = SkyCoord(0.0, 0.0, unit='deg', frame='icrs') if field_fov_type.lower() == "circle": if not len(field_fov_attributes) == 1: return self.error( 'If field_fov_type is circle, then should supply only radius for field_fov_attributes' ) radius = field_fov_attributes[0] regions = CircleSkyRegion(center=center, radius=radius * u.deg) elif field_fov_type.lower() == "rectangle": if not len(field_fov_attributes) == 2: return self.error( 'If field_fov_type is rectangle, then should supply width and height for field_fov_attributes' ) width, height = field_fov_attributes regions = RectangleSkyRegion( center=center, width=width * u.deg, height=height * u.deg ) data['region'] = regions.serialize(format='ds9') schema = Instrument.__schema__() try: instrument = schema.load(data) except ValidationError as exc: return self.error( 'Invalid/missing parameters: ' f'{exc.normalized_messages()}' ) existing_instrument = ( Instrument.query_records_accessible_by( self.current_user, ) .filter( Instrument.name == data.get('name'), Instrument.telescope_id == telescope_id, ) .first() ) if existing_instrument is None: instrument.telescope = telescope DBSession().add(instrument) DBSession().commit() else: instrument = existing_instrument if field_data is not None: if (field_region is None) and (field_fov_type is None): return self.error( 'field_region or field_fov_type is required with field_data' ) if type(field_data) is str: field_data = pd.read_table(StringIO(field_data), sep=",").to_dict( orient='list' ) if not {'ID', 'RA', 'Dec'}.issubset(field_data): return self.error("ID, RA, and Dec required in field_data.") log(f"Started generating fields for instrument {instrument.id}") # run async IOLoop.current().run_in_executor( None, lambda: add_tiles(instrument.id, instrument.name, regions, field_data), ) self.push_all(action="skyportal/REFRESH_INSTRUMENTS") return self.success(data={"id": instrument.id})
from regions import Regions z = Regions.summable_regions() print(z) for r in Regions.district_names(): w = Regions.region_by_district(r) print('\n', r) print(w) # passes now # assert set(Regions.region_by_district("Уральский федеральный округ")) == set (['Курганская область', 'Свердловская область', 'Ханты-Мансийский авт. округ - Югра', 'Ямало-Ненецкий авт. округ', 'Тюменская область без авт. округов', 'Челябинская область']) # must pass: assert set(Regions.region_by_district("Уральский федеральный округ")) == set([ 'Курганская область', 'Свердловская область', #'Ханты-Мансийский авт. округ - Югра', #'Ямало-Ненецкий авт. округ', #'Тюменская область без авт. округов', 'Тюменская область', 'Челябинская область' ]) print(Regions.region_by_district("Северо-Западный федеральный округ")) # Regions.region_by_district("Северо-Западный федеральный округ") must return: [ 'Республика Карелия', 'Республика Коми', #'Ненецкий авт. округ', #'Архангельская область без авт. округа',
def close(self): self.mainWindows.setCentralWidget(Regions(self.mainWindows))
def test_compound_region_center_single(): region = Regions.parse("galactic;circle(1,1,0.1)", format="ds9")[0] center = compound_region_center(region) assert_allclose(center.galactic.l.wrap_at("180d"), 1 * u.deg, atol=1e-6) assert_allclose(center.galactic.b, 1 * u.deg, atol=1e-6)
def put(self, instrument_id): """ --- description: Update instrument tags: - instruments parameters: - in: path name: instrument_id required: true schema: type: integer requestBody: content: application/json: schema: InstrumentNoID responses: 200: content: application/json: schema: Success 400: content: application/json: schema: Error """ data = self.get_json() data['id'] = int(instrument_id) # permission check instrument = Instrument.get_if_accessible_by( int(instrument_id), self.current_user, mode='update' ) if instrument is None: return self.error(f'Missing instrument with ID {instrument_id}') filters = instrument.filters sensitivity_data = data.get('sensitivity_data', None) if sensitivity_data: if not set(sensitivity_data.keys()).issubset(filters): return self.error( 'Filter names must be present in both sensitivity_data property and filters property' ) field_data = data.pop("field_data", None) field_region = data.pop("field_region", None) field_fov_type = data.pop("field_fov_type", None) field_fov_attributes = data.pop("field_fov_attributes", None) if (field_region is not None) and (field_fov_type is not None): return self.error('must supply only one of field_region or field_fov_type') if field_region is not None: regions = Regions.parse(field_region, format='ds9') data['region'] = regions.serialize(format='ds9') if field_fov_type is not None: if field_fov_attributes is None: return self.error( 'field_fov_attributes required if field_fov_type supplied' ) if not field_fov_type.lower() in ["circle", "rectangle"]: return self.error('field_fov_type must be circle or rectangle') if isinstance(field_fov_attributes, list): field_fov_attributes = [float(x) for x in field_fov_attributes] else: field_fov_attributes = [float(field_fov_attributes)] center = SkyCoord(0.0, 0.0, unit='deg', frame='icrs') if field_fov_type.lower() == "circle": if not len(field_fov_attributes) == 1: return self.error( 'If field_fov_type is circle, then should supply only radius for field_fov_attributes' ) radius = field_fov_attributes[0] regions = CircleSkyRegion(center=center, radius=radius * u.deg) elif field_fov_type.lower() == "rectangle": if not len(field_fov_attributes) == 2: return self.error( 'If field_fov_type is rectangle, then should supply width and height for field_fov_attributes' ) width, height = field_fov_attributes regions = RectangleSkyRegion( center=center, width=width * u.deg, height=height * u.deg ) data['region'] = regions.serialize(format='ds9') schema = Instrument.__schema__() try: schema.load(data, partial=True) except ValidationError as exc: return self.error( 'Invalid/missing parameters: ' f'{exc.normalized_messages()}' ) self.verify_and_commit() if field_data is not None: if (field_region is None) and (field_fov_type is None): return self.error( 'field_region or field_fov_type is required with field_data' ) if type(field_data) is str: field_data = pd.read_table(StringIO(field_data), sep=",").to_dict( orient='list' ) if not {'ID', 'RA', 'Dec'}.issubset(field_data): return self.error("ID, RA, and Dec required in field_data.") log(f"Started generating fields for instrument {instrument.id}") # run async IOLoop.current().run_in_executor( None, lambda: add_tiles(instrument.id, instrument.name, regions, field_data), ) self.push_all(action="skyportal/REFRESH_INSTRUMENTS") return self.success()
def test_observationplan_request(driver, user, super_admin_token, public_group): datafile = f'{os.path.dirname(__file__)}/../data/GW190425_initial.xml' with open(datafile, 'rb') as fid: payload = fid.read() data = {'xml': payload} status, data = api('POST', 'gcn_event', data=data, token=super_admin_token) assert status == 200 assert data['status'] == 'success' telescope_name = str(uuid.uuid4()) status, data = api( 'POST', 'telescope', data={ 'name': telescope_name, 'nickname': telescope_name, 'lat': 0.0, 'lon': 0.0, 'elevation': 0.0, 'diameter': 10.0, }, token=super_admin_token, ) assert status == 200 assert data['status'] == 'success' telescope_id = data['data']['id'] fielddatafile = f'{os.path.dirname(__file__)}/../../../data/ZTF_Fields.csv' regionsdatafile = f'{os.path.dirname(__file__)}/../../../data/ZTF_Region.reg' instrument_name = str(uuid.uuid4()) status, data = api( 'POST', 'instrument', data={ 'name': instrument_name, 'type': 'imager', 'band': 'NIR', 'filters': ['f110w'], 'telescope_id': telescope_id, "api_classname_obsplan": "ZTFMMAAPI", 'field_data': pd.read_csv(fielddatafile)[:5].to_dict(orient='list'), 'field_region': Regions.read(regionsdatafile).serialize(format='ds9'), }, token=super_admin_token, ) assert status == 200 assert data['status'] == 'success' instrument_id = data['data']['id'] params = {'includeGeoJSON': True} # wait for the fields to populate nretries = 0 fields_loaded = False while not fields_loaded and nretries < 5: try: status, data = api( 'GET', f'instrument/{instrument_id}', token=super_admin_token, params=params, ) assert status == 200 assert data['status'] == 'success' assert data['data']['band'] == 'NIR' assert len(data['data']['fields']) == 5 fields_loaded = True except AssertionError: nretries = nretries + 1 time.sleep(3) status, data = api( "POST", "allocation", data={ "group_id": public_group.id, "instrument_id": instrument_id, "hours_allocated": 100, "pi": "Ed Hubble", '_altdata': '{"access_token": "testtoken"}', }, token=super_admin_token, ) assert status == 200 assert data["status"] == "success" driver.get(f'/become_user/{user.id}') driver.get('/gcn_events/2019-04-25T08:18:05') driver.wait_for_xpath('//*[text()="190425 08:18:05"]') driver.wait_for_xpath('//*[text()="LVC"]') driver.wait_for_xpath('//*[text()="BNS"]') submit_button_xpath = ( '//div[@data-testid="observationplan-request-form"]//button[@type="submit"]' ) driver.wait_for_xpath(submit_button_xpath) select_box = driver.find_element_by_id( "mui-component-select-followupRequestAllocationSelect") select_box.click() driver.click_xpath( f'//li[contains(text(), "{instrument_name}")][contains(text(), "{public_group.name}")]', scroll_parent=True, ) # Click somewhere outside to remove focus from instrument select driver.click_xpath("//header") driver.click_xpath(submit_button_xpath) driver.wait_for_xpath( f"//div[@data-testid='{instrument_name}-requests-header']", timeout=15) driver.click_xpath( f"//div[@data-testid='{instrument_name}-requests-header']") driver.wait_for_xpath( f'//div[contains(@data-testid, "{instrument_name}_observationplanRequestsTable")]//div[contains(., "g,r,i")]', timeout=15, ) driver.wait_for_xpath( f'''//div[contains(@data-testid, "{instrument_name}_observationplanRequestsTable")]//div[contains(., "complete")]''', timeout=15, ) status, data = api("GET", "observation_plan", token=super_admin_token) assert status == 200 observation_plan_request_id = data['data'][-1]['observation_plans'][0][ 'observation_plan_request_id'] driver.click_xpath( f'//a[contains(@data-testid, "gcnRequest_{observation_plan_request_id}")]', scroll_parent=True, ) driver.click_xpath( f'//button[contains(@data-testid, "treasuremapRequest_{observation_plan_request_id}")]', scroll_parent=True, ) driver.click_xpath( f'//a[contains(@data-testid, "downloadRequest_{observation_plan_request_id}")]', scroll_parent=True, ) driver.click_xpath( f'//button[contains(@data-testid, "sendRequest_{observation_plan_request_id}")]', scroll_parent=True, ) driver.wait_for_xpath( f'''//div[contains(@data-testid, "{instrument_name}_observationplanRequestsTable")]//div[contains(., "submitted to telescope queue")]''', timeout=10, ) driver.click_xpath( f'//button[contains(@data-testid, "removeRequest_{observation_plan_request_id}")]', scroll_parent=True, ) driver.wait_for_xpath( f'''//div[contains(@data-testid, "{instrument_name}_observationplanRequestsTable")]//div[contains(., "deleted from telescope queue")]''', timeout=10, )
def test_gcn_request(driver, user, super_admin_token, public_group): datafile = f'{os.path.dirname(__file__)}/../data/GW190425_initial.xml' with open(datafile, 'rb') as fid: payload = fid.read() data = {'xml': payload} status, data = api('POST', 'gcn_event', data=data, token=super_admin_token) assert status == 200 assert data['status'] == 'success' telescope_name = str(uuid.uuid4()) status, data = api( 'POST', 'telescope', data={ 'name': telescope_name, 'nickname': telescope_name, 'lat': 0.0, 'lon': 0.0, 'elevation': 0.0, 'diameter': 10.0, }, token=super_admin_token, ) assert status == 200 assert data['status'] == 'success' telescope_id = data['data']['id'] fielddatafile = f'{os.path.dirname(__file__)}/../../../data/ZTF_Fields.csv' regionsdatafile = f'{os.path.dirname(__file__)}/../../../data/ZTF_Region.reg' instrument_name = str(uuid.uuid4()) status, data = api( 'POST', 'instrument', data={ 'name': instrument_name, 'type': 'imager', 'band': 'NIR', 'filters': ['ztfr'], 'telescope_id': telescope_id, "api_classname_obsplan": "ZTFMMAAPI", 'field_data': pd.read_csv(fielddatafile)[:5].to_dict(orient='list'), 'field_region': Regions.read(regionsdatafile).serialize(format='ds9'), }, token=super_admin_token, ) assert status == 200 assert data['status'] == 'success' instrument_id = data['data']['id'] params = {'includeGeoJSON': True} # wait for the fields to populate nretries = 0 fields_loaded = False while not fields_loaded and nretries < 5: try: status, data = api( 'GET', f'instrument/{instrument_id}', params=params, token=super_admin_token, ) assert status == 200 assert data['status'] == 'success' assert data['data']['band'] == 'NIR' print(data['data']) assert len(data['data']['fields']) == 5 fields_loaded = True except AssertionError: nretries = nretries + 1 time.sleep(3) datafile = f'{os.path.dirname(__file__)}/../../../data/sample_observation_data.csv' data = { 'telescopeName': telescope_name, 'instrumentName': instrument_name, 'observationData': pd.read_csv(datafile).to_dict(orient='list'), } status, data = api('POST', 'observation', data=data, token=super_admin_token) assert status == 200 assert data['status'] == 'success' params = { 'telescopeName': telescope_name, 'instrumentName': instrument_name, 'startDate': "2019-04-25 08:18:05", 'endDate': "2019-04-28 08:18:05", 'localizationDateobs': "2019-04-25T08:18:05", 'localizationName': "bayestar.fits.gz", 'localizationCumprob': 1.01, 'returnStatistics': True, } # wait for the executed observations to populate nretries = 0 observations_loaded = False while not observations_loaded and nretries < 5: try: status, data = api('GET', 'observation', params=params, token=super_admin_token) assert status == 200 data = data["data"] assert len(data['observations']) == 10 observations_loaded = True except AssertionError: nretries = nretries + 1 time.sleep(3) driver.get(f'/become_user/{user.id}') driver.get('/gcn_events/2019-04-25T08:18:05') driver.wait_for_xpath('//*[text()="190425 08:18:05"]') driver.wait_for_xpath('//*[text()="LVC"]') driver.wait_for_xpath('//*[text()="BNS"]') driver.wait_for_xpath('//*[@id="root_localizationName"]') driver.click_xpath('//*[@id="root_localizationName"]') driver.wait_for_xpath('//li[contains(text(), "bayestar.fits.gz")]') driver.click_xpath('//li[contains(text(), "bayestar.fits.gz")]') driver.wait_for_xpath('//*[@id="root_localizationCumprob"]').clear() driver.wait_for_xpath('//*[@id="root_localizationCumprob"]').send_keys( 1.01) submit_button_xpath = '//button[@type="submit"]' driver.wait_for_xpath(submit_button_xpath) driver.click_xpath(submit_button_xpath) select_box = driver.find_element_by_id( "mui-component-select-followupRequestInstrumentSelect") select_box.click() driver.click_xpath( f'//li[contains(text(), "{telescope_name}")][contains(text(), "{instrument_name}")]', scroll_parent=True, ) driver.click_xpath( f'//a[contains(@data-testid, "observationGcn_{instrument_id}")]', scroll_parent=True, )
def test_observation_plan_galaxy( user, super_admin_token, upload_data_token, view_only_token, public_group ): catalog_name = 'test_galaxy_catalog' # in case the catalog already exists, delete it. status, data = api( 'DELETE', f'galaxy_catalog/{catalog_name}', token=super_admin_token ) datafile = f'{os.path.dirname(__file__)}/../../../data/GW190814.xml' with open(datafile, 'rb') as fid: payload = fid.read() data = {'xml': payload} status, data = api('POST', 'gcn_event', data=data, token=super_admin_token) assert status == 200 assert data['status'] == 'success' gcnevent_id = data['data']['gcnevent_id'] # wait for event to load for n_times in range(26): status, data = api( 'GET', "gcn_event/2019-08-14T21:10:39", token=super_admin_token ) if data['status'] == 'success': break time.sleep(2) assert n_times < 25 # wait for the localization to load params = {"include2DMap": True} for n_times_2 in range(26): status, data = api( 'GET', 'localization/2019-08-14T21:10:39/name/LALInference.v1.fits.gz', token=super_admin_token, params=params, ) if data['status'] == 'success': data = data["data"] assert data["dateobs"] == "2019-08-14T21:10:39" assert data["localization_name"] == "LALInference.v1.fits.gz" assert np.isclose(np.sum(data["flat_2d"]), 1) break else: time.sleep(2) assert n_times_2 < 25 localization_id = data['id'] name = str(uuid.uuid4()) status, data = api( 'POST', 'telescope', data={ 'name': name, 'nickname': name, 'lat': 0.0, 'lon': 0.0, 'elevation': 0.0, 'diameter': 10.0, }, token=super_admin_token, ) assert status == 200 assert data['status'] == 'success' telescope_id = data['data']['id'] fielddatafile = f'{os.path.dirname(__file__)}/../../../data/ZTF_Fields.csv' regionsdatafile = f'{os.path.dirname(__file__)}/../../../data/ZTF_Region.reg' instrument_name = str(uuid.uuid4()) status, data = api( 'POST', 'instrument', data={ 'name': instrument_name, 'type': 'imager', 'band': 'Optical', 'filters': ['ztfr'], 'telescope_id': telescope_id, 'api_classname': 'ZTFAPI', 'api_classname_obsplan': 'ZTFMMAAPI', 'field_data': pd.read_csv(fielddatafile)[:5].to_dict(orient='list'), 'field_region': Regions.read(regionsdatafile).serialize(format='ds9'), }, token=super_admin_token, ) assert status == 200 assert data['status'] == 'success' instrument_id = data['data']['id'] # wait for the fields to populate nretries = 0 fields_loaded = False while not fields_loaded and nretries < 5: try: status, data = api( 'GET', f'instrument/{instrument_id}', token=super_admin_token, ) assert status == 200 assert data['status'] == 'success' assert data['data']['band'] == 'NIR' assert len(data['data']['fields']) == 5 fields_loaded = True except AssertionError: nretries = nretries + 1 time.sleep(3) datafile = f'{os.path.dirname(__file__)}/../../../data/CLU_mini.hdf5' data = { 'catalog_name': catalog_name, 'catalog_data': Table.read(datafile).to_pandas().to_dict(orient='list'), } status, data = api('POST', 'galaxy_catalog', data=data, token=super_admin_token) assert status == 200 assert data['status'] == 'success' params = {'catalog_name': catalog_name} nretries = 0 galaxies_loaded = False while nretries < 10: status, data = api( 'GET', 'galaxy_catalog', token=view_only_token, params=params ) assert status == 200 data = data["data"]["galaxies"] if len(data) == 92 and any( [ d['name'] == '6dFgs gJ0001313-055904' and d['mstar'] == 336.60756522868667 for d in data ] ): galaxies_loaded = True break nretries = nretries + 1 time.sleep(5) assert nretries < 10 assert galaxies_loaded request_data = { 'group_id': public_group.id, 'instrument_id': instrument_id, 'pi': 'Shri Kulkarni', 'hours_allocated': 200, 'start_date': '3021-02-27T00:00:00', 'end_date': '3021-07-20T00:00:00', 'proposal_id': 'COO-2020A-P01', } status, data = api('POST', 'allocation', data=request_data, token=super_admin_token) assert status == 200 assert data['status'] == 'success' allocation_id = data['data']['id'] queue_name = str(uuid.uuid4()) request_data = { 'allocation_id': allocation_id, 'gcnevent_id': gcnevent_id, 'localization_id': localization_id, 'payload': { 'start_date': '2019-04-25 01:01:01', 'end_date': '2019-04-27 01:01:01', 'filter_strategy': 'block', 'schedule_strategy': 'galaxy', 'galaxy_catalog': catalog_name, 'schedule_type': 'greedy_slew', 'exposure_time': 300, 'filters': 'ztfg', 'maximum_airmass': 2.0, 'integrated_probability': 100, 'minimum_time_difference': 30, 'queue_name': queue_name, 'program_id': 'Partnership', 'subprogram_name': 'GRB', }, } status, data = api( 'POST', 'observation_plan', data=request_data, token=super_admin_token ) assert status == 200 assert data['status'] == 'success' id = data['data']['ids'][0] # wait for the observation plan to populate nretries = 0 observation_plan_loaded = False while not observation_plan_loaded and nretries < 5: try: status, data = api( 'GET', f'observation_plan/{id}', params={"includePlannedObservations": "true"}, token=super_admin_token, ) assert status == 200 assert data['status'] == 'success' assert data["data"]["gcnevent_id"] == gcnevent_id assert data["data"]["allocation_id"] == allocation_id assert data["data"]["payload"] == request_data["payload"] assert len(data["data"]["observation_plans"]) == 1 observation_plan = data["data"]["observation_plans"][0] assert ( observation_plan['plan_name'] == request_data["payload"]['queue_name'] ) assert observation_plan['validity_window_start'] == request_data["payload"][ 'start_date' ].replace(" ", "T") assert observation_plan['validity_window_end'] == request_data["payload"][ 'end_date' ].replace(" ", "T") planned_observations = observation_plan['planned_observations'] assert len(planned_observations) > 0 observation_plan_loaded = True except AssertionError: nretries = nretries + 1 time.sleep(10) assert len(planned_observations) == 29 assert all( [ obs['filt'] == request_data["payload"]['filters'] for obs in planned_observations ] ) assert all( [ obs['exposure_time'] == int(request_data["payload"]['exposure_time']) for obs in planned_observations ] )
def test_observation_plan_tiling( user, super_admin_token, upload_data_token, view_only_token, public_group ): datafile = f'{os.path.dirname(__file__)}/../data/GW190425_initial.xml' with open(datafile, 'rb') as fid: payload = fid.read() data = {'xml': payload} status, data = api('POST', 'gcn_event', data=data, token=super_admin_token) assert status == 200 assert data['status'] == 'success' gcnevent_id = data['data']['gcnevent_id'] dateobs = "2019-04-25 08:18:05" skymap = "bayestar.fits.gz" status, data = api( 'GET', f'localization/{dateobs}/name/{skymap}', token=super_admin_token, ) assert status == 200 assert data['status'] == 'success' localization_id = data['data']['id'] name = str(uuid.uuid4()) status, data = api( 'POST', 'telescope', data={ 'name': name, 'nickname': name, 'lat': 0.0, 'lon': 0.0, 'elevation': 0.0, 'diameter': 10.0, }, token=super_admin_token, ) assert status == 200 assert data['status'] == 'success' telescope_id = data['data']['id'] fielddatafile = f'{os.path.dirname(__file__)}/../../../data/ZTF_Fields.csv' regionsdatafile = f'{os.path.dirname(__file__)}/../../../data/ZTF_Region.reg' instrument_name = str(uuid.uuid4()) status, data = api( 'POST', 'instrument', data={ 'name': instrument_name, 'type': 'imager', 'band': 'Optical', 'filters': ['ztfr'], 'telescope_id': telescope_id, 'api_classname': 'ZTFAPI', 'api_classname_obsplan': 'ZTFMMAAPI', 'field_data': pd.read_csv(fielddatafile)[:5].to_dict(orient='list'), 'field_region': Regions.read(regionsdatafile).serialize(format='ds9'), 'sensitivity_data': { 'ztfr': { 'limiting_magnitude': 20.3, 'magsys': 'ab', 'exposure_time': 30, 'zeropoint': 26.3, } }, }, token=super_admin_token, ) assert status == 200 assert data['status'] == 'success' instrument_id = data['data']['id'] # wait for the fields to populate time.sleep(15) request_data = { 'group_id': public_group.id, 'instrument_id': instrument_id, 'pi': 'Shri Kulkarni', 'hours_allocated': 200, 'start_date': '3021-02-27T00:00:00', 'end_date': '3021-07-20T00:00:00', 'proposal_id': 'COO-2020A-P01', } status, data = api('POST', 'allocation', data=request_data, token=super_admin_token) assert status == 200 assert data['status'] == 'success' allocation_id = data['data']['id'] queue_name = str(uuid.uuid4()) request_data = { 'allocation_id': allocation_id, 'gcnevent_id': gcnevent_id, 'localization_id': localization_id, 'payload': { 'start_date': '2019-04-25 01:01:01', 'end_date': '2019-04-27 01:01:01', 'filter_strategy': 'block', 'schedule_strategy': 'tiling', 'schedule_type': 'greedy_slew', 'exposure_time': 300, 'filters': 'ztfr', 'maximum_airmass': 2.0, 'integrated_probability': 100, 'minimum_time_difference': 30, 'queue_name': queue_name, 'program_id': 'Partnership', 'subprogram_name': 'GRB', }, } status, data = api( 'POST', 'observation_plan', data=request_data, token=super_admin_token ) assert status == 200 assert data['status'] == 'success' id = data['data']['ids'][0] # wait for the observation plan to finish time.sleep(15) status, data = api( 'GET', f'observation_plan/{id}', params={"includePlannedObservations": "true"}, token=super_admin_token, ) assert status == 200 assert data['status'] == 'success' assert data["data"]["gcnevent_id"] == gcnevent_id assert data["data"]["allocation_id"] == allocation_id assert data["data"]["payload"] == request_data["payload"] assert len(data["data"]["observation_plans"]) == 1 observation_plan = data["data"]["observation_plans"][0] assert observation_plan['plan_name'] == request_data["payload"]['queue_name'] assert observation_plan['validity_window_start'] == request_data["payload"][ 'start_date' ].replace(" ", "T") assert observation_plan['validity_window_end'] == request_data["payload"][ 'end_date' ].replace(" ", "T") planned_observations = observation_plan['planned_observations'] assert len(planned_observations) == 5 assert all( [ obs['filt'] == request_data["payload"]['filters'] for obs in planned_observations ] ) assert all( [ obs['exposure_time'] == int(request_data["payload"]['exposure_time']) for obs in planned_observations ] ) status, data = api( 'GET', f'observation_plan/{id}/simsurvey', token=super_admin_token ) assert status == 200
import os import sys import pandas as pd from getter import get_dataframe_by_definition from regions import Regions filter_region_name = Regions.filter_region_name reference_region_names = Regions.names() from definitions import definitions # ------------------------------------------------------------------------- # Defintions # ------------------------------------------------------------------------- from config import XL_SAMPLE_FOLDER source_def_sample = { 'varname': 'PPI_PROM_ytd', 'folder': XL_SAMPLE_FOLDER, 'filename': 'industrial_prices.xls', 'sheet': 'пром.товаров', 'anchor': 'B5', 'anchor_value': 96.6 } def_dict_2 = { 'varname': 'SHIPMENTS', 'folder': XL_SAMPLE_FOLDER, 'filename': 'shipment.xls',
def regions(self): self.setCentralWidget(Regions(self)) self.enableTools()
def get_data_items(cls, data_items='all', regions='ES', start_date=None, end_date=None, language='ES', errors='ignore'): """ Collects the required Data Items from associated Data Sources Parameters ---------- data_items : list of str list of data item names. By default, 'all' are collected. regions : list of str list of region names. By default, 'ES' refers to all Spanish regions. start_date : pd.datetime first day to be considered in TEMPORAL data items. By default, None is established. end_date : pd.datetime last day to be considered in TEMPORAL data items. By default, None is established. language : str language of the returned data. 'ES' for Spanish (default value), 'EN' for English. errors : str action to be taken when errors occur. 'ignore' tries to get all possible data items even if some can't be collected, 'raise' throws an exception and the execution is aborted upon detection of any error. Returns ------- pd.DataFrame a DataFrame with the required information. Notes ----- If dates are passed, then it is assumed that TEMPORAL data items are required. Otherwise, a GEOGRAPHICAL retrieval is assumed. A TEMPORAL retrieval produces a DataFrame with daily [Date] as row indexer and [Region, Data Item] as column multiindexer. A GEOGRAPHICAL retrieval produces a DataFrame with [Region] as row indexer and [Data Item] as column indexer. """ # if data sources are not initialized, lets read configurations if not cls.__DATA_SOURCES_INITIALIZED: cls.__init_data_sources() ##### check of parameters ##### if data_items == 'all': data_items = cls.get_data_items_names(data_type=None, language=language) else: ## check if items are implemented ## # get all implemented items implemented_data_sources = cls.get_data_items_names( data_type=None, language=language) implemented_data_items = [] for implemented_data_source in list( implemented_data_sources.keys()): implemented_data_items = implemented_data_items + implemented_data_sources[ implemented_data_source] # check successful_data_items = [] for data_item in data_items: if data_item not in implemented_data_items: print(f'WARNING: Item {data_item} is not implemented') else: successful_data_items.append(data_item) if not successful_data_items: print( 'WARNING: No result found for the specified data items and conditions' ) return None data_items = successful_data_items if regions == 'ES': regions = Regions.get_regions('ES') if start_date is None or end_date is None: assumed_data_type = DataType.GEOGRAPHICAL else: assumed_data_type = DataType.TEMPORAL print("Assumed a " + str(assumed_data_type.name) + " data retrieval...") if assumed_data_type is DataType.TEMPORAL: if start_date > end_date: print('ERROR: start_date (' + str(start_date) + ') should be smaller or equal than end_date (' + str(start_date) + ')') return None if end_date > pd.to_datetime('today').date(): print('ERROR: end_date (' + str(end_date) + ') should not refer to the future') return None ### change data items (display names) to internal representation ### internalname_displayname_dict = cls._get_internal_names_mapping( assumed_data_type, data_items, language=language ) # get internal name - display name dict (then is used to rename again) if internalname_displayname_dict is None: return None data_items = list(internalname_displayname_dict.keys() ) # change data_items to internal representation ### group data items by data source in dictionary ### # existing items for assumed data type items_by_source = cls.get_data_items_names( data_type=assumed_data_type, language=language) # dict with : source -> [item1, item2] items_by_assumed_data_type = [] for items in items_by_source.values(): items_by_assumed_data_type = items_by_assumed_data_type + items # group requested items by data sources requested_items_by_source = defaultdict( list ) # dict datasource : [requested item 1, requested item 2, ...] for data_item in data_items: source_class_found = False source = 0 while source < len( cls.__DATA_SOURCE_CLASSES) and not source_class_found: source_class_found = cls.__DATA_SOURCE_CLASSES[ source].data_item_exists(data_item) source = source + 1 if source_class_found: requested_items_by_source[cls.__DATA_SOURCE_CLASSES[ source - 1]].append(data_item) else: # never should get there print('WARNING: Data source not found for item \'' + str(data_item) + '\'') ##### data retrieval ##### df_all_data_sources = None ## get data by data source ## for DATA_SOURCE_CLASS in requested_items_by_source.keys(): df_data_source = None data_items = requested_items_by_source[DATA_SOURCE_CLASS] # for temporal data type if assumed_data_type is DataType.TEMPORAL: df_data_source = DATA_SOURCE_CLASS(data_items, regions, start_date, end_date).get_data(errors) if df_data_source is not None: df_data_source = cls.__complete_dates( df_data_source, start_date, end_date ) # complete with nan values those days without info # for geographical data type elif assumed_data_type is DataType.GEOGRAPHICAL: df_data_source = DATA_SOURCE_CLASS(data_items, regions).get_data(errors) else: # never should get here return None # continuous joining of data from diverse data sources if df_data_source is not None: if df_all_data_sources is None: df_all_data_sources = df_data_source.sort_index(axis=1) else: df_all_data_sources = pd.concat( [df_all_data_sources, df_data_source], axis='columns').sort_index(axis=1) ## END: get data by data source ## if df_all_data_sources is None: print( 'WARNING: No result found for the specified data items and conditions' ) return None def rename_with_regex(col_name): for internal_name in list(internalname_displayname_dict.keys()): if re.match(f"^{internal_name}$|^{internal_name} \(", col_name): return re.sub( pattern=internal_name, repl=internalname_displayname_dict[internal_name], string=col_name) return 'None' df_all_data_sources.rename(columns=rename_with_regex, level='Item', inplace=True) ### filter retrieved data to match the specific query determined by data_items, regions and dates ### # filter requested data_items (some data sources request in the same query more data items than the ones requested ) df_all_data_sources = df_all_data_sources.loc[:, df_all_data_sources. columns.get_level_values( 'Item') != 'None'] if assumed_data_type is DataType.TEMPORAL: df_all_data_sources = df_all_data_sources[ (df_all_data_sources.index >= start_date) & (df_all_data_sources.index <= end_date)] # to filter requested dates (indexes) df_all_data_sources = df_all_data_sources.loc[:, df_all_data_sources. columns. get_level_values( 'Region' ).isin( regions )] # to filter dates (indexes) else: df_all_data_sources = df_all_data_sources[ df_all_data_sources.index.isin( regions)] # to filter requested regions (indexes) df_all_data_sources = df_all_data_sources.loc[:, ~df_all_data_sources. columns.duplicated()] return df_all_data_sources
def generate_plan(observation_plan_id, request_id, user_id): """Use gwemopt to construct observing plan.""" from ..models import DBSession from skyportal.handlers.api.instrument import add_tiles Session = scoped_session( sessionmaker(bind=DBSession.session_factory.kw["bind"])) import gwemopt import gwemopt.utils import gwemopt.segments import gwemopt.skyportal from ..models import ( EventObservationPlan, Galaxy, InstrumentField, ObservationPlanRequest, PlannedObservation, User, ) session = Session() try: plan = session.query(EventObservationPlan).get(observation_plan_id) request = session.query(ObservationPlanRequest).get(request_id) user = session.query(User).get(user_id) event_time = Time(request.gcnevent.dateobs, format='datetime', scale='utc') start_time = Time(request.payload["start_date"], format='iso', scale='utc') end_time = Time(request.payload["end_date"], format='iso', scale='utc') params = { 'config': { request.instrument.name: { # field list from skyportal 'tesselation': request.instrument.fields, # telescope longitude [deg] 'longitude': request.instrument.telescope.lon, # telescope latitude [deg] 'latitude': request.instrument.telescope.lat, # telescope elevation [m] 'elevation': request.instrument.telescope.elevation, # telescope name 'telescope': request.instrument.name, # telescope horizon 'horizon': -12.0, # time in seconds to change the filter 'filt_change_time': 0.0, # extra overhead in seconds 'overhead_per_exposure': 0.0, # slew rate for the telescope [deg/s] 'slew_rate': 2.6, # camera readout time 'readout': 0.0, # telescope field of view 'FOV': 0.0, # exposure time for the given limiting magnitude 'exposuretime': 1.0, # limiting magnitude given telescope time 'magnitude': 0.0, }, }, # gwemopt filter strategy # options: block (blocks of single filters), integrated (series of alternating filters) 'doAlternativeFilters': request.payload["filter_strategy"] == "block", # flag to indicate fields come from DB 'doDatabase': True, # only keep tiles within powerlaw_cl 'doMinimalTiling': True, # single set of scheduled observations 'doSingleExposure': True, # gwemopt scheduling algorithms # options: greedy, greedy_slew, sear, airmass_weighted 'scheduleType': request.payload["schedule_type"], # list of filters to use for observations 'filters': request.payload["filters"].split(","), # GPS time for event 'gpstime': event_time.gps, # Healpix nside for the skymap 'nside': 512, # maximum integrated probability of the skymap to consider 'powerlaw_cl': request.payload["integrated_probability"], 'telescopes': [request.instrument.name], # minimum difference between observations of the same field 'mindiff': request.payload["minimum_time_difference"], # maximum airmass with which to observae 'airmass': request.payload["maximum_airmass"], # array of exposure times (same length as filter array) 'exposuretimes': np.array([int(request.payload["exposure_time"])] * len(request.payload["filters"].split(","))), } if request.payload["schedule_strategy"] == "galaxy": params = { **params, 'tilesType': 'galaxy', 'galaxy_catalog': request.payload["galaxy_catalog"], 'galaxy_grade': 'S', 'writeCatalog': False, 'catalog_n': 1.0, 'powerlaw_dist_exp': 1.0, } elif request.payload["schedule_strategy"] == "tiling": params = {**params, 'tilesType': 'moc'} else: raise AttributeError( 'scheduling_strategy should be tiling or galaxy') params = gwemopt.utils.params_checker(params) params = gwemopt.segments.get_telescope_segments(params) params["Tobs"] = [ start_time.mjd - event_time.mjd, end_time.mjd - event_time.mjd, ] params['map_struct'] = dict( zip(['prob', 'distmu', 'distsigma', 'distnorm'], request.localization.flat)) params['is3D'] = request.localization.is_3d # Function to read maps map_struct = gwemopt.utils.read_skymap(params, is3D=params["do3D"], map_struct=params['map_struct']) if params["tilesType"] == "galaxy": query = Galaxy.query_records_accessible_by(user, mode="read") query = query.filter( Galaxy.catalog_name == params["galaxy_catalog"]) galaxies = query.all() catalog_struct = {} catalog_struct["ra"] = np.array([g.ra for g in galaxies]) catalog_struct["dec"] = np.array([g.dec for g in galaxies]) catalog_struct["S"] = np.array([1.0 for g in galaxies]) catalog_struct["Sloc"] = np.array([1.0 for g in galaxies]) catalog_struct["Smass"] = np.array([1.0 for g in galaxies]) if params["tilesType"] == "moc": moc_structs = gwemopt.skyportal.create_moc_from_skyportal( params, map_struct=map_struct) tile_structs = gwemopt.tiles.moc(params, map_struct, moc_structs) elif params["tilesType"] == "galaxy": if request.instrument.region is None: raise ValueError( 'Must define the instrument region in the case of galaxy requests' ) regions = Regions.parse(request.instrument.region, format='ds9') tile_structs = gwemopt.skyportal.create_galaxy_from_skyportal( params, map_struct, catalog_struct, regions=regions) tile_structs, coverage_struct = gwemopt.coverage.timeallocation( params, map_struct, tile_structs) # if the fields do not yet exist, we need to add them if params["tilesType"] == "galaxy": regions = Regions.parse(request.instrument.region, format='ds9') data = { 'RA': coverage_struct["data"][:, 0], 'Dec': coverage_struct["data"][:, 1], } field_data = pd.DataFrame.from_dict(data) field_ids = add_tiles( request.instrument.id, request.instrument.name, regions, field_data, session=session, ) planned_observations = [] for ii in range(len(coverage_struct["ipix"])): data = coverage_struct["data"][ii, :] filt = coverage_struct["filters"][ii] mjd = data[2] tt = Time(mjd, format='mjd') overhead_per_exposure = params["config"][ request.instrument.name]["overhead_per_exposure"] exposure_time, prob = data[4], data[6] if params["tilesType"] == "galaxy": field_id = field_ids[ii] else: field_id = data[5] field = InstrumentField.query.filter( InstrumentField.instrument_id == request.instrument.id, InstrumentField.field_id == field_id, ).first() if field is None: return log(f"Missing field {field_id} from list") planned_observation = PlannedObservation( obstime=tt.datetime, dateobs=request.gcnevent.dateobs, field_id=field.id, exposure_time=exposure_time, weight=prob, filt=filt, instrument_id=request.instrument.id, planned_observation_id=ii, observation_plan_id=plan.id, overhead_per_exposure=overhead_per_exposure, ) planned_observations.append(planned_observation) session.add_all(planned_observations) plan.status = 'complete' session.merge(plan) session.commit() request.status = 'complete' session.merge(request) session.commit() flow = Flow() flow.push( '*', "skyportal/REFRESH_GCNEVENT", payload={"gcnEvent_dateobs": request.gcnevent.dateobs}, ) return log( f"Generated plan for observation plan {observation_plan_id}") except Exception as e: return log( f"Unable to generate plan for observation plan {observation_plan_id}: {e}" ) finally: Session.remove()
from regions import Regions z = Regions.summable_regions() print(z) for r in Regions.district_names(): w = Regions.region_by_district(r) print ('\n', r) print (w) # passes now # assert set(Regions.region_by_district("Уральский федеральный округ")) == set (['Курганская область', 'Свердловская область', 'Ханты-Мансийский авт. округ - Югра', 'Ямало-Ненецкий авт. округ', 'Тюменская область без авт. округов', 'Челябинская область']) # must pass: assert set(Regions.region_by_district("Уральский федеральный округ")) == set (['Курганская область', 'Свердловская область', #'Ханты-Мансийский авт. округ - Югра', #'Ямало-Ненецкий авт. округ', #'Тюменская область без авт. округов', 'Тюменская область', 'Челябинская область']) print(Regions.region_by_district("Северо-Западный федеральный округ")) # Regions.region_by_district("Северо-Западный федеральный округ") must return: ['Республика Карелия', 'Республика Коми', #'Ненецкий авт. округ', #'Архангельская область без авт. округа', 'Архангельская область', 'Вологодская область', 'Калининградская область',