def submit(request): """Submit a follow-up request to SEDMv2. Parameters ---------- request: skyportal.models.FollowupRequest The request to submit. """ from ..models import FacilityTransaction, DBSession validate_request_to_sedmv2(request) if cfg['app.sedmv2_endpoint'] is not None: altdata = request.allocation.altdata if not altdata: raise ValueError('Missing allocation information.') payload = { 'obj_id': request.obj_id, 'allocation_id': request.allocation.id, 'payload': request.payload, } r = requests.post( cfg['app.sedmv2_endpoint'], json=payload, headers={"Authorization": f"token {altdata['api_token']}"}, ) if r.status_code == 200: request.status = 'submitted' else: request.status = f'rejected: {r.content}' transaction = FacilityTransaction( request=http.serialize_requests_request(r.request), response=http.serialize_requests_response(r), followup_request=request, initiator_id=request.last_modified_by_id, ) else: request.status = 'submitted' transaction = FacilityTransaction( request=None, response=None, followup_request=request, initiator_id=request.last_modified_by_id, ) DBSession().add(transaction) flow = Flow() flow.push( '*', 'skyportal/REFRESH_SOURCE', payload={'obj_key': request.obj.internal_key}, )
def post_observing_run(data, user_id, session): """Post ObservingRun to database. data: dict Observing run dictionary user_id : int SkyPortal ID of User posting the GcnEvent session: sqlalchemy.Session Database session for this transaction """ user = session.query(User).get(user_id) try: rund = ObservingRunPost.load(data) except ValidationError as exc: raise ValidationError( f"Invalid/missing parameters: {exc.normalized_messages()}" ) run = ObservingRun(**rund) run.owner_id = user.id session.add(run) session.commit() flow = Flow() flow.push('*', "skyportal/FETCH_OBSERVING_RUNS") return run.id
def post_assignment(data, user_id, session): """Post assignment to database. data: dict Assignment dictionary user_id : int SkyPortal ID of User posting the GcnEvent session: sqlalchemy.Session Database session for this transaction """ user = session.query(User).get(user_id) try: assignment = ClassicalAssignment(**AssignmentSchema.load(data=data)) except ValidationError as e: raise ValidationError('Error parsing followup request: ' f'"{e.normalized_messages()}"') run_id = assignment.run_id data['priority'] = assignment.priority.name run = ObservingRun.get_if_accessible_by(run_id, user, raise_if_none=False) if run is None: raise ValueError('Observing run is not accessible.') predecessor = ( ClassicalAssignment.query_records_accessible_by(user).filter( ClassicalAssignment.obj_id == assignment.obj_id, ClassicalAssignment.run_id == run_id, ).first()) if predecessor is not None: raise ValueError('Object is already assigned to this run.') assignment = ClassicalAssignment(**data) assignment.requester_id = user.id session.add(assignment) session.commit() flow = Flow() flow.push( '*', "skyportal/REFRESH_SOURCE", payload={"obj_key": assignment.obj.internal_key}, ) flow.push( '*', "skyportal/REFRESH_OBSERVING_RUN", payload={"run_id": assignment.run_id}, ) return assignment.id
def add_linked_thumbnails_and_push_ws_msg(obj_id, user): try: obj = Obj.get_if_accessible_by(obj_id, user) obj.add_linked_thumbnails() flow = Flow() flow.push('*', "skyportal/REFRESH_SOURCE", payload={"obj_key": obj.internal_key}) flow.push('*', "skyportal/REFRESH_CANDIDATE", payload={"id": obj.internal_key}) except Exception as e: log(f"Unable to add linked thumbnails to {obj_id}: {e}") finally: DBSession.remove()
def add_linked_thumbnails_and_push_ws_msg(obj_id, user_id): with Session() as session: try: user = session.query(User).get(user_id) if Obj.get_if_accessible_by(obj_id, user) is None: raise AccessError( f"Insufficient permissions for User {user_id} to read Obj {obj_id}" ) obj = session.query(Obj).get(obj_id) obj.add_linked_thumbnails(session=session) flow = Flow() flow.push( '*', "skyportal/REFRESH_SOURCE", payload={"obj_key": obj.internal_key} ) flow.push( '*', "skyportal/REFRESH_CANDIDATE", payload={"id": obj.internal_key} ) except Exception as e: log(f"Unable to add linked thumbnails to {obj_id}: {e}") session.rollback()
def submit(request): """Submit a follow-up request to SEDM. Parameters ---------- request: skyportal.models.FollowupRequest The request to submit. """ from ..models import FacilityTransaction, DBSession payload = convert_request_to_sedm(request, method_value='new') content = json.dumps(payload) r = requests.post( cfg['app.sedm_endpoint'], files={'jsonfile': ('jsonfile', content)}, ) if r.status_code == 200: request.status = 'submitted' else: request.status = f'rejected: {r.content}' transaction = FacilityTransaction( request=http.serialize_requests_request(r.request), response=http.serialize_requests_response(r), followup_request=request, initiator_id=request.last_modified_by_id, ) DBSession().add(transaction) flow = Flow() flow.push( '*', 'skyportal/REFRESH_SOURCE', payload={'obj_key': request.obj.internal_key}, )
def update(request): """Update a request in the SEDM queue. Parameters ---------- request: skyportal.models.FollowupRequest The updated request. """ from ..models import FacilityTransaction, DBSession payload = convert_request_to_sedm(request, method_value='edit') content = json.dumps(payload) r = requests.post( cfg['app.sedm_endpoint'], files={'jsonfile': ('jsonfile', content)}, ) if r.status_code == 200: request.status = 'submitted' else: request.status = f'rejected: {r.content}' transaction = FacilityTransaction( request=http.serialize_requests_request(r.request), response=http.serialize_requests_response(r), followup_request=request, initiator_id=request.last_modified_by_id, ) DBSession().add(transaction) flow = Flow() flow.push( '*', "skyportal/REFRESH_FOLLOWUP_REQUESTS", payload={"obj_key": request.obj.internal_key}, )
def receive_after_flush(session, context): listing_subquery = (Listing.query.filter( Listing.list_name == "favorites").filter( Listing.obj_id == target.obj_id).distinct( Listing.user_id).subquery()) users = (User.query.join( listing_subquery, User.id == listing_subquery.c.user_id).filter( User.preferences["favorite_sources_activity_notifications"][ target.__tablename__].astext.cast( sa.Boolean).is_(True)).all()) ws_flow = Flow() for user in users: # Only notify users who have read access to the new record in question if target.__class__.get_if_accessible_by(target.id, user) is not None: session.add( UserNotification( user=user, text= f"New {target.__class__.__name__.lower()} on your favorite source *{target.obj_id}*", url=f"/source/{target.obj_id}", )) ws_flow.push(user.id, "skyportal/FETCH_NOTIFICATIONS")
def delete(request): """Delete a follow-up request from SEDM queue. Parameters ---------- request: skyportal.models.FollowupRequest The request to delete from the queue and the SkyPortal database. """ from ..models import FacilityTransaction, DBSession payload = convert_request_to_sedm(request, method_value='delete') content = json.dumps(payload) r = requests.post( cfg['app.sedm_endpoint'], files={'jsonfile': ('jsonfile', content)}, ) r.raise_for_status() request.status = "deleted" transaction = FacilityTransaction( request=http.serialize_requests_request(r.request), response=http.serialize_requests_response(r), followup_request=request, initiator_id=request.last_modified_by_id, ) DBSession().add(transaction) flow = Flow() flow.push( '*', 'skyportal/REFRESH_SOURCE', payload={'obj_key': request.obj.internal_key}, )
def submit(request): """Generate an observation plan. Parameters ---------- request: skyportal.models.ObservationPlanRequest The request to generate the observation plan. """ from tornado.ioloop import IOLoop from ..models import DBSession, EventObservationPlan plan = EventObservationPlan.query.filter_by( plan_name=request.payload["queue_name"]).first() if plan is None: # check payload required_parameters = { 'start_date', 'end_date', 'schedule_type', 'schedule_strategy', 'filter_strategy', 'exposure_time', 'filters', 'maximum_airmass', 'integrated_probability', 'minimum_time_difference', } if not required_parameters.issubset(set(request.payload.keys())): raise ValueError('Missing required planning parameter') if request.payload["schedule_type"] not in [ "greedy", "greedy_slew", "sear", "airmass_weighted", ]: raise ValueError( 'schedule_type must be one of greedy, greedy_slew, sear, or airmass_weighted' ) if (request.payload["integrated_probability"] < 0 or request.payload["integrated_probability"] > 100): raise ValueError( 'integrated_probability must be between 0 and 100') if request.payload["filter_strategy"] not in [ "block", "integrated" ]: raise ValueError( 'filter_strategy must be either block or integrated') start_time = Time(request.payload["start_date"], format='iso', scale='utc') end_time = Time(request.payload["end_date"], format='iso', scale='utc') plan = EventObservationPlan( observation_plan_request_id=request.id, dateobs=request.gcnevent.dateobs, plan_name=request.payload['queue_name'], instrument_id=request.instrument.id, validity_window_start=start_time.datetime, validity_window_end=end_time.datetime, ) DBSession().add(plan) DBSession().commit() request.status = 'running' DBSession().merge(request) DBSession().commit() flow = Flow() flow.push( '*', "skyportal/REFRESH_GCNEVENT", payload={"gcnEvent_dateobs": request.gcnevent.dateobs}, ) log(f"Generating schedule for observation plan {plan.id}") IOLoop.current().run_in_executor( None, lambda: generate_plan(plan.id, request.id, request. requester.id)) else: raise ValueError( f'plan_name {request.payload["queue_name"]} already exists.')
def generate_plan(observation_plan_id, request_id, user_id): """Use gwemopt to construct observing plan.""" from ..models import DBSession from skyportal.handlers.api.instrument import add_tiles Session = scoped_session( sessionmaker(bind=DBSession.session_factory.kw["bind"])) import gwemopt import gwemopt.utils import gwemopt.segments import gwemopt.skyportal from ..models import ( EventObservationPlan, Galaxy, InstrumentField, ObservationPlanRequest, PlannedObservation, User, ) session = Session() try: plan = session.query(EventObservationPlan).get(observation_plan_id) request = session.query(ObservationPlanRequest).get(request_id) user = session.query(User).get(user_id) event_time = Time(request.gcnevent.dateobs, format='datetime', scale='utc') start_time = Time(request.payload["start_date"], format='iso', scale='utc') end_time = Time(request.payload["end_date"], format='iso', scale='utc') params = { 'config': { request.instrument.name: { # field list from skyportal 'tesselation': request.instrument.fields, # telescope longitude [deg] 'longitude': request.instrument.telescope.lon, # telescope latitude [deg] 'latitude': request.instrument.telescope.lat, # telescope elevation [m] 'elevation': request.instrument.telescope.elevation, # telescope name 'telescope': request.instrument.name, # telescope horizon 'horizon': -12.0, # time in seconds to change the filter 'filt_change_time': 0.0, # extra overhead in seconds 'overhead_per_exposure': 0.0, # slew rate for the telescope [deg/s] 'slew_rate': 2.6, # camera readout time 'readout': 0.0, # telescope field of view 'FOV': 0.0, # exposure time for the given limiting magnitude 'exposuretime': 1.0, # limiting magnitude given telescope time 'magnitude': 0.0, }, }, # gwemopt filter strategy # options: block (blocks of single filters), integrated (series of alternating filters) 'doAlternativeFilters': request.payload["filter_strategy"] == "block", # flag to indicate fields come from DB 'doDatabase': True, # only keep tiles within powerlaw_cl 'doMinimalTiling': True, # single set of scheduled observations 'doSingleExposure': True, # gwemopt scheduling algorithms # options: greedy, greedy_slew, sear, airmass_weighted 'scheduleType': request.payload["schedule_type"], # list of filters to use for observations 'filters': request.payload["filters"].split(","), # GPS time for event 'gpstime': event_time.gps, # Healpix nside for the skymap 'nside': 512, # maximum integrated probability of the skymap to consider 'powerlaw_cl': request.payload["integrated_probability"], 'telescopes': [request.instrument.name], # minimum difference between observations of the same field 'mindiff': request.payload["minimum_time_difference"], # maximum airmass with which to observae 'airmass': request.payload["maximum_airmass"], # array of exposure times (same length as filter array) 'exposuretimes': np.array([int(request.payload["exposure_time"])] * len(request.payload["filters"].split(","))), } if request.payload["schedule_strategy"] == "galaxy": params = { **params, 'tilesType': 'galaxy', 'galaxy_catalog': request.payload["galaxy_catalog"], 'galaxy_grade': 'S', 'writeCatalog': False, 'catalog_n': 1.0, 'powerlaw_dist_exp': 1.0, } elif request.payload["schedule_strategy"] == "tiling": params = {**params, 'tilesType': 'moc'} else: raise AttributeError( 'scheduling_strategy should be tiling or galaxy') params = gwemopt.utils.params_checker(params) params = gwemopt.segments.get_telescope_segments(params) params["Tobs"] = [ start_time.mjd - event_time.mjd, end_time.mjd - event_time.mjd, ] params['map_struct'] = dict( zip(['prob', 'distmu', 'distsigma', 'distnorm'], request.localization.flat)) params['is3D'] = request.localization.is_3d # Function to read maps map_struct = gwemopt.utils.read_skymap(params, is3D=params["do3D"], map_struct=params['map_struct']) if params["tilesType"] == "galaxy": query = Galaxy.query_records_accessible_by(user, mode="read") query = query.filter( Galaxy.catalog_name == params["galaxy_catalog"]) galaxies = query.all() catalog_struct = {} catalog_struct["ra"] = np.array([g.ra for g in galaxies]) catalog_struct["dec"] = np.array([g.dec for g in galaxies]) catalog_struct["S"] = np.array([1.0 for g in galaxies]) catalog_struct["Sloc"] = np.array([1.0 for g in galaxies]) catalog_struct["Smass"] = np.array([1.0 for g in galaxies]) if params["tilesType"] == "moc": moc_structs = gwemopt.skyportal.create_moc_from_skyportal( params, map_struct=map_struct) tile_structs = gwemopt.tiles.moc(params, map_struct, moc_structs) elif params["tilesType"] == "galaxy": if request.instrument.region is None: raise ValueError( 'Must define the instrument region in the case of galaxy requests' ) regions = Regions.parse(request.instrument.region, format='ds9') tile_structs = gwemopt.skyportal.create_galaxy_from_skyportal( params, map_struct, catalog_struct, regions=regions) tile_structs, coverage_struct = gwemopt.coverage.timeallocation( params, map_struct, tile_structs) # if the fields do not yet exist, we need to add them if params["tilesType"] == "galaxy": regions = Regions.parse(request.instrument.region, format='ds9') data = { 'RA': coverage_struct["data"][:, 0], 'Dec': coverage_struct["data"][:, 1], } field_data = pd.DataFrame.from_dict(data) field_ids = add_tiles( request.instrument.id, request.instrument.name, regions, field_data, session=session, ) planned_observations = [] for ii in range(len(coverage_struct["ipix"])): data = coverage_struct["data"][ii, :] filt = coverage_struct["filters"][ii] mjd = data[2] tt = Time(mjd, format='mjd') overhead_per_exposure = params["config"][ request.instrument.name]["overhead_per_exposure"] exposure_time, prob = data[4], data[6] if params["tilesType"] == "galaxy": field_id = field_ids[ii] else: field_id = data[5] field = InstrumentField.query.filter( InstrumentField.instrument_id == request.instrument.id, InstrumentField.field_id == field_id, ).first() if field is None: return log(f"Missing field {field_id} from list") planned_observation = PlannedObservation( obstime=tt.datetime, dateobs=request.gcnevent.dateobs, field_id=field.id, exposure_time=exposure_time, weight=prob, filt=filt, instrument_id=request.instrument.id, planned_observation_id=ii, observation_plan_id=plan.id, overhead_per_exposure=overhead_per_exposure, ) planned_observations.append(planned_observation) session.add_all(planned_observations) plan.status = 'complete' session.merge(plan) session.commit() request.status = 'complete' session.merge(request) session.commit() flow = Flow() flow.push( '*', "skyportal/REFRESH_GCNEVENT", payload={"gcnEvent_dateobs": request.gcnevent.dateobs}, ) return log( f"Generated plan for observation plan {observation_plan_id}") except Exception as e: return log( f"Unable to generate plan for observation plan {observation_plan_id}: {e}" ) finally: Session.remove()
def commit_photometry(json_response, altdata, request_id, instrument_id, user_id): """ Commits ATLAS photometry to the database Parameters ---------- json_response : dict response.json() from call to ATLAS photometry service. altdata: dict Contains ATLAS photometry api_token for the user request_id : int FollowupRequest SkyPortal ID instrument_id : int Instrument SkyPortal ID user_id : int User SkyPortal ID """ from ..models import ( DBSession, FollowupRequest, Instrument, User, ) Session = scoped_session( sessionmaker(bind=DBSession.session_factory.kw["bind"])) session = Session() try: request = session.query(FollowupRequest).get(request_id) instrument = session.query(Instrument).get(instrument_id) user = session.query(User).get(user_id) result_url = json_response['result_url'] request.status = f"Task is complete with results available at {result_url}" s = requests.get( result_url, headers={ 'Authorization': f"Token {altdata['api_token']}", 'Accept': 'application/json', }, ) s.raise_for_status() # ATLAS response looks like """ ###MJD m dm uJy duJy F err chi/N RA Dec x y maj min phi apfit mag5sig Sky Obs 59226.235875 16.177 0.012 1228 15 c 0 54.64 342.45960 51.26340 7768.79 7767.00 2.53 2.39 -63.4 -0.375 19.58 21.54 01a59226o0051c 59228.242600 16.258 0.017 1140 20 c 0 7.87 342.45960 51.26340 2179.59 9252.78 3.41 3.09 -51.0 -0.396 19.28 21.28 02a59228o0102c 59228.246262 16.582 0.021 846 18 c 0 28.37 342.45960 51.26340 2162.23 9213.32 3.53 3.25 -52.3 -0.366 19.14 21.26 02a59228o0110c 59228.252679 16.451 0.019 954 18 c 0 13.76 342.45960 51.26340 2218.02 9291.76 3.34 3.03 -49.8 -0.389 19.17 21.24 02a59228o0124c 59228.265532 17.223 0.049 469 23 c 0 3.90 342.45960 51.26340 2237.25 9167.94 4.31 3.88 -43.7 -0.473 18.95 21.20 02a59228o0152c """ try: df = pd.read_csv(StringIO(s.text.replace("###MJD", "mjd")), delim_whitespace=True) except Exception as e: raise ValueError(f'Format of response not understood: {e.message}') desired_columns = {'mjd', 'RA', 'Dec', 'm', 'dm', 'mag5sig', 'F'} if not desired_columns.issubset(set(df.columns)): raise ValueError('Missing expected column') df.rename( columns={ 'RA': 'ra', 'Dec': 'dec', 'm': 'mag', 'dm': 'magerr', 'mag5sig': 'limiting_mag', 'F': 'filter', }, inplace=True, ) cyan = df['filter'] == 'c' orange = df['filter'] == 'o' snr = df['uJy'] / df['duJy'] < 5 df.loc[cyan, 'filter'] = 'atlasc' df.loc[orange, 'filter'] = 'atlaso' df.loc[snr, 'mag'] = None df.loc[snr, 'magerr'] = None iszero = df['duJy'] == 0.0 df.loc[iszero, 'mag'] = None df.loc[iszero, 'magerr'] = None isnan = np.isnan(df['uJy']) df.loc[isnan, 'mag'] = None df.loc[isnan, 'magerr'] = None df = df.replace({np.nan: None}) drop_columns = list( set(df.columns.values) - {'mjd', 'ra', 'dec', 'mag', 'magerr', 'limiting_mag', 'filter'}) df.drop( columns=drop_columns, inplace=True, ) df['magsys'] = 'ab' data_out = { 'obj_id': request.obj_id, 'instrument_id': instrument.id, 'group_ids': [g.id for g in user.accessible_groups], **df.to_dict(orient='list'), } from skyportal.handlers.api.photometry import add_external_photometry if len(df.index) > 0: add_external_photometry(data_out, request.requester) request.status = "Photometry committed to database" else: request.status = "No photometry to commit to database" session.add(request) session.commit() flow = Flow() flow.push( '*', "skyportal/REFRESH_SOURCE", payload={"obj_key": request.obj.internal_key}, ) except Exception as e: return log(f"Unable to commit photometry for {request_id}: {e}")
def add_observations(instrument_id, obstable): """Post executed observations for a given instrument. obstable is a pandas DataFrame of the form: observation_id field_id obstime seeing limmag exposure_time \ 0 84434604 1 2.458599e+06 1.57415 20.40705 30 1 84434651 1 2.458599e+06 1.58120 20.49405 30 2 84434696 1 2.458599e+06 1.64995 20.56030 30 3 84434741 1 2.458599e+06 1.54945 20.57400 30 4 84434788 1 2.458599e+06 1.62870 20.60385 30 filter processed_fraction airmass 0 ztfr 1.0 None 1 ztfr 1.0 None 2 ztfr 1.0 None 3 ztfr 1.0 None 4 ztfr 1.0 None """ session = Session() # if the fields do not yet exist, we need to add them if ('RA' in obstable) and ('Dec' in obstable) and not ('field_id' in obstable): instrument = session.query(Instrument).get(instrument_id) regions = Regions.parse(instrument.region, format='ds9') field_data = obstable[['RA', 'Dec']] field_ids = add_tiles(instrument.id, instrument.name, regions, field_data, session=session) obstable['field_id'] = field_ids try: observations = [] for index, row in obstable.iterrows(): field_id = int(row["field_id"]) field = (session.query(InstrumentField).filter( InstrumentField.instrument_id == instrument_id, InstrumentField.field_id == field_id, ).first()) if field is None: return log( f"Unable to add observations for instrument {instrument_id}: Missing field {field_id}" ) observation = (session.query(ExecutedObservation).filter_by( instrument_id=instrument_id, observation_id=row["observation_id"]).first()) if observation is not None: log(f"Observation {row['observation_id']} for instrument {instrument_id} already exists... continuing." ) continue # enable multiple obstime formats try: # can catch iso and isot this way obstime = Time(row["obstime"]) except ValueError: # otherwise catch jd as the numerical example obstime = Time(row["obstime"], format='jd') observations.append( ExecutedObservation( instrument_id=instrument_id, observation_id=row["observation_id"], instrument_field_id=field.id, obstime=obstime.datetime, seeing=row["seeing"], limmag=row["limmag"], exposure_time=row["exposure_time"], filt=row["filter"], processed_fraction=row["processed_fraction"], )) session.add_all(observations) session.commit() flow = Flow() flow.push('*', "skyportal/REFRESH_OBSERVATIONS") return log( f"Successfully added observations for instrument {instrument_id}") except Exception as e: return log( f"Unable to add observations for instrument {instrument_id}: {e}") finally: Session.remove()
def commit_photometry(text_response, request_id, instrument_id, user_id): """ Commits PS1 DR2 photometry to the database Parameters ---------- text_response : dict response.text from call to PS1 DR2 photometry service. request_id : int FollowupRequest SkyPortal ID instrument_id : int Instrument SkyPortal ID user_id : int User SkyPortal ID """ from ..models import ( DBSession, FollowupRequest, Instrument, User, ) Session = scoped_session( sessionmaker(bind=DBSession.session_factory.kw["bind"])) session = Session() try: request = session.query(FollowupRequest).get(request_id) instrument = session.query(Instrument).get(instrument_id) user = session.query(User).get(user_id) tab = astropy.io.ascii.read(text_response) # good data only tab = tab[tab['psfQfPerfect'] > 0.9] id2filter = np.array( ['ps1::g', 'ps1::r', 'ps1::i', 'ps1::z', 'ps1::y']) tab['filter'] = id2filter[(tab['filterID'] - 1).data.astype(int)] df = tab.to_pandas() df.rename( columns={ 'obsTime': 'mjd', 'psfFlux': 'flux', 'psfFluxerr': 'fluxerr', }, inplace=True, ) df = df.replace({np.nan: None}) df.drop( columns=[ 'detectID', 'filterID', 'psfQfPerfect', ], inplace=True, ) df['magsys'] = 'ab' df['zp'] = 8.90 data_out = { 'obj_id': request.obj_id, 'instrument_id': instrument.id, 'group_ids': [g.id for g in user.accessible_groups], **df.to_dict(orient='list'), } from skyportal.handlers.api.photometry import add_external_photometry if len(df.index) > 0: add_external_photometry(data_out, request.requester) request.status = "Photometry committed to database" else: request.status = "No photometry to commit to database" session.add(request) session.commit() flow = Flow() flow.push( '*', "skyportal/REFRESH_SOURCE", payload={"obj_key": request.obj.internal_key}, ) except Exception as e: return log(f"Unable to commit photometry for {request_id}: {e}")
def delete(request): """Delete a follow-up request from SEDMv2 queue. Parameters ---------- request: skyportal.models.FollowupRequest The request to delete from the queue and the SkyPortal database. """ from ..models import DBSession, FollowupRequest, FacilityTransaction if cfg['app.sedmv2_endpoint'] is not None: altdata = request.allocation.altdata req = ( DBSession() .query(FollowupRequest) .filter(FollowupRequest.id == request.id) .one() ) altdata = request.allocation.altdata if not altdata: raise ValueError('Missing allocation information.') content = req.transactions[0].response["content"] content = json.loads(content) uid = content["data"]["id"] r = requests.delete( f"{cfg['app.sedmv2_endpoint']}/{uid}", headers={"Authorization": f"token {altdata['api_token']}"}, ) r.raise_for_status() request.status = "deleted" transaction = FacilityTransaction( request=http.serialize_requests_request(r.request), response=http.serialize_requests_response(r), followup_request=request, initiator_id=request.last_modified_by_id, ) else: request.status = 'deleted' transaction = FacilityTransaction( request=None, response=None, followup_request=request, initiator_id=request.last_modified_by_id, ) DBSession().add(transaction) flow = Flow() flow.push( '*', 'skyportal/REFRESH_SOURCE', payload={'obj_key': request.obj.internal_key}, )
async def put(self): """ --- description: | Reprioritize followup requests schedule automatically based on location within skymap. tags: - followup_requests parameters: - in: body name: localizationId schema: type: integer description: Filter by localization ID - in: body name: requestIds schema: type: list of integers description: List of follow-up request IDs - in: body name: minimumPriority schema: type: string description: Minimum priority for the instrument. Defaults to 1. - in: body name: maximumPriority schema: type: string description: Maximum priority for the instrument. Defaults to 5. responses: 200: content: application/json: schema: Success 400: content: application/json: schema: Error """ data = self.get_json() localization_id = data.get('localizationId', None) request_ids = data.get('requestIds', None) minimum_priority = data.get('minimumPriority', 1) maximum_priority = data.get('maximumPriority', 5) if localization_id is None: return self.error('localizationId is required') if request_ids is None: return self.error('requestIds is required') localization = (Localization.query_records_accessible_by( self.current_user).filter( Localization.id == localization_id, ).first()) if localization is None: return self.error( message=f"Missing localization with id {localization_id}") followup_requests = [] for request_id in request_ids: # get owned assignments followup_request = FollowupRequest.get_if_accessible_by( request_id, self.current_user, mode="update") if followup_request is None: return self.error( message=f"Missing FollowUpRequest with id {request_id}") followup_requests.append(followup_request) if len(followup_requests) == 0: return self.error('Need at least one observation to modify.') ras = np.array([ followup_request.obj.ra for followup_request in followup_requests ]) decs = np.array([ followup_request.obj.dec for followup_request in followup_requests ]) dists = np.array([ cosmo.luminosity_distance(followup_request.obj.redshift).value if followup_request.obj.redshift is not None else -1 for followup_request in followup_requests ]) tab = localization.flat ipix = hp.ang2pix(Localization.nside, ras, decs, lonlat=True) if localization.is_3d: prob, distmu, distsigma, distnorm = tab if not all([dist > 0 for dist in dists]): weights = prob[ipix] else: weights = prob[ipix] * (distnorm[ipix] * norm( distmu[ipix], distsigma[ipix]).pdf(dists)) else: weights = prob[ipix] weights = weights / np.max(weights) priorities = [ int( np.round(weight * (maximum_priority - minimum_priority) + minimum_priority)) for weight in weights ] with DBSession() as session: for request_id, priority in zip(request_ids, priorities): # get owned assignments followup_request = session.query(FollowupRequest).get( request_id) api = followup_request.instrument.api_class if not api.implements()['update']: return self.error( 'Cannot update requests on this instrument.') payload = followup_request.payload payload["priority"] = priority session.query(FollowupRequest).filter( FollowupRequest.id == request_id).update( {'payload': payload}) session.commit() followup_request.payload = payload followup_request.instrument.api_class.update(followup_request) flow = Flow() flow.push( '*', "skyportal/REFRESH_FOLLOWUP_REQUESTS", ) return self.success()
def commit_photometry(url, altdata, df_request, request_id, instrument_id, user_id): """ Commits ZTF forced photometry to the database Parameters ---------- url : str ZTF forced photometry service data file location. altdata: dict Contains ZTF photometry api_token for the user df_request: pandas.DataFrame DataFrame containing request parameters (ra, dec, start jd, end jd) request_id : int FollowupRequest SkyPortal ID instrument_id : int Instrument SkyPortal ID user_id: int User SkyPortal ID """ from ..models import ( DBSession, FollowupRequest, Instrument, User, ) Session = scoped_session( sessionmaker(bind=DBSession.session_factory.kw["bind"])) session = Session() try: request = session.query(FollowupRequest).get(request_id) instrument = session.query(Instrument).get(instrument_id) user = session.query(User).get(user_id) r = requests.get( url, auth=HTTPBasicAuth(altdata['ipac_http_user'], altdata['ipac_http_password']), ) df = ascii.read(r.content.decode(), header_start=0, data_start=1, comment='#').to_pandas() df.columns = df.columns.str.replace(',', '') desired_columns = { 'jd', 'forcediffimflux', 'forcediffimfluxunc', 'diffmaglim', 'zpdiff', 'filter', } if not desired_columns.issubset(set(df.columns)): raise ValueError('Missing expected column') df['ra'] = df_request['ra'] df['dec'] = df_request['dec'] df.rename( columns={'diffmaglim': 'limiting_mag'}, inplace=True, ) df = df.replace({"null": np.nan}) df['mjd'] = astropy.time.Time(df['jd'], format='jd').mjd df['filter'] = df['filter'].str.replace('_', '') df['filter'] = df['filter'].str.lower() df = df.astype({ 'forcediffimflux': 'float64', 'forcediffimfluxunc': 'float64' }) df['mag'] = df['zpdiff'] - 2.5 * np.log10(df['forcediffimflux']) df['magerr'] = 1.0857 * df['forcediffimfluxunc'] / df['forcediffimflux'] snr = df['forcediffimflux'] / df['forcediffimfluxunc'] < 5 df['mag'].loc[snr] = None df['magerr'].loc[snr] = None iszero = df['forcediffimfluxunc'] == 0.0 df['mag'].loc[iszero] = None df['magerr'].loc[iszero] = None isnan = np.isnan(df['forcediffimflux']) df['mag'].loc[isnan] = None df['magerr'].loc[isnan] = None df = df.replace({np.nan: None}) drop_columns = list( set(df.columns.values) - set([ 'mjd', 'ra', 'dec', 'mag', 'magerr', 'limiting_mag', 'filter' ])) df.drop( columns=drop_columns, inplace=True, ) df['magsys'] = 'ab' data_out = { 'obj_id': request.obj_id, 'instrument_id': instrument.id, 'group_ids': [g.id for g in user.accessible_groups], **df.to_dict(orient='list'), } from skyportal.handlers.api.photometry import add_external_photometry add_external_photometry(data_out, request.requester) request.status = "Photometry committed to database" session.add(request) session.commit() flow = Flow() flow.push( '*', "skyportal/REFRESH_SOURCE", payload={"obj_key": request.obj.internal_key}, ) except Exception as e: return log(f"Unable to commit photometry for {request_id}: {e}") finally: Session.remove()
def receive_after_flush(session, context): is_gcnnotice = "dateobs" in target.to_dict() is_facility_transaction = "initiator_id" in target.to_dict() if is_gcnnotice: users = User.query.filter(User.preferences["slack_integration"] ["gcnnotices"].astext.cast( sa.Boolean).is_(True)).all() elif is_facility_transaction: users = User.query.filter(User.preferences["slack_integration"] ["facilitytransactions"].astext.cast( sa.Boolean).is_(True)).all() else: listing_subquery = (Listing.query.filter( Listing.list_name == "favorites").filter( Listing.obj_id == target.obj_id).distinct( Listing.user_id).subquery()) users = (User.query.join( listing_subquery, User.id == listing_subquery.c.user_id).filter( User.preferences["favorite_sources_activity_notifications"] [target.__tablename__].astext.cast( sa.Boolean).is_(True)).all()) ws_flow = Flow() for user in users: # Only notify users who have read access to the new record in question if target.__class__.get_if_accessible_by(target.id, user) is not None: if is_gcnnotice: session.add( UserNotification( user=user, text= f"New {target.__class__.__name__.lower()} on GcnEvent *{target.dateobs}*", url= f"/gcn_events/{str(target.dateobs).replace(' ','T')}", )) elif is_facility_transaction: if "observation_plan_request" in target.to_dict(): allocation_id = target.observation_plan_request.allocation_id allocation = session.query(Allocation).get( allocation_id) instrument = allocation.instrument localization_id = ( target.observation_plan_request.localization_id) localization = session.query(Localization).get( localization_id) session.add( UserNotification( user=user, text= f"New observation plan submission for GcnEvent *{localization.dateobs}* by *{instrument.name}*", url= f"/gcn_events/{str(localization.dateobs).replace(' ','T')}", )) elif "followup_request" in target.to_dict(): allocation_id = target.followup_request.allocation_id allocation = session.query(Allocation).get( allocation_id) instrument = allocation.instrument session.add( UserNotification( user=user, text= f"New follow-up submission for object *{target.followup_request.obj_id}* by *{instrument.name}*", url=f"/source/{target.followup_request.obj_id}", )) else: session.add( UserNotification( user=user, text= f"New {target.__class__.__name__.lower()} on your favorite source *{target.obj_id}*", url=f"/source/{target.obj_id}", )) ws_flow.push(user.id, "skyportal/FETCH_NOTIFICATIONS")