def commit_photometry(json_response, altdata, request_id, instrument_id,
user_id):
"""
Commits ATLAS photometry to the database
Parameters
----------
json_response : dict
response.json() from call to ATLAS photometry service.
altdata: dict
Contains ATLAS photometry api_token for the user
request_id : int
FollowupRequest SkyPortal ID
instrument_id : int
Instrument SkyPortal ID
user_id : int
User SkyPortal ID
"""
from ..models import (
DBSession,
FollowupRequest,
Instrument,
User,
)
Session = scoped_session(
sessionmaker(bind=DBSession.session_factory.kw["bind"]))
session = Session()
try:
request = session.query(FollowupRequest).get(request_id)
instrument = session.query(Instrument).get(instrument_id)
user = session.query(User).get(user_id)
result_url = json_response['result_url']
request.status = f"Task is complete with results available at {result_url}"
s = requests.get(
result_url,
headers={
'Authorization': f"Token {altdata['api_token']}",
'Accept': 'application/json',
},
)
s.raise_for_status()
# ATLAS response looks like
"""
###MJD m dm uJy duJy F err chi/N RA Dec x y maj min phi apfit mag5sig Sky Obs
59226.235875 16.177 0.012 1228 15 c 0 54.64 342.45960 51.26340 7768.79 7767.00 2.53 2.39 -63.4 -0.375 19.58 21.54 01a59226o0051c
59228.242600 16.258 0.017 1140 20 c 0 7.87 342.45960 51.26340 2179.59 9252.78 3.41 3.09 -51.0 -0.396 19.28 21.28 02a59228o0102c
59228.246262 16.582 0.021 846 18 c 0 28.37 342.45960 51.26340 2162.23 9213.32 3.53 3.25 -52.3 -0.366 19.14 21.26 02a59228o0110c
59228.252679 16.451 0.019 954 18 c 0 13.76 342.45960 51.26340 2218.02 9291.76 3.34 3.03 -49.8 -0.389 19.17 21.24 02a59228o0124c
59228.265532 17.223 0.049 469 23 c 0 3.90 342.45960 51.26340 2237.25 9167.94 4.31 3.88 -43.7 -0.473 18.95 21.20 02a59228o0152c
"""
try:
df = pd.read_csv(StringIO(s.text.replace("###MJD", "mjd")),
delim_whitespace=True)
except Exception as e:
raise ValueError(f'Format of response not understood: {e.message}')
desired_columns = {'mjd', 'RA', 'Dec', 'm', 'dm', 'mag5sig', 'F'}
if not desired_columns.issubset(set(df.columns)):
raise ValueError('Missing expected column')
df.rename(
columns={
'RA': 'ra',
'Dec': 'dec',
'm': 'mag',
'dm': 'magerr',
'mag5sig': 'limiting_mag',
'F': 'filter',
},
inplace=True,
)
cyan = df['filter'] == 'c'
orange = df['filter'] == 'o'
snr = df['uJy'] / df['duJy'] < 5
df.loc[cyan, 'filter'] = 'atlasc'
df.loc[orange, 'filter'] = 'atlaso'
df.loc[snr, 'mag'] = None
df.loc[snr, 'magerr'] = None
iszero = df['duJy'] == 0.0
df.loc[iszero, 'mag'] = None
df.loc[iszero, 'magerr'] = None
isnan = np.isnan(df['uJy'])
df.loc[isnan, 'mag'] = None
df.loc[isnan, 'magerr'] = None
df = df.replace({np.nan: None})
drop_columns = list(
set(df.columns.values) -
{'mjd', 'ra', 'dec', 'mag', 'magerr', 'limiting_mag', 'filter'})
df.drop(
columns=drop_columns,
inplace=True,
)
df['magsys'] = 'ab'
data_out = {
'obj_id': request.obj_id,
'instrument_id': instrument.id,
'group_ids': [g.id for g in user.accessible_groups],
**df.to_dict(orient='list'),
}
from skyportal.handlers.api.photometry import add_external_photometry
if len(df.index) > 0:
add_external_photometry(data_out, request.requester)
request.status = "Photometry committed to database"
else:
request.status = "No photometry to commit to database"
session.add(request)
session.commit()
flow = Flow()
flow.push(
'*',
"skyportal/REFRESH_SOURCE",
payload={"obj_key": request.obj.internal_key},
)
except Exception as e:
return log(f"Unable to commit photometry for {request_id}: {e}")
def commit_photometry(text_response, request_id, instrument_id, user_id):
"""
Commits PS1 DR2 photometry to the database
Parameters
----------
text_response : dict
response.text from call to PS1 DR2 photometry service.
request_id : int
FollowupRequest SkyPortal ID
instrument_id : int
Instrument SkyPortal ID
user_id : int
User SkyPortal ID
"""
from ..models import (
DBSession,
FollowupRequest,
Instrument,
User,
)
Session = scoped_session(
sessionmaker(bind=DBSession.session_factory.kw["bind"]))
session = Session()
try:
request = session.query(FollowupRequest).get(request_id)
instrument = session.query(Instrument).get(instrument_id)
user = session.query(User).get(user_id)
tab = astropy.io.ascii.read(text_response)
# good data only
tab = tab[tab['psfQfPerfect'] > 0.9]
id2filter = np.array(
['ps1::g', 'ps1::r', 'ps1::i', 'ps1::z', 'ps1::y'])
tab['filter'] = id2filter[(tab['filterID'] - 1).data.astype(int)]
df = tab.to_pandas()
df.rename(
columns={
'obsTime': 'mjd',
'psfFlux': 'flux',
'psfFluxerr': 'fluxerr',
},
inplace=True,
)
df = df.replace({np.nan: None})
df.drop(
columns=[
'detectID',
'filterID',
'psfQfPerfect',
],
inplace=True,
)
df['magsys'] = 'ab'
df['zp'] = 8.90
data_out = {
'obj_id': request.obj_id,
'instrument_id': instrument.id,
'group_ids': [g.id for g in user.accessible_groups],
**df.to_dict(orient='list'),
}
from skyportal.handlers.api.photometry import add_external_photometry
if len(df.index) > 0:
add_external_photometry(data_out, request.requester)
request.status = "Photometry committed to database"
else:
request.status = "No photometry to commit to database"
session.add(request)
session.commit()
flow = Flow()
flow.push(
'*',
"skyportal/REFRESH_SOURCE",
payload={"obj_key": request.obj.internal_key},
)
except Exception as e:
return log(f"Unable to commit photometry for {request_id}: {e}")
def commit_photometry(url, altdata, df_request, request_id, instrument_id,
user_id):
"""
Commits ZTF forced photometry to the database
Parameters
----------
url : str
ZTF forced photometry service data file location.
altdata: dict
Contains ZTF photometry api_token for the user
df_request: pandas.DataFrame
DataFrame containing request parameters (ra, dec, start jd, end jd)
request_id : int
FollowupRequest SkyPortal ID
instrument_id : int
Instrument SkyPortal ID
user_id: int
User SkyPortal ID
"""
from ..models import (
DBSession,
FollowupRequest,
Instrument,
User,
)
Session = scoped_session(
sessionmaker(bind=DBSession.session_factory.kw["bind"]))
session = Session()
try:
request = session.query(FollowupRequest).get(request_id)
instrument = session.query(Instrument).get(instrument_id)
user = session.query(User).get(user_id)
r = requests.get(
url,
auth=HTTPBasicAuth(altdata['ipac_http_user'],
altdata['ipac_http_password']),
)
df = ascii.read(r.content.decode(),
header_start=0,
data_start=1,
comment='#').to_pandas()
df.columns = df.columns.str.replace(',', '')
desired_columns = {
'jd',
'forcediffimflux',
'forcediffimfluxunc',
'diffmaglim',
'zpdiff',
'filter',
}
if not desired_columns.issubset(set(df.columns)):
raise ValueError('Missing expected column')
df['ra'] = df_request['ra']
df['dec'] = df_request['dec']
df.rename(
columns={'diffmaglim': 'limiting_mag'},
inplace=True,
)
df = df.replace({"null": np.nan})
df['mjd'] = astropy.time.Time(df['jd'], format='jd').mjd
df['filter'] = df['filter'].str.replace('_', '')
df['filter'] = df['filter'].str.lower()
df = df.astype({
'forcediffimflux': 'float64',
'forcediffimfluxunc': 'float64'
})
df['mag'] = df['zpdiff'] - 2.5 * np.log10(df['forcediffimflux'])
df['magerr'] = 1.0857 * df['forcediffimfluxunc'] / df['forcediffimflux']
snr = df['forcediffimflux'] / df['forcediffimfluxunc'] < 5
df['mag'].loc[snr] = None
df['magerr'].loc[snr] = None
iszero = df['forcediffimfluxunc'] == 0.0
df['mag'].loc[iszero] = None
df['magerr'].loc[iszero] = None
isnan = np.isnan(df['forcediffimflux'])
df['mag'].loc[isnan] = None
df['magerr'].loc[isnan] = None
df = df.replace({np.nan: None})
drop_columns = list(
set(df.columns.values) - set([
'mjd', 'ra', 'dec', 'mag', 'magerr', 'limiting_mag', 'filter'
]))
df.drop(
columns=drop_columns,
inplace=True,
)
df['magsys'] = 'ab'
data_out = {
'obj_id': request.obj_id,
'instrument_id': instrument.id,
'group_ids': [g.id for g in user.accessible_groups],
**df.to_dict(orient='list'),
}
from skyportal.handlers.api.photometry import add_external_photometry
add_external_photometry(data_out, request.requester)
request.status = "Photometry committed to database"
session.add(request)
session.commit()
flow = Flow()
flow.push(
'*',
"skyportal/REFRESH_SOURCE",
payload={"obj_key": request.obj.internal_key},
)
except Exception as e:
return log(f"Unable to commit photometry for {request_id}: {e}")
finally:
Session.remove()
def test_post_external_photometry(upload_data_token, super_admin_token,
super_admin_user, public_group):
obj_id = str(uuid.uuid4())
status, data = api(
"POST",
"sources",
data={
"id": obj_id,
"ra": 234.22,
"dec": -22.33,
"group_ids": [public_group.id],
},
token=upload_data_token,
)
assert status == 200
assert data["data"]["id"] == obj_id
name = str(uuid.uuid4())
status, data = api(
'POST',
'telescope',
data={
'name': name,
'nickname': name,
'lat': 0.0,
'lon': 0.0,
'elevation': 0.0,
'diameter': 10.0,
},
token=super_admin_token,
)
assert status == 200
assert data['status'] == 'success'
telescope_id = data['data']['id']
instrument_name = str(uuid.uuid4())
status, data = api(
'POST',
'instrument',
data={
'name': instrument_name,
'type': 'imager',
'band': 'NIR',
'filters': ['atlaso', 'atlasc'],
'telescope_id': telescope_id,
},
token=super_admin_token,
)
assert status == 200
assert data['status'] == 'success'
instrument_id = data['data']['id']
datafile = f'{os.path.dirname(__file__)}/../data/ZTFrlh6cyjh_ATLAS.csv'
df = pd.read_csv(datafile)
df.drop(columns=['index'], inplace=True)
data_out = {
'obj_id': obj_id,
'instrument_id': instrument_id,
'group_ids': 'all',
**df.to_dict(orient='list'),
}
add_external_photometry(data_out, super_admin_user)
# Check the photometry sent back with the source
status, data = api(
"GET",
f"sources/{obj_id}",
params={"includePhotometry": "true"},
token=super_admin_token,
)
assert status == 200
assert len(data["data"]["photometry"]) == 384
assert all(p['obj_id'] == obj_id for p in data["data"]["photometry"])
assert all(p['instrument_id'] == instrument_id
for p in data["data"]["photometry"])