def setUp(self):
self.broker = MARSBroker()
self.expected_keys = [
'avro', 'candid', 'candidate', 'lco_id', 'objectId', 'publisher'
]
self.expected_candidate_keys = [
'aimage', 'aimagerat', 'b', 'bimage', 'bimagerat', 'candid',
'chinr', 'chipsf', 'classtar', 'clrcoeff', 'clrcounc', 'clrmed',
'clrrms', 'dec', 'decnr', 'deltamaglatest', 'deltamagref',
'diffmaglim', 'distnr', 'distpsnr1', 'distpsnr2', 'distpsnr3',
'drb', 'drbversion', 'dsdiff', 'dsnrms', 'elong', 'exptime', 'fid',
'field', 'filter', 'fwhm', 'isdiffpos', 'jd', 'jdendhist',
'jdendref', 'jdstarthist', 'jdstartref', 'l', 'magap', 'magapbig',
'magdiff', 'magfromlim', 'maggaia', 'maggaiabright', 'magnr',
'magpsf', 'magzpsci', 'magzpscirms', 'magzpsciunc', 'mindtoedge',
'nbad', 'ncovhist', 'ndethist', 'neargaia', 'neargaiabright',
'nframesref', 'nid', 'nmatches', 'nmtchps', 'nneg', 'objectidps1',
'objectidps2', 'objectidps3', 'pdiffimfilename', 'pid',
'programid', 'programpi', 'ra', 'ranr', 'rb', 'rbversion', 'rcid',
'rfid', 'scorr', 'seeratio', 'sgmag1', 'sgmag2', 'sgmag3',
'sgscore1', 'sgscore2', 'sgscore3', 'sharpnr', 'sigmagap',
'sigmagapbig', 'sigmagnr', 'sigmapsf', 'simag1', 'simag2',
'simag3', 'sky', 'srmag1', 'srmag2', 'srmag3', 'ssdistnr',
'ssmagnr', 'ssnamenr', 'ssnrms', 'sumrat', 'szmag1', 'szmag2',
'szmag3', 'tblid', 'tooflag', 'wall_time', 'xpos', 'ypos',
'zpclrcov', 'zpmed'
]
class TestMARSModuleCanary(TestCase):
def setUp(self):
self.broker = MARSBroker()
self.expected_keys = ['avro', 'candid', 'candidate', 'lco_id', 'objectId', 'publisher']
self.expected_candidate_keys = ['aimage', 'aimagerat', 'b', 'bimage', 'bimagerat', 'candid', 'chinr', 'chipsf',
'classtar', 'clrcoeff', 'clrcounc', 'clrmed', 'clrrms', 'dec', 'decnr',
'deltamaglatest', 'deltamagref', 'diffmaglim', 'distnr', 'distpsnr1',
'distpsnr2', 'distpsnr3', 'drb', 'drbversion', 'dsdiff', 'dsnrms', 'elong',
'exptime', 'fid', 'field', 'filter', 'fwhm', 'isdiffpos', 'jd', 'jdendhist',
'jdendref', 'jdstarthist', 'jdstartref', 'l', 'magap', 'magapbig', 'magdiff',
'magfromlim', 'maggaia', 'maggaiabright', 'magnr', 'magpsf', 'magzpsci',
'magzpscirms', 'magzpsciunc', 'mindtoedge', 'nbad', 'ncovhist', 'ndethist',
'neargaia', 'neargaiabright', 'nframesref', 'nid', 'nmatches', 'nmtchps',
'nneg', 'objectidps1', 'objectidps2', 'objectidps3', 'pdiffimfilename', 'pid',
'programid', 'programpi', 'ra', 'ranr', 'rb', 'rbversion', 'rcid', 'rfid',
'scorr', 'seeratio', 'sgmag1', 'sgmag2', 'sgmag3', 'sgscore1', 'sgscore2',
'sgscore3', 'sharpnr', 'sigmagap', 'sigmagapbig', 'sigmagnr', 'sigmapsf',
'simag1', 'simag2', 'simag3', 'sky', 'srmag1', 'srmag2', 'srmag3', 'ssdistnr',
'ssmagnr', 'ssnamenr', 'ssnrms', 'sumrat', 'szmag1', 'szmag2', 'szmag3',
'tblid', 'tooflag', 'wall_time', 'xpos', 'ypos', 'zpclrcov', 'zpmed']
def test_fetch_alerts(self):
response = self.broker.fetch_alerts({'time__gt': '2018-06-01', 'time__lt': '2018-06-30'})
alerts = []
for alert in islice(response, 10):
alerts.append(alert)
self.assertEqual(len(alerts), 10)
for key in self.expected_keys:
self.assertTrue(key in alerts[0].keys())
for key in self.expected_candidate_keys:
self.assertTrue(key in alerts[0]['candidate'].keys())
def test_fetch_alert(self):
alert = self.broker.fetch_alert(1065519)
for key in self.expected_keys:
self.assertTrue(key in alert.keys())
for key in self.expected_candidate_keys:
self.assertTrue(key in alert['candidate'].keys())
def test_process_reduced_data(self):
alert = self.broker.fetch_alert(1065519)
t = Target.objects.create(name='test target', ra=1, dec=2)
self.broker.process_reduced_data(t, alert=alert)
self.assertGreaterEqual(ReducedDatum.objects.filter(target=t, timestamp__lte=datetime(2020, 11, 3)).count(),
526)
def test_process_reduced_data_with_alert(self):
test_alert = self.test_data[1]
test_alert['prv_candidate'] = [{
'candidate': {
'jd': 2458372.6225231,
'magpsf': 13,
'fid': 0
}
}]
MARSBroker().process_reduced_data(self.test_target, alert=test_alert)
reduced_data = ReducedDatum.objects.filter(target=self.test_target,
source_name='MARS')
self.assertEqual(reduced_data.count(), 2)
def test_fetch_alerts(self, mock_requests_get):
mock_return_data = {
"has_next": "false",
"has_prev": "false",
"pages": 1,
"results": [self.test_data[1]]
}
mock_response = Response()
mock_response._content = str.encode(json.dumps(mock_return_data))
mock_response.status_code = 200
mock_requests_get.return_value = mock_response
alerts = MARSBroker().fetch_alerts({'objectId': 'ZTF18aberpsh'})
self.assertEqual(self.test_data[1]['objectId'], list(alerts)[0]['objectId'])
def test_process_reduced_data_no_alert(self, mock_fetch_alert):
self.test_data = self.test_data[1]
self.test_data['prv_candidate'] = [{
'candidate': {
'jd': 2458372.6225231,
'magpsf': 13,
'fid': 0
}
}]
mock_fetch_alert.return_value = self.test_data
MARSBroker().process_reduced_data(self.test_target)
reduced_data = ReducedDatum.objects.filter(target=self.test_target,
source_name='MARS')
self.assertEqual(reduced_data.count(), 2)
def test_to_generic_alert(self):
test_alert = self.test_data[1]
created_alert = MARSBroker().to_generic_alert(test_alert)
self.assertEqual(created_alert.name, 'ZTF18aberpsh')
def test_to_target(self):
test_alert = self.test_data[0]
created_target = MARSBroker().to_target(test_alert)
self.assertEqual(created_target.name, 'ZTF18abbkloa')
def fetch_alerts(self, ):
mars = MARSBroker()
#download from ZTF web server (i.e. Przemek webpage)
ztf_ipac = urllib.request.urlopen(BROKER_URL).readlines()
list_of_events = [
str(i)[6:-8] for i in ztf_ipac if '<td>ZTF' in str(i)
]
list_of_mars_links = [
str(i).split('"')[1] for i in ztf_ipac
if '<td><a href="https://mars.lco.global/' in str(i)
]
for index, event in enumerate(list_of_events):
MARS_candidates = requests.get(list_of_mars_links[index] +
'&format=json').json()
if len(MARS_candidates['results']) == 0:
pass
else:
cone_search = str(MARS_candidates['results'][0]['candidate']
['ra']) + ',' + str(
MARS_candidates['results'][0]
['candidate']['dec']) + ',' + str(0.0003)
mars_form = MARSQueryForm({
'cone': cone_search,
'query_name': 'Query ZTF IPAC : ' + event,
'broker': 'MARS'
})
mars_form.is_valid()
query = BrokerQuery.objects.create(
name='Query ZTF IPAC : ' + event,
broker=mars.name,
parameters=mars_form.cleaned_data)
alerts = mars.fetch_alerts(query.parameters)
alerts = [*alerts]
name = event
ra = np.median([alert['candidate']['ra'] for alert in alerts])
dec = np.median(
[alert['candidate']['dec'] for alert in alerts])
try:
target = Target.objects.get(name=name)
except:
target, created = Target.objects.get_or_create(
name=name, ra=ra, dec=dec, type='SIDEREAL', epoch=2000)
if created:
target.save()
filters = {1: 'g_ZTF', 2: 'r_ZTF', 3: 'i_ZTF'}
try:
times = [
Time(i.timestamp).jd
for i in ReducedDatum.objects.filter(target=target)
if i.data_type == 'photometry'
]
except:
times = []
for alert in alerts:
if all([
key in alert['candidate']
for key in ['jd', 'magpsf', 'fid', 'sigmapsf']
]):
jd = Time(alert['candidate']['jd'],
format='jd',
scale='utc')
jd.to_datetime(timezone=TimezoneInfo())
if alert['candidate']['isdiffpos'] == 't':
signe = 1
else:
signe = -1
flux = 10**(
-0.4 * alert['candidate']['magnr']
) + signe * 10**(-0.4 * alert['candidate']['magpsf'])
eflux = ((10**(-0.4 * alert['candidate']['magnr']) *
alert['candidate']['sigmagnr'])**2 +
(signe *
10**(-0.4 * alert['candidate']['magpsf']) *
alert['candidate']['sigmapsf'])**2)**0.5
mag = -2.5 * np.log10(flux)
emag = eflux / flux
value = {
'magnitude': mag,
'filter': filters[alert['candidate']['fid']],
'error': emag
}
if (jd.value not in times):
rd, _ = ReducedDatum.objects.get_or_create(
timestamp=jd.to_datetime(
timezone=TimezoneInfo()),
value=value,
source_name='ZTF IPAC',
source_location='IRSA',
data_type='photometry',
target=target)
rd.save()
else:
pass
else:
pass