class TestMARSModuleCanary(TestCase):
def setUp(self):
self.broker = MARSBroker()
self.expected_keys = ['avro', 'candid', 'candidate', 'lco_id', 'objectId', 'publisher']
self.expected_candidate_keys = ['aimage', 'aimagerat', 'b', 'bimage', 'bimagerat', 'candid', 'chinr', 'chipsf',
'classtar', 'clrcoeff', 'clrcounc', 'clrmed', 'clrrms', 'dec', 'decnr',
'deltamaglatest', 'deltamagref', 'diffmaglim', 'distnr', 'distpsnr1',
'distpsnr2', 'distpsnr3', 'drb', 'drbversion', 'dsdiff', 'dsnrms', 'elong',
'exptime', 'fid', 'field', 'filter', 'fwhm', 'isdiffpos', 'jd', 'jdendhist',
'jdendref', 'jdstarthist', 'jdstartref', 'l', 'magap', 'magapbig', 'magdiff',
'magfromlim', 'maggaia', 'maggaiabright', 'magnr', 'magpsf', 'magzpsci',
'magzpscirms', 'magzpsciunc', 'mindtoedge', 'nbad', 'ncovhist', 'ndethist',
'neargaia', 'neargaiabright', 'nframesref', 'nid', 'nmatches', 'nmtchps',
'nneg', 'objectidps1', 'objectidps2', 'objectidps3', 'pdiffimfilename', 'pid',
'programid', 'programpi', 'ra', 'ranr', 'rb', 'rbversion', 'rcid', 'rfid',
'scorr', 'seeratio', 'sgmag1', 'sgmag2', 'sgmag3', 'sgscore1', 'sgscore2',
'sgscore3', 'sharpnr', 'sigmagap', 'sigmagapbig', 'sigmagnr', 'sigmapsf',
'simag1', 'simag2', 'simag3', 'sky', 'srmag1', 'srmag2', 'srmag3', 'ssdistnr',
'ssmagnr', 'ssnamenr', 'ssnrms', 'sumrat', 'szmag1', 'szmag2', 'szmag3',
'tblid', 'tooflag', 'wall_time', 'xpos', 'ypos', 'zpclrcov', 'zpmed']
def test_fetch_alerts(self):
response = self.broker.fetch_alerts({'time__gt': '2018-06-01', 'time__lt': '2018-06-30'})
alerts = []
for alert in islice(response, 10):
alerts.append(alert)
self.assertEqual(len(alerts), 10)
for key in self.expected_keys:
self.assertTrue(key in alerts[0].keys())
for key in self.expected_candidate_keys:
self.assertTrue(key in alerts[0]['candidate'].keys())
def test_fetch_alert(self):
alert = self.broker.fetch_alert(1065519)
for key in self.expected_keys:
self.assertTrue(key in alert.keys())
for key in self.expected_candidate_keys:
self.assertTrue(key in alert['candidate'].keys())
def test_process_reduced_data(self):
alert = self.broker.fetch_alert(1065519)
t = Target.objects.create(name='test target', ra=1, dec=2)
self.broker.process_reduced_data(t, alert=alert)
self.assertGreaterEqual(ReducedDatum.objects.filter(target=t, timestamp__lte=datetime(2020, 11, 3)).count(),
526)
def fetch_alerts(self, ):
mars = MARSBroker()
#download from ZTF web server (i.e. Przemek webpage)
ztf_ipac = urllib.request.urlopen(BROKER_URL).readlines()
list_of_events = [
str(i)[6:-8] for i in ztf_ipac if '<td>ZTF' in str(i)
]
list_of_mars_links = [
str(i).split('"')[1] for i in ztf_ipac
if '<td><a href="https://mars.lco.global/' in str(i)
]
for index, event in enumerate(list_of_events):
MARS_candidates = requests.get(list_of_mars_links[index] +
'&format=json').json()
if len(MARS_candidates['results']) == 0:
pass
else:
cone_search = str(MARS_candidates['results'][0]['candidate']
['ra']) + ',' + str(
MARS_candidates['results'][0]
['candidate']['dec']) + ',' + str(0.0003)
mars_form = MARSQueryForm({
'cone': cone_search,
'query_name': 'Query ZTF IPAC : ' + event,
'broker': 'MARS'
})
mars_form.is_valid()
query = BrokerQuery.objects.create(
name='Query ZTF IPAC : ' + event,
broker=mars.name,
parameters=mars_form.cleaned_data)
alerts = mars.fetch_alerts(query.parameters)
alerts = [*alerts]
name = event
ra = np.median([alert['candidate']['ra'] for alert in alerts])
dec = np.median(
[alert['candidate']['dec'] for alert in alerts])
try:
target = Target.objects.get(name=name)
except:
target, created = Target.objects.get_or_create(
name=name, ra=ra, dec=dec, type='SIDEREAL', epoch=2000)
if created:
target.save()
filters = {1: 'g_ZTF', 2: 'r_ZTF', 3: 'i_ZTF'}
try:
times = [
Time(i.timestamp).jd
for i in ReducedDatum.objects.filter(target=target)
if i.data_type == 'photometry'
]
except:
times = []
for alert in alerts:
if all([
key in alert['candidate']
for key in ['jd', 'magpsf', 'fid', 'sigmapsf']
]):
jd = Time(alert['candidate']['jd'],
format='jd',
scale='utc')
jd.to_datetime(timezone=TimezoneInfo())
if alert['candidate']['isdiffpos'] == 't':
signe = 1
else:
signe = -1
flux = 10**(
-0.4 * alert['candidate']['magnr']
) + signe * 10**(-0.4 * alert['candidate']['magpsf'])
eflux = ((10**(-0.4 * alert['candidate']['magnr']) *
alert['candidate']['sigmagnr'])**2 +
(signe *
10**(-0.4 * alert['candidate']['magpsf']) *
alert['candidate']['sigmapsf'])**2)**0.5
mag = -2.5 * np.log10(flux)
emag = eflux / flux
value = {
'magnitude': mag,
'filter': filters[alert['candidate']['fid']],
'error': emag
}
if (jd.value not in times):
rd, _ = ReducedDatum.objects.get_or_create(
timestamp=jd.to_datetime(
timezone=TimezoneInfo()),
value=value,
source_name='ZTF IPAC',
source_location='IRSA',
data_type='photometry',
target=target)
rd.save()
else:
pass
else:
pass