def dead_times(_path_pipe, _path_output, _date):
""" RMJC's dead-time calculator
:param _path_pipe:
:param _path_output:
:param _date:
:return:
"""
start_times = []
end_times = []
for pot in ('high_flux', 'faint', 'zero_flux'):
path_pot = os.path.join(_path_pipe, _date, pot)
if os.path.isdir(path_pot):
source_data = ([d for d in os.listdir(path_pot) if os.path.isdir(os.path.join(path_pot, d))])
for source in source_data:
if 'pointing' not in source:
try:
hdulist = fits.open(os.path.join(path_pot, source, '100p.fits'))
header_start = datetime.datetime.strptime(hdulist[0].header['UTSHUT'], '%Y%m%d_%H%M%S.%f')
header_end = datetime.datetime.strptime(hdulist[0].header['END_TIME'], '%Y%m%d_%H%M%S.%f')
except KeyError:
print('Header Error: {:s}'.format(os.path.join(path_pot, source)))
continue
except IOError:
print('Open Error: {:s}'.format(os.path.join(path_pot, source)))
continue
# succeed? append then:
start_times.append(header_start)
end_times.append(header_end)
data_starts = Time(map(str, start_times), format='iso', scale='utc')
sorted_starts = Time.sort(data_starts)
data_ends = Time(map(str, end_times), format='iso', scale='utc')
sorted_ends = Time.sort(data_ends)
diffs = np.zeros(len(sorted_ends)-1)
for i in range(len(sorted_ends)-1):
diffs[i] = TimeDelta(sorted_starts[i+1].jd - sorted_ends[i].jd, format='jd').sec
fig_names = []
try:
fig = plt.figure('1h', figsize=(7, 3.5))
ax = fig.add_subplot(111)
ax.hist((diffs[diffs < 3600])/60, 10, facecolor=plt.cm.Oranges(0.7)) # "#ff3300"
ax.set_yscale('log')
ax.set_xlabel('Minutes between the end of one exposure and the start of the next') # , fontsize=18)
ax.set_ylabel('Number of Instances') # , fontsize=18)
ax.set_title('Dead Time Distribution (<1hr)', fontsize=20)
ax.grid(linewidth=0.5)
plt.tight_layout()
f_name = 'deadtime_1hr.{:s}.png'.format(_date)
fig.savefig(os.path.join(_path_output, f_name), dpi=200)
fig_names.append(f_name)
except Exception as err:
print(err)
try:
fig = plt.figure('5m', figsize=(7, 3.5))
ax = fig.add_subplot(111)
ax.hist((diffs[diffs < 60*5]), 10, facecolor=plt.cm.Oranges(0.7)) # "#ff3300"
ax.set_yscale('log')
ax.set_xlabel('Seconds between the end of one exposure and the start of the next') # , fontsize=18)
ax.set_ylabel('Number of Instances') # , fontsize=18)
ax.set_title('Dead Time Distribution (<5min)', fontsize=20)
ax.grid(linewidth=0.5)
plt.tight_layout()
f_name = 'deadtime_5min.{:s}.png'.format(_date)
fig.savefig(os.path.join(_path_output, 'deadtime_5min.{:s}.png'.format(_date)), dpi=200)
fig_names.append(f_name)
except Exception as err:
print(err)
try:
fig = plt.figure('1min', figsize=(7, 3.5))
ax = fig.add_subplot(111)
ax.hist((diffs[diffs < 60]), 10, facecolor=plt.cm.Oranges(0.7)) # "#ff3300"
ax.set_yscale('log')
ax.set_xlabel('Seconds between the end of one exposure and the start of the next') # , fontsize=18)
ax.set_ylabel('Number of Instances') # , fontsize=18)
ax.set_title('Dead Time Distribution (<1min)', fontsize=20)
ax.grid(linewidth=0.5)
plt.tight_layout()
f_name = 'deadtime_1min.{:s}.png'.format(_date)
fig.savefig(os.path.join(_path_output, 'deadtime_1min.{:s}.png'.format(_date)), dpi=200)
fig_names.append(f_name)
except Exception as err:
print(err)
# return diffs for future reference
return diffs, fig_names
class TestArithmetic:
"""Arithmetic on Time objects, using both doubles."""
kwargs = ({}, {'axis': None}, {'axis': 0}, {'axis': 1}, {'axis': 2})
functions = ('min', 'max', 'sort')
def setup(self):
mjd = np.arange(50000, 50100, 10).reshape(2, 5, 1)
frac = np.array([0.1, 0.1 + 1.e-15, 0.1 - 1.e-15, 0.9 + 2.e-16, 0.9])
if use_masked_data:
frac = np.ma.array(frac)
frac[1] = np.ma.masked
self.t0 = Time(mjd, frac, format='mjd', scale='utc')
# Define arrays with same ordinal properties
frac = np.array([1, 2, 0, 4, 3])
if use_masked_data:
frac = np.ma.array(frac)
frac[1] = np.ma.masked
self.t1 = Time(mjd + frac, format='mjd', scale='utc')
self.jd = mjd + frac
@pytest.mark.parametrize('kw, func', itertools.product(kwargs, functions))
def test_argfuncs(self, kw, func, masked):
"""
Test that ``np.argfunc(jd, **kw)`` is the same as ``t0.argfunc(**kw)``
where ``jd`` is a similarly shaped array with the same ordinal properties
but all integer values. Also test the same for t1 which has the same
integral values as jd.
"""
t0v = getattr(self.t0, 'arg' + func)(**kw)
t1v = getattr(self.t1, 'arg' + func)(**kw)
jdv = getattr(np, 'arg' + func)(self.jd, **kw)
if self.t0.masked and kw == {'axis': None} and func == 'sort':
t0v = np.ma.array(t0v, mask=self.t0.mask.reshape(t0v.shape)[t0v])
t1v = np.ma.array(t1v, mask=self.t1.mask.reshape(t1v.shape)[t1v])
jdv = np.ma.array(jdv, mask=self.jd.mask.reshape(jdv.shape)[jdv])
assert np.all(t0v == jdv)
assert np.all(t1v == jdv)
assert t0v.shape == jdv.shape
assert t1v.shape == jdv.shape
@pytest.mark.parametrize('kw, func', itertools.product(kwargs, functions))
def test_funcs(self, kw, func, masked):
"""
Test that ``np.func(jd, **kw)`` is the same as ``t1.func(**kw)`` where
``jd`` is a similarly shaped array and the same integral values.
"""
t1v = getattr(self.t1, func)(**kw)
jdv = getattr(np, func)(self.jd, **kw)
assert np.all(t1v.value == jdv)
assert t1v.shape == jdv.shape
def test_argmin(self, masked):
assert self.t0.argmin() == 2
assert np.all(self.t0.argmin(axis=0) == 0)
assert np.all(self.t0.argmin(axis=1) == 0)
assert np.all(self.t0.argmin(axis=2) == 2)
def test_argmax(self, masked):
assert self.t0.argmax() == self.t0.size - 2
if masked:
# The 0 is where all entries are masked in that axis
assert np.all(self.t0.argmax(axis=0) == [1, 0, 1, 1, 1])
assert np.all(self.t0.argmax(axis=1) == [4, 0, 4, 4, 4])
else:
assert np.all(self.t0.argmax(axis=0) == 1)
assert np.all(self.t0.argmax(axis=1) == 4)
assert np.all(self.t0.argmax(axis=2) == 3)
def test_argsort(self, masked):
order = [2, 0, 4, 3, 1] if masked else [2, 0, 1, 4, 3]
assert np.all(self.t0.argsort() == np.array(order))
assert np.all(self.t0.argsort(axis=0) == np.arange(2).reshape(2, 1, 1))
assert np.all(self.t0.argsort(axis=1) == np.arange(5).reshape(5, 1))
assert np.all(self.t0.argsort(axis=2) == np.array(order))
ravel = np.arange(50).reshape(-1, 5)[:, order].ravel()
if masked:
t0v = self.t0.argsort(axis=None)
# Manually remove elements in ravel that correspond to masked
# entries in self.t0. This removes the 10 entries that are masked
# which show up at the end of the list.
mask = self.t0.mask.ravel()[ravel]
ravel = ravel[~mask]
assert np.all(t0v[:-10] == ravel)
else:
assert np.all(self.t0.argsort(axis=None) == ravel)
@pytest.mark.parametrize('scale', Time.SCALES)
def test_argsort_warning(self, masked, scale):
if scale == 'utc':
pytest.xfail()
with warnings.catch_warnings(record=True) as wlist:
Time([1, 2, 3], format='jd', scale=scale).argsort()
assert len(wlist) == 0
def test_min(self, masked):
assert self.t0.min() == self.t0[0, 0, 2]
assert np.all(self.t0.min(0) == self.t0[0])
assert np.all(self.t0.min(1) == self.t0[:, 0])
assert np.all(self.t0.min(2) == self.t0[:, :, 2])
assert self.t0.min(0).shape == (5, 5)
assert self.t0.min(0, keepdims=True).shape == (1, 5, 5)
assert self.t0.min(1).shape == (2, 5)
assert self.t0.min(1, keepdims=True).shape == (2, 1, 5)
assert self.t0.min(2).shape == (2, 5)
assert self.t0.min(2, keepdims=True).shape == (2, 5, 1)
def test_max(self, masked):
assert self.t0.max() == self.t0[-1, -1, -2]
assert np.all(self.t0.max(0) == self.t0[1])
assert np.all(self.t0.max(1) == self.t0[:, 4])
assert np.all(self.t0.max(2) == self.t0[:, :, 3])
assert self.t0.max(0).shape == (5, 5)
assert self.t0.max(0, keepdims=True).shape == (1, 5, 5)
def test_ptp(self, masked):
assert self.t0.ptp() == self.t0.max() - self.t0.min()
assert np.all(self.t0.ptp(0) == self.t0.max(0) - self.t0.min(0))
assert self.t0.ptp(0).shape == (5, 5)
assert self.t0.ptp(0, keepdims=True).shape == (1, 5, 5)
def test_sort(self, masked):
order = [2, 0, 4, 3, 1] if masked else [2, 0, 1, 4, 3]
assert np.all(self.t0.sort() == self.t0[:, :, order])
assert np.all(self.t0.sort(0) == self.t0)
assert np.all(self.t0.sort(1) == self.t0)
assert np.all(self.t0.sort(2) == self.t0[:, :, order])
if not masked:
assert np.all(self.t0.sort(None) == self.t0[:, :, order].ravel())
# Bit superfluous, but good to check.
assert np.all(self.t0.sort(-1)[:, :, 0] == self.t0.min(-1))
assert np.all(self.t0.sort(-1)[:, :, -1] == self.t0.max(-1))
t.add_column(dd['OBSERVATORY'],name='OBS')
t.add_column(dd['FILTER'],name='FILTER')
t.add_column(dd['detect'],name='DET')
#t.add_column(dd['FILE'],name='FILE')
t.write('SN2017ein-FLI-B.dat',format='ascii.commented_header',overwrite=True)
# fil=['R']*175
# t.add_column(fil,name='FILTER')
# each filter
t=Table()
for c in ddd:
if '-V.dat' in c:
cc=ascii.read(c)
t=vstack([t,cc])
t.sort('MJD','OBS')
t.write('SN2017ein-V.dat',format='ascii.commented_header',overwrite=True)
# all to one dat file
t=Table()
for c in ddd:
cc=ascii.read(c)
t=vstack([t,cc])
t.sort(['MJD','OBS'])
t.write('SN2017ein-BVR.dat',format='ascii.commented_header',overwrite=True)
dpath='/data7/cschoi/sngal/NGC3938/phot/cut/'
ddd=glob.glob(dpath+'SN2017ein*dat')
dd=ddd[0]
# marker
def dead_times(_path_pipe, _path_output, _date):
""" RMJC's dead-time calculator
:param _path_pipe:
:param _path_output:
:param _date:
:return:
"""
start_times = []
end_times = []
for pot in ('high_flux', 'faint', 'zero_flux'):
path_pot = os.path.join(_path_pipe, _date, pot)
if os.path.isdir(path_pot):
source_data = ([
d for d in os.listdir(path_pot)
if os.path.isdir(os.path.join(path_pot, d))
])
for source in source_data:
if 'pointing' not in source:
try:
hdulist = fits.open(
os.path.join(path_pot, source, '100p.fits'))
header_start = datetime.datetime.strptime(
hdulist[0].header['UTSHUT'], '%Y%m%d_%H%M%S.%f')
header_end = datetime.datetime.strptime(
hdulist[0].header['END_TIME'], '%Y%m%d_%H%M%S.%f')
except KeyError:
print('Header Error: {:s}'.format(
os.path.join(path_pot, source)))
continue
except IOError:
print('Open Error: {:s}'.format(
os.path.join(path_pot, source)))
continue
# succeed? append then:
start_times.append(header_start)
end_times.append(header_end)
data_starts = Time(map(str, start_times), format='iso', scale='utc')
sorted_starts = Time.sort(data_starts)
data_ends = Time(map(str, end_times), format='iso', scale='utc')
sorted_ends = Time.sort(data_ends)
diffs = np.zeros(len(sorted_ends) - 1)
for i in range(len(sorted_ends) - 1):
diffs[i] = TimeDelta(sorted_starts[i + 1].jd - sorted_ends[i].jd,
format='jd').sec
fig_names = []
try:
fig = plt.figure('1h', figsize=(7, 3.5))
ax = fig.add_subplot(111)
ax.hist((diffs[diffs < 3600]) / 60, 10,
facecolor=plt.cm.Oranges(0.7)) # "#ff3300"
ax.set_yscale('log')
ax.set_xlabel(
'Minutes between the end of one exposure and the start of the next'
) # , fontsize=18)
ax.set_ylabel('Number of Instances') # , fontsize=18)
ax.set_title('Dead Time Distribution (<1hr)', fontsize=20)
ax.grid(linewidth=0.5)
plt.tight_layout()
f_name = 'deadtime_1hr.{:s}.png'.format(_date)
fig.savefig(os.path.join(_path_output, f_name), dpi=200)
fig_names.append(f_name)
except Exception as err:
print(err)
try:
fig = plt.figure('5m', figsize=(7, 3.5))
ax = fig.add_subplot(111)
ax.hist((diffs[diffs < 60 * 5]), 10,
facecolor=plt.cm.Oranges(0.7)) # "#ff3300"
ax.set_yscale('log')
ax.set_xlabel(
'Seconds between the end of one exposure and the start of the next'
) # , fontsize=18)
ax.set_ylabel('Number of Instances') # , fontsize=18)
ax.set_title('Dead Time Distribution (<5min)', fontsize=20)
ax.grid(linewidth=0.5)
plt.tight_layout()
f_name = 'deadtime_5min.{:s}.png'.format(_date)
fig.savefig(os.path.join(_path_output,
'deadtime_5min.{:s}.png'.format(_date)),
dpi=200)
fig_names.append(f_name)
except Exception as err:
print(err)
try:
fig = plt.figure('1min', figsize=(7, 3.5))
ax = fig.add_subplot(111)
ax.hist((diffs[diffs < 60]), 10,
facecolor=plt.cm.Oranges(0.7)) # "#ff3300"
ax.set_yscale('log')
ax.set_xlabel(
'Seconds between the end of one exposure and the start of the next'
) # , fontsize=18)
ax.set_ylabel('Number of Instances') # , fontsize=18)
ax.set_title('Dead Time Distribution (<1min)', fontsize=20)
ax.grid(linewidth=0.5)
plt.tight_layout()
f_name = 'deadtime_1min.{:s}.png'.format(_date)
fig.savefig(os.path.join(_path_output,
'deadtime_1min.{:s}.png'.format(_date)),
dpi=200)
fig_names.append(f_name)
except Exception as err:
print(err)
# return diffs for future reference
return diffs, fig_names
class TestArithmetic():
"""Arithmetic on Time objects, using both doubles."""
kwargs = ({}, {'axis': None}, {'axis': 0}, {'axis': 1}, {'axis': 2})
functions = ('min', 'max', 'sort')
def setup(self):
mjd = np.arange(50000, 50100, 10).reshape(2, 5, 1)
frac = np.array([0.1, 0.1+1.e-15, 0.1-1.e-15, 0.9+2.e-16, 0.9])
if use_masked_data:
frac = np.ma.array(frac)
frac[1] = np.ma.masked
self.t0 = Time(mjd, frac, format='mjd', scale='utc')
# Define arrays with same ordinal properties
frac = np.array([1, 2, 0, 4, 3])
if use_masked_data:
frac = np.ma.array(frac)
frac[1] = np.ma.masked
self.t1 = Time(mjd + frac, format='mjd', scale='utc')
self.jd = mjd + frac
@pytest.mark.parametrize('kw, func', itertools.product(kwargs, functions))
def test_argfuncs(self, kw, func, masked):
"""
Test that np.argfunc(jd, **kw) is the same as t0.argfunc(**kw) where
jd is a similarly shaped array with the same ordinal properties but
all integer values. Also test the same for t1 which has the same
integral values as jd.
"""
t0v = getattr(self.t0, 'arg' + func)(**kw)
t1v = getattr(self.t1, 'arg' + func)(**kw)
jdv = getattr(np, 'arg' + func)(self.jd, **kw)
if self.t0.masked and kw == {'axis': None} and func == 'sort':
t0v = np.ma.array(t0v, mask=self.t0.mask.reshape(t0v.shape)[t0v])
t1v = np.ma.array(t1v, mask=self.t1.mask.reshape(t1v.shape)[t1v])
jdv = np.ma.array(jdv, mask=self.jd.mask.reshape(jdv.shape)[jdv])
assert np.all(t0v == jdv)
assert np.all(t1v == jdv)
assert t0v.shape == jdv.shape
assert t1v.shape == jdv.shape
@pytest.mark.parametrize('kw, func', itertools.product(kwargs, functions))
def test_funcs(self, kw, func, masked):
"""
Test that np.func(jd, **kw) is the same as t1.func(**kw) where
jd is a similarly shaped array and the same integral values.
"""
t1v = getattr(self.t1, func)(**kw)
jdv = getattr(np, func)(self.jd, **kw)
assert np.all(t1v.value == jdv)
assert t1v.shape == jdv.shape
def test_argmin(self, masked):
assert self.t0.argmin() == 2
assert np.all(self.t0.argmin(axis=0) == 0)
assert np.all(self.t0.argmin(axis=1) == 0)
assert np.all(self.t0.argmin(axis=2) == 2)
def test_argmax(self, masked):
assert self.t0.argmax() == self.t0.size - 2
if masked:
# The 0 is where all entries are masked in that axis
assert np.all(self.t0.argmax(axis=0) == [1, 0, 1, 1, 1])
assert np.all(self.t0.argmax(axis=1) == [4, 0, 4, 4, 4])
else:
assert np.all(self.t0.argmax(axis=0) == 1)
assert np.all(self.t0.argmax(axis=1) == 4)
assert np.all(self.t0.argmax(axis=2) == 3)
def test_argsort(self, masked):
order = [2, 0, 4, 3, 1] if masked else [2, 0, 1, 4, 3]
assert np.all(self.t0.argsort() == np.array(order))
assert np.all(self.t0.argsort(axis=0) == np.arange(2).reshape(2, 1, 1))
assert np.all(self.t0.argsort(axis=1) == np.arange(5).reshape(5, 1))
assert np.all(self.t0.argsort(axis=2) == np.array(order))
ravel = np.arange(50).reshape(-1, 5)[:, order].ravel()
if masked:
t0v = self.t0.argsort(axis=None)
# Manually remove elements in ravel that correspond to masked
# entries in self.t0. This removes the 10 entries that are masked
# which show up at the end of the list.
mask = self.t0.mask.ravel()[ravel]
ravel = ravel[~mask]
assert np.all(t0v[:-10] == ravel)
else:
assert np.all(self.t0.argsort(axis=None) == ravel)
def test_min(self, masked):
assert self.t0.min() == self.t0[0, 0, 2]
assert np.all(self.t0.min(0) == self.t0[0])
assert np.all(self.t0.min(1) == self.t0[:, 0])
assert np.all(self.t0.min(2) == self.t0[:, :, 2])
assert self.t0.min(0).shape == (5, 5)
assert self.t0.min(0, keepdims=True).shape == (1, 5, 5)
assert self.t0.min(1).shape == (2, 5)
assert self.t0.min(1, keepdims=True).shape == (2, 1, 5)
assert self.t0.min(2).shape == (2, 5)
assert self.t0.min(2, keepdims=True).shape == (2, 5, 1)
def test_max(self, masked):
assert self.t0.max() == self.t0[-1, -1, -2]
assert np.all(self.t0.max(0) == self.t0[1])
assert np.all(self.t0.max(1) == self.t0[:, 4])
assert np.all(self.t0.max(2) == self.t0[:, :, 3])
assert self.t0.max(0).shape == (5, 5)
assert self.t0.max(0, keepdims=True).shape == (1, 5, 5)
def test_ptp(self, masked):
assert self.t0.ptp() == self.t0.max() - self.t0.min()
assert np.all(self.t0.ptp(0) == self.t0.max(0) - self.t0.min(0))
assert self.t0.ptp(0).shape == (5, 5)
assert self.t0.ptp(0, keepdims=True).shape == (1, 5, 5)
def test_sort(self, masked):
order = [2, 0, 4, 3, 1] if masked else [2, 0, 1, 4, 3]
assert np.all(self.t0.sort() == self.t0[:, :, order])
assert np.all(self.t0.sort(0) == self.t0)
assert np.all(self.t0.sort(1) == self.t0)
assert np.all(self.t0.sort(2) == self.t0[:, :, order])
if not masked:
assert np.all(self.t0.sort(None) ==
self.t0[:, :, order].ravel())
# Bit superfluous, but good to check.
assert np.all(self.t0.sort(-1)[:, :, 0] == self.t0.min(-1))
assert np.all(self.t0.sort(-1)[:, :, -1] == self.t0.max(-1))