def dead_times(_path_pipe, _path_output, _date): """ RMJC's dead-time calculator :param _path_pipe: :param _path_output: :param _date: :return: """ start_times = [] end_times = [] for pot in ('high_flux', 'faint', 'zero_flux'): path_pot = os.path.join(_path_pipe, _date, pot) if os.path.isdir(path_pot): source_data = ([d for d in os.listdir(path_pot) if os.path.isdir(os.path.join(path_pot, d))]) for source in source_data: if 'pointing' not in source: try: hdulist = fits.open(os.path.join(path_pot, source, '100p.fits')) header_start = datetime.datetime.strptime(hdulist[0].header['UTSHUT'], '%Y%m%d_%H%M%S.%f') header_end = datetime.datetime.strptime(hdulist[0].header['END_TIME'], '%Y%m%d_%H%M%S.%f') except KeyError: print('Header Error: {:s}'.format(os.path.join(path_pot, source))) continue except IOError: print('Open Error: {:s}'.format(os.path.join(path_pot, source))) continue # succeed? append then: start_times.append(header_start) end_times.append(header_end) data_starts = Time(map(str, start_times), format='iso', scale='utc') sorted_starts = Time.sort(data_starts) data_ends = Time(map(str, end_times), format='iso', scale='utc') sorted_ends = Time.sort(data_ends) diffs = np.zeros(len(sorted_ends)-1) for i in range(len(sorted_ends)-1): diffs[i] = TimeDelta(sorted_starts[i+1].jd - sorted_ends[i].jd, format='jd').sec fig_names = [] try: fig = plt.figure('1h', figsize=(7, 3.5)) ax = fig.add_subplot(111) ax.hist((diffs[diffs < 3600])/60, 10, facecolor=plt.cm.Oranges(0.7)) # "#ff3300" ax.set_yscale('log') ax.set_xlabel('Minutes between the end of one exposure and the start of the next') # , fontsize=18) ax.set_ylabel('Number of Instances') # , fontsize=18) ax.set_title('Dead Time Distribution (<1hr)', fontsize=20) ax.grid(linewidth=0.5) plt.tight_layout() f_name = 'deadtime_1hr.{:s}.png'.format(_date) fig.savefig(os.path.join(_path_output, f_name), dpi=200) fig_names.append(f_name) except Exception as err: print(err) try: fig = plt.figure('5m', figsize=(7, 3.5)) ax = fig.add_subplot(111) ax.hist((diffs[diffs < 60*5]), 10, facecolor=plt.cm.Oranges(0.7)) # "#ff3300" ax.set_yscale('log') ax.set_xlabel('Seconds between the end of one exposure and the start of the next') # , fontsize=18) ax.set_ylabel('Number of Instances') # , fontsize=18) ax.set_title('Dead Time Distribution (<5min)', fontsize=20) ax.grid(linewidth=0.5) plt.tight_layout() f_name = 'deadtime_5min.{:s}.png'.format(_date) fig.savefig(os.path.join(_path_output, 'deadtime_5min.{:s}.png'.format(_date)), dpi=200) fig_names.append(f_name) except Exception as err: print(err) try: fig = plt.figure('1min', figsize=(7, 3.5)) ax = fig.add_subplot(111) ax.hist((diffs[diffs < 60]), 10, facecolor=plt.cm.Oranges(0.7)) # "#ff3300" ax.set_yscale('log') ax.set_xlabel('Seconds between the end of one exposure and the start of the next') # , fontsize=18) ax.set_ylabel('Number of Instances') # , fontsize=18) ax.set_title('Dead Time Distribution (<1min)', fontsize=20) ax.grid(linewidth=0.5) plt.tight_layout() f_name = 'deadtime_1min.{:s}.png'.format(_date) fig.savefig(os.path.join(_path_output, 'deadtime_1min.{:s}.png'.format(_date)), dpi=200) fig_names.append(f_name) except Exception as err: print(err) # return diffs for future reference return diffs, fig_names
class TestArithmetic: """Arithmetic on Time objects, using both doubles.""" kwargs = ({}, {'axis': None}, {'axis': 0}, {'axis': 1}, {'axis': 2}) functions = ('min', 'max', 'sort') def setup(self): mjd = np.arange(50000, 50100, 10).reshape(2, 5, 1) frac = np.array([0.1, 0.1 + 1.e-15, 0.1 - 1.e-15, 0.9 + 2.e-16, 0.9]) if use_masked_data: frac = np.ma.array(frac) frac[1] = np.ma.masked self.t0 = Time(mjd, frac, format='mjd', scale='utc') # Define arrays with same ordinal properties frac = np.array([1, 2, 0, 4, 3]) if use_masked_data: frac = np.ma.array(frac) frac[1] = np.ma.masked self.t1 = Time(mjd + frac, format='mjd', scale='utc') self.jd = mjd + frac @pytest.mark.parametrize('kw, func', itertools.product(kwargs, functions)) def test_argfuncs(self, kw, func, masked): """ Test that ``np.argfunc(jd, **kw)`` is the same as ``t0.argfunc(**kw)`` where ``jd`` is a similarly shaped array with the same ordinal properties but all integer values. Also test the same for t1 which has the same integral values as jd. """ t0v = getattr(self.t0, 'arg' + func)(**kw) t1v = getattr(self.t1, 'arg' + func)(**kw) jdv = getattr(np, 'arg' + func)(self.jd, **kw) if self.t0.masked and kw == {'axis': None} and func == 'sort': t0v = np.ma.array(t0v, mask=self.t0.mask.reshape(t0v.shape)[t0v]) t1v = np.ma.array(t1v, mask=self.t1.mask.reshape(t1v.shape)[t1v]) jdv = np.ma.array(jdv, mask=self.jd.mask.reshape(jdv.shape)[jdv]) assert np.all(t0v == jdv) assert np.all(t1v == jdv) assert t0v.shape == jdv.shape assert t1v.shape == jdv.shape @pytest.mark.parametrize('kw, func', itertools.product(kwargs, functions)) def test_funcs(self, kw, func, masked): """ Test that ``np.func(jd, **kw)`` is the same as ``t1.func(**kw)`` where ``jd`` is a similarly shaped array and the same integral values. """ t1v = getattr(self.t1, func)(**kw) jdv = getattr(np, func)(self.jd, **kw) assert np.all(t1v.value == jdv) assert t1v.shape == jdv.shape def test_argmin(self, masked): assert self.t0.argmin() == 2 assert np.all(self.t0.argmin(axis=0) == 0) assert np.all(self.t0.argmin(axis=1) == 0) assert np.all(self.t0.argmin(axis=2) == 2) def test_argmax(self, masked): assert self.t0.argmax() == self.t0.size - 2 if masked: # The 0 is where all entries are masked in that axis assert np.all(self.t0.argmax(axis=0) == [1, 0, 1, 1, 1]) assert np.all(self.t0.argmax(axis=1) == [4, 0, 4, 4, 4]) else: assert np.all(self.t0.argmax(axis=0) == 1) assert np.all(self.t0.argmax(axis=1) == 4) assert np.all(self.t0.argmax(axis=2) == 3) def test_argsort(self, masked): order = [2, 0, 4, 3, 1] if masked else [2, 0, 1, 4, 3] assert np.all(self.t0.argsort() == np.array(order)) assert np.all(self.t0.argsort(axis=0) == np.arange(2).reshape(2, 1, 1)) assert np.all(self.t0.argsort(axis=1) == np.arange(5).reshape(5, 1)) assert np.all(self.t0.argsort(axis=2) == np.array(order)) ravel = np.arange(50).reshape(-1, 5)[:, order].ravel() if masked: t0v = self.t0.argsort(axis=None) # Manually remove elements in ravel that correspond to masked # entries in self.t0. This removes the 10 entries that are masked # which show up at the end of the list. mask = self.t0.mask.ravel()[ravel] ravel = ravel[~mask] assert np.all(t0v[:-10] == ravel) else: assert np.all(self.t0.argsort(axis=None) == ravel) @pytest.mark.parametrize('scale', Time.SCALES) def test_argsort_warning(self, masked, scale): if scale == 'utc': pytest.xfail() with warnings.catch_warnings(record=True) as wlist: Time([1, 2, 3], format='jd', scale=scale).argsort() assert len(wlist) == 0 def test_min(self, masked): assert self.t0.min() == self.t0[0, 0, 2] assert np.all(self.t0.min(0) == self.t0[0]) assert np.all(self.t0.min(1) == self.t0[:, 0]) assert np.all(self.t0.min(2) == self.t0[:, :, 2]) assert self.t0.min(0).shape == (5, 5) assert self.t0.min(0, keepdims=True).shape == (1, 5, 5) assert self.t0.min(1).shape == (2, 5) assert self.t0.min(1, keepdims=True).shape == (2, 1, 5) assert self.t0.min(2).shape == (2, 5) assert self.t0.min(2, keepdims=True).shape == (2, 5, 1) def test_max(self, masked): assert self.t0.max() == self.t0[-1, -1, -2] assert np.all(self.t0.max(0) == self.t0[1]) assert np.all(self.t0.max(1) == self.t0[:, 4]) assert np.all(self.t0.max(2) == self.t0[:, :, 3]) assert self.t0.max(0).shape == (5, 5) assert self.t0.max(0, keepdims=True).shape == (1, 5, 5) def test_ptp(self, masked): assert self.t0.ptp() == self.t0.max() - self.t0.min() assert np.all(self.t0.ptp(0) == self.t0.max(0) - self.t0.min(0)) assert self.t0.ptp(0).shape == (5, 5) assert self.t0.ptp(0, keepdims=True).shape == (1, 5, 5) def test_sort(self, masked): order = [2, 0, 4, 3, 1] if masked else [2, 0, 1, 4, 3] assert np.all(self.t0.sort() == self.t0[:, :, order]) assert np.all(self.t0.sort(0) == self.t0) assert np.all(self.t0.sort(1) == self.t0) assert np.all(self.t0.sort(2) == self.t0[:, :, order]) if not masked: assert np.all(self.t0.sort(None) == self.t0[:, :, order].ravel()) # Bit superfluous, but good to check. assert np.all(self.t0.sort(-1)[:, :, 0] == self.t0.min(-1)) assert np.all(self.t0.sort(-1)[:, :, -1] == self.t0.max(-1))
t.add_column(dd['OBSERVATORY'],name='OBS') t.add_column(dd['FILTER'],name='FILTER') t.add_column(dd['detect'],name='DET') #t.add_column(dd['FILE'],name='FILE') t.write('SN2017ein-FLI-B.dat',format='ascii.commented_header',overwrite=True) # fil=['R']*175 # t.add_column(fil,name='FILTER') # each filter t=Table() for c in ddd: if '-V.dat' in c: cc=ascii.read(c) t=vstack([t,cc]) t.sort('MJD','OBS') t.write('SN2017ein-V.dat',format='ascii.commented_header',overwrite=True) # all to one dat file t=Table() for c in ddd: cc=ascii.read(c) t=vstack([t,cc]) t.sort(['MJD','OBS']) t.write('SN2017ein-BVR.dat',format='ascii.commented_header',overwrite=True) dpath='/data7/cschoi/sngal/NGC3938/phot/cut/' ddd=glob.glob(dpath+'SN2017ein*dat') dd=ddd[0] # marker
def dead_times(_path_pipe, _path_output, _date): """ RMJC's dead-time calculator :param _path_pipe: :param _path_output: :param _date: :return: """ start_times = [] end_times = [] for pot in ('high_flux', 'faint', 'zero_flux'): path_pot = os.path.join(_path_pipe, _date, pot) if os.path.isdir(path_pot): source_data = ([ d for d in os.listdir(path_pot) if os.path.isdir(os.path.join(path_pot, d)) ]) for source in source_data: if 'pointing' not in source: try: hdulist = fits.open( os.path.join(path_pot, source, '100p.fits')) header_start = datetime.datetime.strptime( hdulist[0].header['UTSHUT'], '%Y%m%d_%H%M%S.%f') header_end = datetime.datetime.strptime( hdulist[0].header['END_TIME'], '%Y%m%d_%H%M%S.%f') except KeyError: print('Header Error: {:s}'.format( os.path.join(path_pot, source))) continue except IOError: print('Open Error: {:s}'.format( os.path.join(path_pot, source))) continue # succeed? append then: start_times.append(header_start) end_times.append(header_end) data_starts = Time(map(str, start_times), format='iso', scale='utc') sorted_starts = Time.sort(data_starts) data_ends = Time(map(str, end_times), format='iso', scale='utc') sorted_ends = Time.sort(data_ends) diffs = np.zeros(len(sorted_ends) - 1) for i in range(len(sorted_ends) - 1): diffs[i] = TimeDelta(sorted_starts[i + 1].jd - sorted_ends[i].jd, format='jd').sec fig_names = [] try: fig = plt.figure('1h', figsize=(7, 3.5)) ax = fig.add_subplot(111) ax.hist((diffs[diffs < 3600]) / 60, 10, facecolor=plt.cm.Oranges(0.7)) # "#ff3300" ax.set_yscale('log') ax.set_xlabel( 'Minutes between the end of one exposure and the start of the next' ) # , fontsize=18) ax.set_ylabel('Number of Instances') # , fontsize=18) ax.set_title('Dead Time Distribution (<1hr)', fontsize=20) ax.grid(linewidth=0.5) plt.tight_layout() f_name = 'deadtime_1hr.{:s}.png'.format(_date) fig.savefig(os.path.join(_path_output, f_name), dpi=200) fig_names.append(f_name) except Exception as err: print(err) try: fig = plt.figure('5m', figsize=(7, 3.5)) ax = fig.add_subplot(111) ax.hist((diffs[diffs < 60 * 5]), 10, facecolor=plt.cm.Oranges(0.7)) # "#ff3300" ax.set_yscale('log') ax.set_xlabel( 'Seconds between the end of one exposure and the start of the next' ) # , fontsize=18) ax.set_ylabel('Number of Instances') # , fontsize=18) ax.set_title('Dead Time Distribution (<5min)', fontsize=20) ax.grid(linewidth=0.5) plt.tight_layout() f_name = 'deadtime_5min.{:s}.png'.format(_date) fig.savefig(os.path.join(_path_output, 'deadtime_5min.{:s}.png'.format(_date)), dpi=200) fig_names.append(f_name) except Exception as err: print(err) try: fig = plt.figure('1min', figsize=(7, 3.5)) ax = fig.add_subplot(111) ax.hist((diffs[diffs < 60]), 10, facecolor=plt.cm.Oranges(0.7)) # "#ff3300" ax.set_yscale('log') ax.set_xlabel( 'Seconds between the end of one exposure and the start of the next' ) # , fontsize=18) ax.set_ylabel('Number of Instances') # , fontsize=18) ax.set_title('Dead Time Distribution (<1min)', fontsize=20) ax.grid(linewidth=0.5) plt.tight_layout() f_name = 'deadtime_1min.{:s}.png'.format(_date) fig.savefig(os.path.join(_path_output, 'deadtime_1min.{:s}.png'.format(_date)), dpi=200) fig_names.append(f_name) except Exception as err: print(err) # return diffs for future reference return diffs, fig_names
class TestArithmetic(): """Arithmetic on Time objects, using both doubles.""" kwargs = ({}, {'axis': None}, {'axis': 0}, {'axis': 1}, {'axis': 2}) functions = ('min', 'max', 'sort') def setup(self): mjd = np.arange(50000, 50100, 10).reshape(2, 5, 1) frac = np.array([0.1, 0.1+1.e-15, 0.1-1.e-15, 0.9+2.e-16, 0.9]) if use_masked_data: frac = np.ma.array(frac) frac[1] = np.ma.masked self.t0 = Time(mjd, frac, format='mjd', scale='utc') # Define arrays with same ordinal properties frac = np.array([1, 2, 0, 4, 3]) if use_masked_data: frac = np.ma.array(frac) frac[1] = np.ma.masked self.t1 = Time(mjd + frac, format='mjd', scale='utc') self.jd = mjd + frac @pytest.mark.parametrize('kw, func', itertools.product(kwargs, functions)) def test_argfuncs(self, kw, func, masked): """ Test that np.argfunc(jd, **kw) is the same as t0.argfunc(**kw) where jd is a similarly shaped array with the same ordinal properties but all integer values. Also test the same for t1 which has the same integral values as jd. """ t0v = getattr(self.t0, 'arg' + func)(**kw) t1v = getattr(self.t1, 'arg' + func)(**kw) jdv = getattr(np, 'arg' + func)(self.jd, **kw) if self.t0.masked and kw == {'axis': None} and func == 'sort': t0v = np.ma.array(t0v, mask=self.t0.mask.reshape(t0v.shape)[t0v]) t1v = np.ma.array(t1v, mask=self.t1.mask.reshape(t1v.shape)[t1v]) jdv = np.ma.array(jdv, mask=self.jd.mask.reshape(jdv.shape)[jdv]) assert np.all(t0v == jdv) assert np.all(t1v == jdv) assert t0v.shape == jdv.shape assert t1v.shape == jdv.shape @pytest.mark.parametrize('kw, func', itertools.product(kwargs, functions)) def test_funcs(self, kw, func, masked): """ Test that np.func(jd, **kw) is the same as t1.func(**kw) where jd is a similarly shaped array and the same integral values. """ t1v = getattr(self.t1, func)(**kw) jdv = getattr(np, func)(self.jd, **kw) assert np.all(t1v.value == jdv) assert t1v.shape == jdv.shape def test_argmin(self, masked): assert self.t0.argmin() == 2 assert np.all(self.t0.argmin(axis=0) == 0) assert np.all(self.t0.argmin(axis=1) == 0) assert np.all(self.t0.argmin(axis=2) == 2) def test_argmax(self, masked): assert self.t0.argmax() == self.t0.size - 2 if masked: # The 0 is where all entries are masked in that axis assert np.all(self.t0.argmax(axis=0) == [1, 0, 1, 1, 1]) assert np.all(self.t0.argmax(axis=1) == [4, 0, 4, 4, 4]) else: assert np.all(self.t0.argmax(axis=0) == 1) assert np.all(self.t0.argmax(axis=1) == 4) assert np.all(self.t0.argmax(axis=2) == 3) def test_argsort(self, masked): order = [2, 0, 4, 3, 1] if masked else [2, 0, 1, 4, 3] assert np.all(self.t0.argsort() == np.array(order)) assert np.all(self.t0.argsort(axis=0) == np.arange(2).reshape(2, 1, 1)) assert np.all(self.t0.argsort(axis=1) == np.arange(5).reshape(5, 1)) assert np.all(self.t0.argsort(axis=2) == np.array(order)) ravel = np.arange(50).reshape(-1, 5)[:, order].ravel() if masked: t0v = self.t0.argsort(axis=None) # Manually remove elements in ravel that correspond to masked # entries in self.t0. This removes the 10 entries that are masked # which show up at the end of the list. mask = self.t0.mask.ravel()[ravel] ravel = ravel[~mask] assert np.all(t0v[:-10] == ravel) else: assert np.all(self.t0.argsort(axis=None) == ravel) def test_min(self, masked): assert self.t0.min() == self.t0[0, 0, 2] assert np.all(self.t0.min(0) == self.t0[0]) assert np.all(self.t0.min(1) == self.t0[:, 0]) assert np.all(self.t0.min(2) == self.t0[:, :, 2]) assert self.t0.min(0).shape == (5, 5) assert self.t0.min(0, keepdims=True).shape == (1, 5, 5) assert self.t0.min(1).shape == (2, 5) assert self.t0.min(1, keepdims=True).shape == (2, 1, 5) assert self.t0.min(2).shape == (2, 5) assert self.t0.min(2, keepdims=True).shape == (2, 5, 1) def test_max(self, masked): assert self.t0.max() == self.t0[-1, -1, -2] assert np.all(self.t0.max(0) == self.t0[1]) assert np.all(self.t0.max(1) == self.t0[:, 4]) assert np.all(self.t0.max(2) == self.t0[:, :, 3]) assert self.t0.max(0).shape == (5, 5) assert self.t0.max(0, keepdims=True).shape == (1, 5, 5) def test_ptp(self, masked): assert self.t0.ptp() == self.t0.max() - self.t0.min() assert np.all(self.t0.ptp(0) == self.t0.max(0) - self.t0.min(0)) assert self.t0.ptp(0).shape == (5, 5) assert self.t0.ptp(0, keepdims=True).shape == (1, 5, 5) def test_sort(self, masked): order = [2, 0, 4, 3, 1] if masked else [2, 0, 1, 4, 3] assert np.all(self.t0.sort() == self.t0[:, :, order]) assert np.all(self.t0.sort(0) == self.t0) assert np.all(self.t0.sort(1) == self.t0) assert np.all(self.t0.sort(2) == self.t0[:, :, order]) if not masked: assert np.all(self.t0.sort(None) == self.t0[:, :, order].ravel()) # Bit superfluous, but good to check. assert np.all(self.t0.sort(-1)[:, :, 0] == self.t0.min(-1)) assert np.all(self.t0.sort(-1)[:, :, -1] == self.t0.max(-1))