def main(base_dir,
processed_dir=None,
picid=None,
force=False,
camera_bias=2048,
num_workers=8,
chunk_size=12,
):
print(f'Finding similar stars for observation in {base_dir}')
fields_dir = os.path.join(os.environ['PANDIR'], 'images', 'fields')
source_filename = os.path.join(fields_dir, base_dir, f'point-sources-filtered.csv.bz2')
assert os.path.isfile(source_filename)
print(f'Using sources in {source_filename}')
# Get the sources
sources = pipeline.lookup_sources_for_observation(
filename=source_filename).set_index(['picid'], append=True)
if picid:
print(f"Creating stamp for {picid}")
sources = sources.query(f'picid == {picid}')
if not len(sources):
print(f"{picid} does not exist, exiting")
return
picid_list = list(sources.index.levels[1].unique())
# Used for progress display
num_sources = len(picid_list)
print(f'Finding similar stars for {num_sources} sources')
call_params = {
'picid_list': picid_list, # Pass the full list
'base_dir': base_dir,
'processed_dir': processed_dir,
'force': force,
'camera_bias': camera_bias,
}
# Build up the parameter list (NB: "clever" zip_longest usage)
if picid:
params = zip_longest([picid], [], fillvalue=call_params)
else:
params = zip_longest(picid_list, [], fillvalue=call_params)
start_time = current_time()
print(f'Starting at {start_time}')
with concurrent.futures.ProcessPoolExecutor(max_workers=num_workers) as executor:
picids = list(tqdm(executor.map(find_similar, params,
chunksize=chunk_size), total=len(picid_list)))
logging.debug(f'Found similar stars for {len(picids)} sources')
end_time = current_time()
print(f'Ending at {end_time}')
total_time = (end_time - start_time).sec
print(f'Total: {total_time:.02f} seconds')
def test_pretty_time():
t0 = '2016-08-13 10:00:00'
os.environ['POCSTIME'] = t0
t1 = current_time(pretty=True)
assert t1 == t0
# This will increment one second - see docs
t2 = current_time(flatten=True)
assert t2 != t0
assert t2 == '20160813T100001'
# This will increment one second - see docs
t3 = current_time(datetime=True)
assert t3 == dt(2016, 8, 13, 10, 0, 2, tzinfo=tz.utc)
def obj():
return {
"name": "Generic PANOPTES Unit",
"pan_id": "PAN000",
"location": {
"name": "Mauna Loa Observatory",
"latitude": 19.54 * u.degree, # Astropy unit
"longitude": "-155.58 deg", # String unit
"elevation": "3400.0 m",
"horizon": 30 * u.degree,
"flat_horizon": -6 * u.degree,
"focus_horizon": -12 * u.degree,
"observe_horizon": -18 * u.degree,
"timezone": "US/Hawaii",
"gmt_offset": -600,
},
"directories": {
"base": "/var/panoptes",
"images": "images",
"data": "data",
"resources": "POCS/resources/",
"targets": "POCS/resources/targets",
"mounts": "POCS/resources/mounts",
},
"db": {
"name": "panoptes",
"type": "file"
},
"empty": {},
"current_time": current_time(),
"bool": True,
}
def do_last_reading(self, device):
""" Gets the last reading from the device. """
if not device:
print_warning('Usage: last_reading <device>')
return
if not hasattr(self, device):
print_warning('No such sensor: {!r}'.format(device))
return
rec = self.db.get_current(device)
if rec is None:
print_warning('No reading found for {!r}'.format(device))
return
print_info('*' * 80)
print("{}:".format(device.upper()))
pprint(rec)
print_info('*' * 80)
# Display the age in seconds of the record
if isinstance(rec.get('date'), datetime.datetime):
now = current_time(datetime=True).astimezone(utc)
record_date = rec['date'].astimezone(utc)
age = (now - record_date).total_seconds()
if age < 120:
print_info('{:.1f} seconds old'.format(age))
else:
print_info('{:.1f} minutes old'.format(age / 60.0))
def send_message(self, topic, message):
""" Responsible for actually sending message across a topic
Args:
topic(str): Name of topic to send on. The name must
match topic_name_re.
message: Message to be sent (a string or a dict).
"""
if not isinstance(topic, str):
raise ValueError('Topic name must be a string')
elif not self.topic_name_re.fullmatch(topic):
raise ValueError('Topic name ("{}") is not valid'.format(topic))
if topic == 'PANCHAT':
self.logger.info(f"{topic} {message}")
if isinstance(message, str):
message = to_json({
"message": message,
"timestamp": current_time(pretty=True),
})
elif isinstance(message, dict):
message = to_json(message)
else:
raise ValueError('Message value must be a string or dict')
# Build the full message with topic
full_message = f'{topic} {message}'
# Send the message
# self.socket.send_string(full_message, flags=zmq.NOBLOCK)
self.socket.send_string(full_message)
def test_countdown_timer_bad_input():
with pytest.raises(ValueError):
assert CountdownTimer('d')
with pytest.raises(ValueError):
assert CountdownTimer(current_time())
with pytest.raises(AssertionError):
assert CountdownTimer(-1)
def _make_pretty_from_fits(fname=None,
title=None,
figsize=(10, 10 / 1.325),
dpi=150,
alpha=0.2,
number_ticks=7,
clip_percent=99.9,
**kwargs):
with open_fits(fname) as hdu:
header = hdu[0].header
data = hdu[0].data
data = focus_utils.mask_saturated(data)
wcs = WCS(header)
if not title:
field = header.get('FIELD', 'Unknown field')
exptime = header.get('EXPTIME', 'Unknown exptime')
filter_type = header.get('FILTER', 'Unknown filter')
try:
date_time = header['DATE-OBS']
except KeyError:
# If we don't have DATE-OBS, check filename for date
try:
basename = os.path.splitext(os.path.basename(fname))[0]
date_time = date_parser.parse(basename).isoformat()
except Exception:
# Otherwise use now
date_time = current_time(pretty=True)
date_time = date_time.replace('T', ' ', 1)
title = '{} ({}s {}) {}'.format(field, exptime, filter_type, date_time)
norm = ImageNormalize(interval=PercentileInterval(clip_percent), stretch=LogStretch())
fig = Figure()
FigureCanvas(fig)
fig.set_size_inches(*figsize)
fig.dpi = dpi
if wcs.is_celestial:
ax = fig.add_subplot(1, 1, 1, projection=wcs)
ax.coords.grid(True, color='white', ls='-', alpha=alpha)
ra_axis = ax.coords['ra']
ra_axis.set_axislabel('Right Ascension')
ra_axis.set_major_formatter('hh:mm')
ra_axis.set_ticks(
number=number_ticks,
color='white',
exclude_overlapping=True
)
dec_axis = ax.coords['dec']
dec_axis.set_axislabel('Declination')
dec_axis.set_major_formatter('dd:mm')
dec_axis.set_ticks(
number=number_ticks,
color='white',
exclude_overlapping=True
)
else:
ax = fig.add_subplot(111)
ax.grid(True, color='white', ls='-', alpha=alpha)
ax.set_xlabel('X / pixels')
ax.set_ylabel('Y / pixels')
im = ax.imshow(data, norm=norm, cmap=palette, origin='lower')
fig.colorbar(im)
fig.suptitle(title)
new_filename = fname.replace('.fits', '.jpg')
fig.savefig(new_filename, bbox_inches='tight')
# explicitly close and delete figure
fig.clf()
del fig
return new_filename
def main(base_dir,
processed_dir=None,
camera_bias=2048,
gain=1.5,
readout_noise=10.5,
frame_slice=None,
table_filter=None,
num_refs=50,
aperture_size=5,
make_plots=False,
color_correction=False,
picid=None,
force=False,
num_workers=8,
chunk_size=12,
):
logger.info(f'Building references for stars for observation in {base_dir}')
if picid:
logger.info(f'Searching for picid={picid}')
output_dir = os.path.join(processed_dir, str(picid), base_dir)
else:
output_dir = os.path.join(processed_dir, '*', base_dir)
psc_files = glob(os.path.join(output_dir, 'psc.csv'), recursive=True)
logger.info(f'Found {len(psc_files)} PSC files')
call_params = {
'base_dir': base_dir,
'output_dir': output_dir,
'processed_dir': processed_dir,
'force': force,
'frame_slice': frame_slice,
'table_filter': table_filter,
'num_refs': num_refs,
'camera_bias': camera_bias,
'gain': gain,
'readout_noise': readout_noise,
'make_plots': make_plots,
'color_correction': color_correction,
}
logger.debug(f'Call params: {call_params}')
# Build up the parameter list (NB: "clever" zip_longest usage)
params = zip_longest(psc_files, [], fillvalue=call_params)
start_time = current_time()
print(f'Starting at {start_time}')
with concurrent.futures.ProcessPoolExecutor(max_workers=num_workers) as executor:
picids = list(tqdm(
executor.map(build_ref, params, chunksize=chunk_size
),
total=len(psc_files))
)
logger.info(f'Created {len(picids)} PSC references')
end_time = current_time()
print(f'Ending at {end_time}')
total_time = (end_time - start_time).sec
print(f'Total: {total_time:.02f} seconds')
def main(
base_dir,
processed_dir=None,
camera_bias=2048,
gain=1.5,
readout_noise=10.5,
frame_slice=None,
table_filter=None,
num_refs=50,
aperture_size=5,
make_plots=False,
color_correction=False,
picid=None,
force=False,
num_workers=8,
chunk_size=12,
):
logger.info(f'Building references for stars for observation in {base_dir}')
if picid:
logger.info(f'Searching for picid={picid}')
output_dir = os.path.join(processed_dir, str(picid), base_dir)
else:
output_dir = os.path.join(processed_dir, '*', base_dir)
psc_files = glob(os.path.join(output_dir, 'psc.csv'), recursive=True)
logger.info(f'Found {len(psc_files)} PSC files')
call_params = {
'base_dir': base_dir,
'output_dir': output_dir,
'processed_dir': processed_dir,
'force': force,
'frame_slice': frame_slice,
'table_filter': table_filter,
'num_refs': num_refs,
'camera_bias': camera_bias,
'gain': gain,
'readout_noise': readout_noise,
'make_plots': make_plots,
'color_correction': color_correction,
}
logger.debug(f'Call params: {call_params}')
# Build up the parameter list (NB: "clever" zip_longest usage)
params = zip_longest(psc_files, [], fillvalue=call_params)
start_time = current_time()
print(f'Starting at {start_time}')
with concurrent.futures.ProcessPoolExecutor(
max_workers=num_workers) as executor:
picids = list(
tqdm(executor.map(build_ref, params, chunksize=chunk_size),
total=len(psc_files)))
logger.info(f'Created {len(picids)} PSC references')
end_time = current_time()
print(f'Ending at {end_time}')
total_time = (end_time - start_time).sec
print(f'Total: {total_time:.02f} seconds')
def main(base_dir=None,
output_dir=None,
stamp_size=10,
picid=None,
force=False,
num_workers=8,
chunk_size=12
):
fields_dir = os.path.join(os.environ['PANDIR'], 'images', 'fields')
# Get the sources from the stored file.
source_filename = os.path.join(fields_dir, base_dir, f'point-sources-filtered.csv.bz2')
# Check for existence of file otherwise `lookup_sources_for_observation` will try to create.
if not os.path.isfile(source_filename):
raise UserWarning(f'Please do a source detection and filtering first.')
# Load the sources from the file.
sources = pipeline.lookup_sources_for_observation(filename=source_filename)
sources.set_index(['picid'], append=True, inplace=True)
# Used for progress display.
num_sources = len(list(sources.index.levels[1].unique()))
# Add full path to filename in table.
sources.file = sources.file.apply(lambda fn: os.path.join(fields_dir, base_dir, fn))
if picid:
print(f"Creating stamp for {picid}")
sources = sources.query(f'picid == {picid}')
if not len(sources):
print(f"{picid} does not exist, exiting")
return
else:
print(f'Building PSC for {num_sources} sources')
start_time = current_time()
call_params = {
'observation_dir': base_dir,
'output_dir': output_dir,
'force': force,
'stamp_size': stamp_size,
}
print(f'Starting at {start_time}')
# Run everything in parallel.
with concurrent.futures.ProcessPoolExecutor(max_workers=num_workers) as executor:
grouped_sources = sources.groupby('picid')
params = zip_longest(grouped_sources, [], fillvalue=call_params)
picids = list(tqdm(executor.map(make_psc, params, chunksize=chunk_size),
total=len(grouped_sources)))
print(f'Created {len(picids)} PSCs')
end_time = current_time()
print(f'Ending at {end_time}')
total_time = (end_time - start_time).sec
print(f'Total: {total_time:.02f} seconds')
def create_storage_obj(collection, data, obj_id=None):
"""Returns the object to be stored in the database"""
obj = dict(data=data, type=collection, date=current_time(datetime=True))
if obj_id:
obj['_id'] = obj_id
return obj