def main(startdate, enddate, logger, nice=False, skip=False):
if startdate > enddate:
oneday = tdelta(-1, 0)
else:
oneday = tdelta(1, 0)
##------------------------------------------------------------------------
## Use pyephem determine sunrise and sunset times
##------------------------------------------------------------------------
now = dt.utcnow()
if nice:
Observatory = ephem.Observer()
Observatory.lon = "-155:34:33.9"
Observatory.lat = "+19:32:09.66"
Observatory.elevation = 3400.0
Observatory.temp = 10.0
Observatory.pressure = 680.0
Observatory.horizon = '0.0'
Observatory.date = now.strftime('%Y/%m/%d %H:%M:%S')
TheSun = ephem.Sun()
MatchFilename = re.compile("(.*)\-([0-9]{8})at([0-9]{6})\.fts")
MatchEmpty = re.compile(".*\-Empty\-.*\.fts")
date = startdate
while True:
date_string = date.strftime('%Y%m%dUT')
logger.info('Checking for images from {}'.format(date_string))
images = []
V5_path = os.path.join("/Volumes", "Drobo", "V5", "Images", date_string)
V20_path = os.path.join("/Volumes", "Drobo", "V20", "Images", date_string)
if os.path.exists(V5_path):
V5_images = glob(os.path.join(V5_path, '*.fts'))
logger.info(' Found {} images for the night of {} for V5'.format(\
len(V5_images), date_string))
images.extend(V5_images)
if os.path.exists(V20_path):
V20_images = glob(os.path.join(V20_path, '*.fts'))
logger.info(' Found {} images for the night of {} for V20'.format(\
len(V20_images), date_string))
images.extend(V20_images)
## Sort Images by Observation time
properties = []
for image in images:
skip_this_image = False
imagename = os.path.split(image)[1]
FNmatch = MatchFilename.match(imagename)
Ematch = MatchEmpty.match(imagename)
## If skip is enabled, skip images which are already in mongo db
if skip:
telescope = None
V5match = re.match("V5.*\.fi?ts", imagename)
V20match = re.match("V20.*\.fi?ts", imagename)
if V5match and not V20match:
telescope = "V5"
elif V20match and not V5match:
telescope = "V20"
else:
with fits.open(image) as hdulist:
if hdulist[0].header['OBSERVAT']:
if re.search('VYSOS-?20', hdulist[0].header['OBSERVAT']):
telescope = "V20"
elif re.search('VYSOS-?5', hdulist[0].header['OBSERVAT']):
telescope = "V5"
else:
print("Can not determine valid telescope from arguments or filename or header.")
else:
print("Can not determine valid telescope from arguments or filename or header.")
if telescope:
config_file = os.path.join(os.path.expanduser('~'), '.{}.yaml'.format(telescope))
tel = IQMon.Telescope(config_file)
client = MongoClient(tel.mongo_address, tel.mongo_port)
db = client[tel.mongo_db]
data = db[tel.mongo_collection]
matches = [item for item in data.find( {"filename" : imagename} )]
if len(matches) > 0:
# images.remove(image)
skip_this_image = True
## Remove images with Empty in filenme
if Ematch:
# images.remove(image)
skip_this_image = True
if not FNmatch:
# images.remove(image)
skip_this_image = True
if not skip_this_image:
try:
image_dt = dt.strptime('{} {}'.format(\
FNmatch.group(2), FNmatch.group(3)), '%Y%m%d %H%M%S')
except:
image_dt = dt.utcnow()
properties.append([image, image_dt])
properties = sorted(properties, key=lambda entry:entry[1])
## Process Images
count = 0
for entry in properties:
count += 1
print('')
print('Examining image {} out of {} for the night of {}'.format(\
count, len(properties), date_string))
image = entry[0]
if nice:
now = dt.utcnow()
Observatory.date = now.strftime('%Y/%m/%d %H:%M:%S')
TheSun.compute(Observatory)
if TheSun.alt < 0:
print('The Sun is down (alt = {:.1f})'.format(TheSun.alt*180./ephem.pi))
sunrise = Observatory.next_rising(TheSun).datetime()
until_sunrise = (sunrise - now).total_seconds()/60./60.
logger.info('Sleeping {:.1f} hours until sunrise'.format(until_sunrise))
time.sleep(until_sunrise + 300)
now = dt.utcnow()
Observatory.date = now.strftime('%Y/%m/%d %H:%M:%S')
sunset = Observatory.next_setting(ephem.Sun()).datetime()
sunrise = Observatory.next_rising(ephem.Sun()).datetime()
logger.info('Resuming processing ...')
logger.info(' Next sunset at {}'.format(sunset.strftime('%Y/%m/%d %H:%M:%S')))
if MatchFilename.match(image) and not MatchEmpty.match(image):
try:
measure_image.MeasureImage(image,\
clobber_logs=True,\
zero_point=True,\
analyze_image=True)
except:
logger.warning('MeasureImage failed on {}'.format(image))
measure_image.MeasureImage(image,\
clobber_logs=False,\
zero_point=True,\
analyze_image=False)
make_nightly_plots.make_plots(date_string, 'V5', logger)
make_nightly_plots.make_plots(date_string, 'V20', logger)
if date == enddate:
break
date += oneday
def main(startdate, enddate, logger):
logger.info("Writing results to mongo db at 192.168.1.101")
try:
client = MongoClient("192.168.1.101", 27017)
except:
logger.error("Could not connect to mongo db")
raise
else:
V5status = client.vysos["V5.status"]
logger.debug(" Getting V5.status collection")
V20status = client.vysos["V20.status"]
logger.debug(" Getting V20.status collection")
oneday = tdelta(1, 0)
date = startdate
while date <= enddate:
date_string = date.strftime("%Y%m%dUT")
logger.info("")
logger.info("Checking for environmental logs from {}".format(date_string))
## VYSOS-5
telescope = "V5"
logfile = os.path.join("/", "Volumes", "Drobo", telescope, "Logs", date_string, "EnvironmentalLog.txt")
if not os.path.exists(logfile):
logger.warning(" No logfile found for {} on {}".format(telescope, date_string))
else:
logger.info(" Found logfile: {}".format(logfile))
# ColStarts = [ 0, 11, 22, 32, 42, 52, 62, 72, 82, 92, 102, 112]
# ColEnds = [ 9, 20, 31, 41, 51, 61, 71, 81, 91, 101, 111, 121]
# ColNames = ['Date', 'TimeString', 'TubeTemp', 'FocusPos',
# 'SkyTemp', 'OutsideTemp', 'WindSpeed', 'Humidity', 'DewPoint', 'Altitude', 'Azimuth', 'Condition']
# env_table = ascii.read(logfile, data_start=2, Reader=ascii.FixedWidth,
# col_starts=ColStarts, col_ends=ColEnds, names=ColNames,
# guess=False, comment=";", header_start=0,
# converters={
# 'Date': [ascii.convert_numpy(numpy.str)],
# 'TimeString': [ascii.convert_numpy(numpy.str)],
# 'TubeTemp': [ascii.convert_numpy(numpy.float)],
# 'FocusPos': [ascii.convert_numpy(numpy.int)],
# 'SkyTemp': [ascii.convert_numpy(numpy.float)],
# 'OutsideTemp': [ascii.convert_numpy(numpy.float)],
# 'WindSpeed': [ascii.convert_numpy(numpy.float)],
# 'Humidity': [ascii.convert_numpy(numpy.int)],
# 'DewPoint': [ascii.convert_numpy(numpy.float)],
# 'Altitude': [ascii.convert_numpy(numpy.float)],
# 'Azimuth': [ascii.convert_numpy(numpy.float)],
# 'Condition': [ascii.convert_numpy(numpy.str)]
# }
# )
with open(logfile, "r") as FO:
env_table = FO.readlines()
for line in env_table:
if line[0] != "#":
try:
entry = line.split()
new_data = {}
## Date and Time
dto_utc = dt.strptime("{} {}".format(entry[0], entry[1]), "%Y/%m/%d %H:%M:%SUT")
dto_hst = dto_utc - tdelta(0, 10 * 60 * 60)
new_data.update(
{
"UT date": dto_utc.strftime("%Y%m%dUT"),
"UT time": dto_utc.strftime("%H:%M:%S"),
"UT timestamp": dto_utc,
}
)
## Define Boltwood Data
boltwood = {}
boltwood["boltwood date"] = dto_hst.strftime("%Y-%m-%d") # local date (yyyy-mm-dd)
boltwood["boltwood time"] = dto_hst.strftime("%H:%M:%S.00") # local time (hh:mm:ss.ss)
boltwood["boltwood timestamp"] = dto_hst
boltwood["boltwood temp units"] = "F"
boltwood["boltwood wind units"] = "K"
boltwood["boltwood sky temp"] = float(entry[4])
boltwood["boltwood ambient temp"] = float(entry[5])
boltwood["boltwood wind speed"] = float(entry[6])
boltwood["boltwood humidity"] = int(entry[7])
boltwood["boltwood dew point"] = float(entry[8])
boltwood["boltwood rain condition"] = int(entry[11][0])
boltwood["boltwood cloud condition"] = int(entry[11][1])
boltwood["boltwood wind condition"] = int(entry[11][2])
new_data.update(boltwood)
except:
print(line)
print(entry)
raise
##-------------------------------------------------------------------------
## Write Environmental Log
##-------------------------------------------------------------------------
## Check if this image is already in the collection
matches = [item for item in V5status.find({"UT timestamp": new_data["UT timestamp"]})]
if len(matches) > 0:
logger.debug(
" Found {} previous entries for {} {}. Deleting old entries.".format(
len(matches), new_data["UT date"], new_data["UT time"]
)
)
for match in matches:
logger.debug(" Removing entry for {} {}".format(match["UT date"], match["UT time"]))
V5status.remove({"_id": match["_id"]})
logger.debug(' Removed "_id": {}'.format(match["_id"]))
id = V5status.insert(new_data)
logger.info(
" Inserted datum for {} on {} {}".format(telescope, new_data["UT date"], new_data["UT time"])
)
make_nightly_plots.make_plots(date_string, "V5", logger)
## VYSOS-20
telescope = "V20"
logfile = os.path.join("/", "Volumes", "Drobo", telescope, "Logs", date_string, "EnvironmentalLog.txt")
if not os.path.exists(logfile):
logger.warning(" No logfile found for {} on {}".format(telescope, date_string))
else:
logger.info(" Found logfile: {}".format(logfile))
# ColStarts = [ 0, 11, 22, 32, 42, 52, 62, 72, 82, 92, 102, 112, 122, 132, 142, 152, 162]
# ColEnds = [ 9, 20, 31, 41, 51, 61, 71, 81, 91, 101, 111, 121, 131, 141, 151, 161, 171]
# ColNames = ['Date', 'TimeString', 'TubeTemp', 'PrimaryTemp', 'SecTemp', 'FanPower', 'FocusPos',
# 'SkyTemp', 'OutsideTemp', 'WindSpeed', 'Humidity', 'DewPoint', 'Altitude', 'Azimuth',
# 'Condition', 'DomeTemp', 'DomeFanState']
# env_table = ascii.read(logfile, data_start=2, Reader=ascii.FixedWidth,
# col_starts=ColStarts, col_ends=ColEnds, names=ColNames,
# guess=False, comment=";", header_start=0,
# converters={
# 'Date': [ascii.convert_numpy('S10')],
# 'TimeString': [ascii.convert_numpy('S10')],
# 'TubeTemp': [ascii.convert_numpy('f4')],
# 'SecTemp': [ascii.convert_numpy('f4')],
# 'FanPower': [ascii.convert_numpy('f4')],
# 'FocusPos': [ascii.convert_numpy('i4')],
# 'SkyTemp': [ascii.convert_numpy('f4')],
# 'OutsideTemp': [ascii.convert_numpy('f4')],
# 'WindSpeed': [ascii.convert_numpy('f4')],
# 'Humidity': [ascii.convert_numpy('i4')],
# 'DewPoint': [ascii.convert_numpy('f4')],
# 'Altitude': [ascii.convert_numpy('f4')],
# 'Azimuth': [ascii.convert_numpy('f4')],
# 'Condition': [ascii.convert_numpy('S3')],
# 'DomeTemp': [ascii.convert_numpy('f4')],
# 'DomeFanState': [ascii.convert_numpy('i4')]
# }
# )
with open(logfile, "r") as FO:
env_table = FO.readlines()
for line in env_table:
if line[0] != "#":
try:
entry = line.split()
new_data = {}
## Date and Time
dto_utc = dt.strptime("{} {}".format(entry[0], entry[1]), "%Y/%m/%d %H:%M:%SUT")
dto_hst = dto_utc - tdelta(0, 10 * 60 * 60)
new_data.update(
{
"UT date": dto_utc.strftime("%Y%m%dUT"),
"UT time": dto_utc.strftime("%H:%M:%S"),
"UT timestamp": dto_utc,
}
)
## Define Boltwood Data
boltwood = {}
boltwood["boltwood date"] = dto_hst.strftime("%Y-%m-%d") # local date (yyyy-mm-dd)
boltwood["boltwood time"] = dto_hst.strftime("%H:%M:%S.00") # local time (hh:mm:ss.ss)
boltwood["boltwood timestamp"] = dto_hst
boltwood["boltwood temp units"] = "F"
boltwood["boltwood wind units"] = "K"
boltwood["boltwood sky temp"] = float(entry[7])
boltwood["boltwood ambient temp"] = float(entry[8])
boltwood["boltwood wind speed"] = float(entry[9])
boltwood["boltwood humidity"] = int(entry[10])
boltwood["boltwood dew point"] = float(entry[11])
boltwood["boltwood rain condition"] = int(entry[14][0])
boltwood["boltwood cloud condition"] = int(entry[14][1])
boltwood["boltwood wind condition"] = int(entry[14][2])
new_data.update(boltwood)
## Define Focuser Data
focuser_info = {}
focuser_info["RCOS temperature units"] = "F"
focuser_info["RCOS temperature (truss)"] = float(entry[2])
focuser_info["RCOS temperature (primary)"] = float(entry[3])
focuser_info["RCOS temperature (secondary)"] = float(entry[4])
focuser_info["RCOS fan speed"] = int(entry[5])
focuser_info["RCOS focuser position"] = int(entry[6])
new_data.update(focuser_info)
except:
print(entry)
raise
##-------------------------------------------------------------------------
## Write Environmental Log
##-------------------------------------------------------------------------
## Check if this image is already in the collection
matches = [item for item in V20status.find({"UT timestamp": new_data["UT timestamp"]})]
if len(matches) > 0:
logger.debug(
" Found {} previous entries for {} {}. Deleting old entries.".format(
len(matches), new_data["UT date"], new_data["UT time"]
)
)
for match in matches:
logger.debug(" Removing entry for {} {}".format(match["UT date"], match["UT time"]))
V20status.remove({"_id": match["_id"]})
logger.debug(' Removed "_id": {}'.format(match["_id"]))
id = V20status.insert(new_data)
logger.info(
" Inserted datum for {} on {} {}".format(telescope, new_data["UT date"], new_data["UT time"])
)
make_nightly_plots.make_plots(date_string, "V20", logger)
date += oneday
def main(startdate, enddate, logger, nice=False, skip=False):
if startdate > enddate:
oneday = tdelta(-1, 0)
else:
oneday = tdelta(1, 0)
##------------------------------------------------------------------------
## Use pyephem determine sunrise and sunset times
##------------------------------------------------------------------------
now = dt.utcnow()
if nice:
Observatory = ephem.Observer()
Observatory.lon = "-155:34:33.9"
Observatory.lat = "+19:32:09.66"
Observatory.elevation = 3400.0
Observatory.temp = 10.0
Observatory.pressure = 680.0
Observatory.horizon = '0.0'
Observatory.date = now.strftime('%Y/%m/%d %H:%M:%S')
TheSun = ephem.Sun()
MatchFilename = re.compile("(.*)\-([0-9]{8})at([0-9]{6})\.fts")
MatchEmpty = re.compile(".*\-Empty\-.*\.fts")
date = startdate
while True:
date_string = date.strftime('%Y%m%dUT')
logger.info('Checking for images from {}'.format(date_string))
images = []
V5_path = os.path.join("/Volumes", "Drobo", "V5", "Images",
date_string)
V20_path = os.path.join("/Volumes", "Drobo", "V20", "Images",
date_string)
if os.path.exists(V5_path):
V5_images = glob(os.path.join(V5_path, '*.fts'))
logger.info(' Found {} images for the night of {} for V5'.format(\
len(V5_images), date_string))
images.extend(V5_images)
if os.path.exists(V20_path):
V20_images = glob(os.path.join(V20_path, '*.fts'))
logger.info(' Found {} images for the night of {} for V20'.format(\
len(V20_images), date_string))
images.extend(V20_images)
## Sort Images by Observation time
properties = []
for image in images:
skip_this_image = False
imagename = os.path.split(image)[1]
FNmatch = MatchFilename.match(imagename)
Ematch = MatchEmpty.match(imagename)
## If skip is enabled, skip images which are already in mongo db
if skip:
telescope = None
V5match = re.match("V5.*\.fi?ts", imagename)
V20match = re.match("V20.*\.fi?ts", imagename)
if V5match and not V20match:
telescope = "V5"
elif V20match and not V5match:
telescope = "V20"
else:
with fits.open(image) as hdulist:
if hdulist[0].header['OBSERVAT']:
if re.search('VYSOS-?20',
hdulist[0].header['OBSERVAT']):
telescope = "V20"
elif re.search('VYSOS-?5',
hdulist[0].header['OBSERVAT']):
telescope = "V5"
else:
print(
"Can not determine valid telescope from arguments or filename or header."
)
else:
print(
"Can not determine valid telescope from arguments or filename or header."
)
if telescope:
config_file = os.path.join(os.path.expanduser('~'),
'.{}.yaml'.format(telescope))
tel = IQMon.Telescope(config_file)
client = MongoClient(tel.mongo_address, tel.mongo_port)
db = client[tel.mongo_db]
data = db[tel.mongo_collection]
matches = [
item for item in data.find({"filename": imagename})
]
if len(matches) > 0:
# images.remove(image)
skip_this_image = True
## Remove images with Empty in filenme
if Ematch:
# images.remove(image)
skip_this_image = True
if not FNmatch:
# images.remove(image)
skip_this_image = True
if not skip_this_image:
try:
image_dt = dt.strptime('{} {}'.format(\
FNmatch.group(2), FNmatch.group(3)), '%Y%m%d %H%M%S')
except:
image_dt = dt.utcnow()
properties.append([image, image_dt])
properties = sorted(properties, key=lambda entry: entry[1])
## Process Images
count = 0
for entry in properties:
count += 1
print('')
print('Examining image {} out of {} for the night of {}'.format(\
count, len(properties), date_string))
image = entry[0]
if nice:
now = dt.utcnow()
Observatory.date = now.strftime('%Y/%m/%d %H:%M:%S')
TheSun.compute(Observatory)
if TheSun.alt < 0:
print('The Sun is down (alt = {:.1f})'.format(
TheSun.alt * 180. / ephem.pi))
sunrise = Observatory.next_rising(TheSun).datetime()
until_sunrise = (sunrise - now).total_seconds() / 60. / 60.
logger.info('Sleeping {:.1f} hours until sunrise'.format(
until_sunrise))
time.sleep(until_sunrise + 300)
now = dt.utcnow()
Observatory.date = now.strftime('%Y/%m/%d %H:%M:%S')
sunset = Observatory.next_setting(ephem.Sun()).datetime()
sunrise = Observatory.next_rising(ephem.Sun()).datetime()
logger.info('Resuming processing ...')
logger.info(' Next sunset at {}'.format(
sunset.strftime('%Y/%m/%d %H:%M:%S')))
if MatchFilename.match(image) and not MatchEmpty.match(image):
try:
measure_image.MeasureImage(image,\
clobber_logs=True,\
zero_point=True,\
analyze_image=True)
except:
logger.warning('MeasureImage failed on {}'.format(image))
measure_image.MeasureImage(image,\
clobber_logs=False,\
zero_point=True,\
analyze_image=False)
make_nightly_plots.make_plots(date_string, 'V5', logger)
make_nightly_plots.make_plots(date_string, 'V20', logger)
if date == enddate:
break
date += oneday