def search(self, query = {}, attributes=[]):
#Copy the queyr into a dict where all values
#are case insensitive regex
regex_query = dict(zip(query.iterkeys(), [re.compile(p, re.IGNORECASE) for p in filter(utils.pattern_from_javascript, query.itervalues())]))
#Can we save a table scan?
#We have an index, but it will work only if there is no regex
#Let's try it blindly
try:
dn = regex_query['dn'].pattern.lower()
if dn in self.shelf.keys():
rawobject = self.shelf[dn]
#Found it! (or else we would be in the KeyError handler)
#The object was not save with the metadata, so let's make sure its there
utils.add_metadata(self.sanitize_result(rawobject, dn), dn)
yield rawobject
except KeyError:
#Our only key is not in the query
#no choice but to do a table scan
for dn, rawobject in self.shelf.iteritems():
#The object was not save with the metadata, so let's make sure its there
utils.add_metadata(self.sanitize_result(rawobject, dn), dn)
if self.ldapobjectMatchesQuery(rawobject, regex_query):
#If we get here, the objet matches the query
#we will copy the object in a new dict
yield rawobject
def search(self, query = {}, attributes=[]):
base, query = self.convert_query(copy.deepcopy(query))
#Async paged search
result_generator = self.connection.paged_search_ext_s(base, ldap.SCOPE_SUBTREE, query, attrlist=attributes, serverctrls=self.serverctrls)
for dn,entry in result_generator:
# process dn and entry
wtf = utils.add_metadata(self.sanitize_result(entry, dn), dn)
yield wtf
def write(self, original, ldapobject):
'''Saves the ldapobject in a Mongo database
'''
#If there is a DN (should be, but we don't force it)
#we use it as a spec (criteria) key for update
try:
#We use a specification so that multiple imports does not
#create multiple objects. Making the dn the _id also works
#but it makes everybody in the pipeline aware of the hack.
spec = { 'dn' : ldapobject['dn'][0] }
#Insert private metadata that we will use later
#The mongolito.parent field allows to sort results so that
#parents are always created first
utils.add_metadata(ldapobject, ldapobject['dn'][0])
#I though there would be an implict conversion to dict, but
#there is not. Let's make one.
#This line works in most cases :
#ldapobject = dict(zip(ldapobject.keys(), ldapobject.values()))
#But Mongo will fail to insert any binary objects that are in
#ldapobject.values(). The zip statement above can be written in
#a single line, but the following is clearer imo
mongoobject = {}
for attribute, value in ldapobject.iteritems():
mongoobject[attribute] = self.convert_to_mongo_datatype(value)
#Upsert the object
self.collection.update(spec, mongoobject, upsert=True)
except KeyError:
raise SaveException(-1, 'dn', 'Object does not have a dn attribute')
except bson.errors.InvalidDocument:
raise SaveException(-1, ldapobject['dn'], 'Encoding the object returned an invalid BSON document')
except bson.errors.InvalidStringData:
raise SaveException(-1, ldapobject['dn'], 'Object has a value that is not properly encoded')
def run_do_rest(args):
thread_count = args.threads
vastdir = utils.add_trailing_slash(args.datadir)
resultdir = clean_and_create_resultdir(args.resultdir, vastdir)
do_light = args.light
do_phase = args.phase
do_aavso = args.aavso
logging.info(
f"Dir with VaST files: '{vastdir}'\nResults dir: '{resultdir}'")
# get wcs model from the reference header. Used in writing world positions and field charts (can fail)
wcs_file, wcs = reading.read_wcs_file(vastdir)
ref_jd, _, _, reference_frame = reading.extract_reference_frame(vastdir)
_, _, _, first_frame = reading.extract_first_frame(vastdir)
reference_frame_path = Path(reference_frame)
reference_frame_filename = reference_frame_path.name
wcs_file, wcs = perform_astrometry_net(args, vastdir, wcs_file, wcs,
reference_frame_filename)
logging.info(f"The reference frame is '{reference_frame}' at JD: {ref_jd}")
logging.info(f"The first frame is '{first_frame}'")
logging.info(f"Reference header is '{wcs_file}'")
check_that_reference_image_not_within_jdfilter(ref_jd, args.jdfilter,
args.jdrefignore)
star_descriptions = construct_star_descriptions(vastdir, resultdir, wcs,
args)
stardict = get_localid_to_sd_dict(star_descriptions)
logging.debug(
f"First (max) 10 star descriptions: "
f"{star_descriptions[:10] if (len(star_descriptions) >= 10) else star_descriptions}"
)
write_augmented_autocandidates(vastdir, resultdir, stardict)
write_augmented_all_stars(vastdir, resultdir, stardict)
candidate_stars = utils.get_stars_with_metadata(star_descriptions,
"CANDIDATE",
exclude=["VSX"])
if args.selectcandidates:
tag_candidates_as_selected(candidate_stars)
logging.info(f"There are {len(candidate_stars)} candidate stars")
vsx_stars = utils.get_stars_with_metadata(star_descriptions, "VSX")
logging.info(f"There are {len(vsx_stars)} vsx stars")
if args.allstars:
tag_all_stars_as_selected(star_descriptions)
selected_stars = utils.get_stars_with_metadata(star_descriptions,
"SELECTEDTAG")
logging.info(f"There are {len(selected_stars)} selected stars")
compstar_needing_stars = utils.concat_sd_lists(selected_stars, vsx_stars,
candidate_stars)
comp_stars = set_comp_stars_and_ucac4(star_descriptions, selected_stars,
args.checkstarfile, vastdir,
stardict, ref_jd)
# Set comp stars for all interesting stars (stars which are interesting enough to measure)
logging.info("Setting per star comparison stars...")
if args.checkstarfile:
utils.add_metadata(star_descriptions,
CompStarData(compstar_ids=comp_stars.ids))
else:
do_compstars.add_closest_compstars(compstar_needing_stars, comp_stars,
10)
logging.info(
f"Using {thread_count} threads for phase plots, lightcurves, ...")
if args.allstars:
do_charts_vast.run(
star_descriptions,
comp_stars,
vastdir,
resultdir,
"phase_all/",
"light_all/",
"aavso_all/",
do_phase=do_phase,
do_light=do_light,
do_light_raw=do_light,
do_aavso=do_aavso,
nr_threads=thread_count,
jdfilter=args.jdfilter,
desc="Phase/light/aavso of ALL stars",
)
else:
if args.vsx:
logging.info(f"Plotting {len(vsx_stars)} vsx stars...")
do_charts_vast.run(
vsx_stars,
comp_stars,
vastdir,
resultdir,
"phase_vsx/",
"light_vsx/",
"aavso_vsx/",
do_phase=do_phase,
do_light=do_light,
do_light_raw=do_light,
do_aavso=do_aavso,
nr_threads=thread_count,
jdfilter=args.jdfilter,
desc="Phase/light/aavso of VSX stars",
)
if args.radeccatalog or args.localidcatalog:
do_charts_vast.run(
selected_stars,
comp_stars,
vastdir,
resultdir,
"phase_selected/",
"light_selected/",
"aavso_selected",
do_phase=do_phase,
do_light=do_light,
do_light_raw=do_light,
do_aavso=do_aavso,
nr_threads=thread_count,
jdfilter=args.jdfilter,
desc="Phase/light/aavso of selected stars",
)
if args.candidates:
logging.info(f"Plotting {len(candidate_stars)} candidates...")
do_charts_vast.run(
candidate_stars,
comp_stars,
vastdir,
resultdir,
"phase_candidates/",
"light_candidates/",
"aavso_candidates/",
do_phase=do_phase,
do_light=do_light,
do_light_raw=do_light,
do_aavso=do_aavso,
nr_threads=thread_count,
jdfilter=args.jdfilter,
desc="Phase/light/aavso of candidates",
)
# starfiledata is filled in during the phase plotting, so should come after it. Without phase it will be incomplete
ids = [x.local_id for x in selected_stars]
logging.info(
f"Writing selected files with {len(selected_stars)} selected stars: {ids}"
)
write_selected_files(resultdir, vastdir, selected_stars)
# only create fieldcharts dir if we use it
fieldchartsdir = resultdir + "fieldcharts/"
if args.field or args.stats:
trash_and_recreate_dir(fieldchartsdir)
if args.field:
do_charts_field.run_standard_field_charts(star_descriptions, wcs,
fieldchartsdir, wcs_file,
comp_stars)
if args.stats:
do_charts_stats.plot_comparison_stars(fieldchartsdir, selected_stars,
stardict, args.jdfilter)
do_charts_stats.plot_aperture_vs_jd(fieldchartsdir, vastdir,
args.jdfilter)
do_charts_stats.plot_aperture_vs_airmass(fieldchartsdir, vastdir, wcs,
args.jdfilter)
do_charts_stats.plot_merr_vs_jd(fieldchartsdir, selected_stars,
args.jdfilter)
if args.site:
ids = [x.local_id for x in selected_stars]
logging.info(
f"Creating HTML site with {len(selected_stars)} selected stars: {ids}"
)
hugo_site.run(args.site, selected_stars, len(vsx_stars),
reference_frame_path, resultdir, args.explore)
def run_do_rest(args):
thread_count = args.threads
vastdir = utils.add_trailing_slash(args.datadir)
resultdir = clean_and_create_resultdir(args.resultdir, vastdir)
fieldchartsdir = resultdir + "fieldcharts/"
do_light = args.light
do_phase = args.phase
do_aavso = args.aavso
logging.info(
f"Dir with VaST files: '{vastdir}'\nResults dir: '{resultdir}'")
# get wcs model from the reference header. Used in writing world positions and field charts (can fail)
wcs_file, wcs = reading.read_wcs_file(vastdir)
ref_jd, _, _, reference_frame = reading.extract_reference_frame(vastdir)
_, _, _, first_frame = reading.extract_first_frame(vastdir)
referene_frame_path = Path(reference_frame)
reference_frame_filename = referene_frame_path.name
logging.info(f"The reference frame is '{reference_frame}' at JD: {ref_jd}")
logging.info(f"The first frame is '{first_frame}'")
logging.info(f"Reference header is '{wcs_file}'")
# Log filtering settings + check that reference frame is not inside of filter
if args.jdfilter:
logging.info(f"Filtering JD's: {args.jdfilter}")
ref_inside_filter = (float(ref_jd) > args.jdfilter[0]
and float(ref_jd) > args.jdfilter[1])
if ref_inside_filter:
if not args.jdrefignore:
assert not ref_inside_filter, "Reference frame JD is filtered"
else:
logging.info(
"Reference frame JD is inside of the JD filter, but you indicated that's ok."
)
if not os.path.isfile(wcs_file):
full_ref_path = Path(args.fitsdir) / reference_frame_filename
if not args.fitsdir and args.apikey:
logging.error(
"There is no plate-solved reference frame {wcs_file}, please specify both --apikey "
"and --fitsdir.")
sys.exit(0)
rotation = reading.extract_reference_frame_rotation(
vastdir, reference_frame_filename)
assert (rotation == 0.0
), f"Error: rotation is {rotation} and should always be 0.0"
subprocess.Popen(
f"python3 ./src/astrometry_api.py --apikey={args.apikey} "
f"--upload={full_ref_path} --newfits={wcs_file} --private --no_commercial",
shell=True,
)
while not os.path.isfile(wcs_file):
logging.info(f"Waiting for the astrometry.net plate solve...")
time.sleep(10)
wcs_file, wcs = reading.read_wcs_file(vastdir)
star_descriptions = construct_star_descriptions(vastdir, resultdir, wcs,
args)
stardict = get_localid_to_sd_dict(star_descriptions)
logging.debug(
f"First (max) 10 star descriptions: "
f"{star_descriptions[:10] if (len(star_descriptions) >= 10) else star_descriptions}"
)
write_augmented_autocandidates(vastdir, resultdir, stardict)
write_augmented_all_stars(vastdir, resultdir, stardict)
candidate_stars = utils.get_stars_with_metadata(star_descriptions,
"CANDIDATE",
exclude=["VSX"])
if args.selectcandidates:
tag_candidates_as_selected(candidate_stars)
logging.info(f"There are {len(candidate_stars)} candidate stars")
vsx_stars = utils.get_stars_with_metadata(star_descriptions, "VSX")
logging.info(f"There are {len(vsx_stars)} vsx stars")
selected_stars = utils.get_stars_with_metadata(star_descriptions,
"SELECTEDTAG")
# if args.selectvsx:
# selected_stars = utils.concat_sd_lists(selected_stars, vsx_stars)
logging.info(f"There are {len(selected_stars)} selected stars")
compstar_needing_stars = utils.concat_sd_lists(selected_stars, vsx_stars,
candidate_stars)
comp_stars = set_comp_stars_and_ucac4(star_descriptions, selected_stars,
args.checkstarfile, vastdir,
stardict, ref_jd)
# Set comp stars for all interesting stars (stars which are interesting enough to measure)
logging.info("Setting per star comparison stars...")
if args.checkstarfile:
utils.add_metadata(star_descriptions,
CompStarData(compstar_ids=comp_stars.ids))
else:
do_compstars.add_closest_compstars(compstar_needing_stars, comp_stars,
10)
logging.info(
f"Using {thread_count} threads for phase plots, lightcurves, ...")
if args.allstars:
do_charts_vast.run(
star_descriptions,
comp_stars,
vastdir,
resultdir,
"phase_all/",
"light_all/",
"aavso_all/",
do_phase=do_phase,
do_light=do_light,
do_light_raw=do_light,
do_aavso=do_aavso,
nr_threads=thread_count,
jdfilter=args.jdfilter,
desc="Phase/light/aavso of ALL stars",
)
else:
if args.vsx:
logging.info(f"Plotting {len(vsx_stars)} vsx stars...")
do_charts_vast.run(
vsx_stars,
comp_stars,
vastdir,
resultdir,
"phase_vsx/",
"light_vsx/",
"aavso_vsx/",
do_phase=do_phase,
do_light=do_light,
do_light_raw=do_light,
do_aavso=do_aavso,
nr_threads=thread_count,
jdfilter=args.jdfilter,
desc="Phase/light/aavso of VSX stars",
)
if args.radeccatalog or args.localidcatalog:
do_charts_vast.run(
selected_stars,
comp_stars,
vastdir,
resultdir,
"phase_selected/",
"light_selected/",
"aavso_selected",
do_phase=do_phase,
do_light=do_light,
do_light_raw=do_light,
do_aavso=do_aavso,
nr_threads=thread_count,
jdfilter=args.jdfilter,
desc="Phase/light/aavso of selected stars",
)
if args.candidates:
logging.info(f"Plotting {len(candidate_stars)} candidates...")
do_charts_vast.run(
candidate_stars,
comp_stars,
vastdir,
resultdir,
"phase_candidates/",
"light_candidates/",
"aavso_candidates/",
do_phase=do_phase,
do_light=do_light,
do_light_raw=do_light,
do_aavso=do_aavso,
nr_threads=thread_count,
jdfilter=args.jdfilter,
desc="Phase/light/aavso of candidates",
)
# starfiledata is filled in during the phase plotting, so should come after it. Without phase it will be incomplete
ids = [x.local_id for x in selected_stars]
logging.info(
f"Writing selected files with {len(selected_stars)} selected stars: {ids}"
)
write_selected_files(resultdir, vastdir, selected_stars)
if args.field:
do_charts_field.run_standard_field_charts(star_descriptions, wcs,
fieldchartsdir, wcs_file,
comp_stars)
if args.stats:
do_charts_stats.plot_comparison_stars(fieldchartsdir, selected_stars,
stardict, args.jdfilter)
do_charts_stats.plot_aperture_vs_jd(fieldchartsdir, vastdir,
args.jdfilter)
do_charts_stats.plot_aperture_vs_airmass(fieldchartsdir, vastdir, wcs,
args.jdfilter)
do_charts_stats.plot_merr_vs_jd(fieldchartsdir, selected_stars,
args.jdfilter)
if args.site:
ids = [x.local_id for x in selected_stars]
logging.info(
f"Creating HTML site with {len(selected_stars)} selected stars: {ids}"
)
hugo_site.run(args.site, selected_stars, len(vsx_stars),
referene_frame_path, resultdir, args.explore)