def search(self, query = {}, attributes=[]): #Copy the queyr into a dict where all values #are case insensitive regex regex_query = dict(zip(query.iterkeys(), [re.compile(p, re.IGNORECASE) for p in filter(utils.pattern_from_javascript, query.itervalues())])) #Can we save a table scan? #We have an index, but it will work only if there is no regex #Let's try it blindly try: dn = regex_query['dn'].pattern.lower() if dn in self.shelf.keys(): rawobject = self.shelf[dn] #Found it! (or else we would be in the KeyError handler) #The object was not save with the metadata, so let's make sure its there utils.add_metadata(self.sanitize_result(rawobject, dn), dn) yield rawobject except KeyError: #Our only key is not in the query #no choice but to do a table scan for dn, rawobject in self.shelf.iteritems(): #The object was not save with the metadata, so let's make sure its there utils.add_metadata(self.sanitize_result(rawobject, dn), dn) if self.ldapobjectMatchesQuery(rawobject, regex_query): #If we get here, the objet matches the query #we will copy the object in a new dict yield rawobject
def search(self, query = {}, attributes=[]): base, query = self.convert_query(copy.deepcopy(query)) #Async paged search result_generator = self.connection.paged_search_ext_s(base, ldap.SCOPE_SUBTREE, query, attrlist=attributes, serverctrls=self.serverctrls) for dn,entry in result_generator: # process dn and entry wtf = utils.add_metadata(self.sanitize_result(entry, dn), dn) yield wtf
def write(self, original, ldapobject): '''Saves the ldapobject in a Mongo database ''' #If there is a DN (should be, but we don't force it) #we use it as a spec (criteria) key for update try: #We use a specification so that multiple imports does not #create multiple objects. Making the dn the _id also works #but it makes everybody in the pipeline aware of the hack. spec = { 'dn' : ldapobject['dn'][0] } #Insert private metadata that we will use later #The mongolito.parent field allows to sort results so that #parents are always created first utils.add_metadata(ldapobject, ldapobject['dn'][0]) #I though there would be an implict conversion to dict, but #there is not. Let's make one. #This line works in most cases : #ldapobject = dict(zip(ldapobject.keys(), ldapobject.values())) #But Mongo will fail to insert any binary objects that are in #ldapobject.values(). The zip statement above can be written in #a single line, but the following is clearer imo mongoobject = {} for attribute, value in ldapobject.iteritems(): mongoobject[attribute] = self.convert_to_mongo_datatype(value) #Upsert the object self.collection.update(spec, mongoobject, upsert=True) except KeyError: raise SaveException(-1, 'dn', 'Object does not have a dn attribute') except bson.errors.InvalidDocument: raise SaveException(-1, ldapobject['dn'], 'Encoding the object returned an invalid BSON document') except bson.errors.InvalidStringData: raise SaveException(-1, ldapobject['dn'], 'Object has a value that is not properly encoded')
def run_do_rest(args): thread_count = args.threads vastdir = utils.add_trailing_slash(args.datadir) resultdir = clean_and_create_resultdir(args.resultdir, vastdir) do_light = args.light do_phase = args.phase do_aavso = args.aavso logging.info( f"Dir with VaST files: '{vastdir}'\nResults dir: '{resultdir}'") # get wcs model from the reference header. Used in writing world positions and field charts (can fail) wcs_file, wcs = reading.read_wcs_file(vastdir) ref_jd, _, _, reference_frame = reading.extract_reference_frame(vastdir) _, _, _, first_frame = reading.extract_first_frame(vastdir) reference_frame_path = Path(reference_frame) reference_frame_filename = reference_frame_path.name wcs_file, wcs = perform_astrometry_net(args, vastdir, wcs_file, wcs, reference_frame_filename) logging.info(f"The reference frame is '{reference_frame}' at JD: {ref_jd}") logging.info(f"The first frame is '{first_frame}'") logging.info(f"Reference header is '{wcs_file}'") check_that_reference_image_not_within_jdfilter(ref_jd, args.jdfilter, args.jdrefignore) star_descriptions = construct_star_descriptions(vastdir, resultdir, wcs, args) stardict = get_localid_to_sd_dict(star_descriptions) logging.debug( f"First (max) 10 star descriptions: " f"{star_descriptions[:10] if (len(star_descriptions) >= 10) else star_descriptions}" ) write_augmented_autocandidates(vastdir, resultdir, stardict) write_augmented_all_stars(vastdir, resultdir, stardict) candidate_stars = utils.get_stars_with_metadata(star_descriptions, "CANDIDATE", exclude=["VSX"]) if args.selectcandidates: tag_candidates_as_selected(candidate_stars) logging.info(f"There are {len(candidate_stars)} candidate stars") vsx_stars = utils.get_stars_with_metadata(star_descriptions, "VSX") logging.info(f"There are {len(vsx_stars)} vsx stars") if args.allstars: tag_all_stars_as_selected(star_descriptions) selected_stars = utils.get_stars_with_metadata(star_descriptions, "SELECTEDTAG") logging.info(f"There are {len(selected_stars)} selected stars") compstar_needing_stars = utils.concat_sd_lists(selected_stars, vsx_stars, candidate_stars) comp_stars = set_comp_stars_and_ucac4(star_descriptions, selected_stars, args.checkstarfile, vastdir, stardict, ref_jd) # Set comp stars for all interesting stars (stars which are interesting enough to measure) logging.info("Setting per star comparison stars...") if args.checkstarfile: utils.add_metadata(star_descriptions, CompStarData(compstar_ids=comp_stars.ids)) else: do_compstars.add_closest_compstars(compstar_needing_stars, comp_stars, 10) logging.info( f"Using {thread_count} threads for phase plots, lightcurves, ...") if args.allstars: do_charts_vast.run( star_descriptions, comp_stars, vastdir, resultdir, "phase_all/", "light_all/", "aavso_all/", do_phase=do_phase, do_light=do_light, do_light_raw=do_light, do_aavso=do_aavso, nr_threads=thread_count, jdfilter=args.jdfilter, desc="Phase/light/aavso of ALL stars", ) else: if args.vsx: logging.info(f"Plotting {len(vsx_stars)} vsx stars...") do_charts_vast.run( vsx_stars, comp_stars, vastdir, resultdir, "phase_vsx/", "light_vsx/", "aavso_vsx/", do_phase=do_phase, do_light=do_light, do_light_raw=do_light, do_aavso=do_aavso, nr_threads=thread_count, jdfilter=args.jdfilter, desc="Phase/light/aavso of VSX stars", ) if args.radeccatalog or args.localidcatalog: do_charts_vast.run( selected_stars, comp_stars, vastdir, resultdir, "phase_selected/", "light_selected/", "aavso_selected", do_phase=do_phase, do_light=do_light, do_light_raw=do_light, do_aavso=do_aavso, nr_threads=thread_count, jdfilter=args.jdfilter, desc="Phase/light/aavso of selected stars", ) if args.candidates: logging.info(f"Plotting {len(candidate_stars)} candidates...") do_charts_vast.run( candidate_stars, comp_stars, vastdir, resultdir, "phase_candidates/", "light_candidates/", "aavso_candidates/", do_phase=do_phase, do_light=do_light, do_light_raw=do_light, do_aavso=do_aavso, nr_threads=thread_count, jdfilter=args.jdfilter, desc="Phase/light/aavso of candidates", ) # starfiledata is filled in during the phase plotting, so should come after it. Without phase it will be incomplete ids = [x.local_id for x in selected_stars] logging.info( f"Writing selected files with {len(selected_stars)} selected stars: {ids}" ) write_selected_files(resultdir, vastdir, selected_stars) # only create fieldcharts dir if we use it fieldchartsdir = resultdir + "fieldcharts/" if args.field or args.stats: trash_and_recreate_dir(fieldchartsdir) if args.field: do_charts_field.run_standard_field_charts(star_descriptions, wcs, fieldchartsdir, wcs_file, comp_stars) if args.stats: do_charts_stats.plot_comparison_stars(fieldchartsdir, selected_stars, stardict, args.jdfilter) do_charts_stats.plot_aperture_vs_jd(fieldchartsdir, vastdir, args.jdfilter) do_charts_stats.plot_aperture_vs_airmass(fieldchartsdir, vastdir, wcs, args.jdfilter) do_charts_stats.plot_merr_vs_jd(fieldchartsdir, selected_stars, args.jdfilter) if args.site: ids = [x.local_id for x in selected_stars] logging.info( f"Creating HTML site with {len(selected_stars)} selected stars: {ids}" ) hugo_site.run(args.site, selected_stars, len(vsx_stars), reference_frame_path, resultdir, args.explore)
def run_do_rest(args): thread_count = args.threads vastdir = utils.add_trailing_slash(args.datadir) resultdir = clean_and_create_resultdir(args.resultdir, vastdir) fieldchartsdir = resultdir + "fieldcharts/" do_light = args.light do_phase = args.phase do_aavso = args.aavso logging.info( f"Dir with VaST files: '{vastdir}'\nResults dir: '{resultdir}'") # get wcs model from the reference header. Used in writing world positions and field charts (can fail) wcs_file, wcs = reading.read_wcs_file(vastdir) ref_jd, _, _, reference_frame = reading.extract_reference_frame(vastdir) _, _, _, first_frame = reading.extract_first_frame(vastdir) referene_frame_path = Path(reference_frame) reference_frame_filename = referene_frame_path.name logging.info(f"The reference frame is '{reference_frame}' at JD: {ref_jd}") logging.info(f"The first frame is '{first_frame}'") logging.info(f"Reference header is '{wcs_file}'") # Log filtering settings + check that reference frame is not inside of filter if args.jdfilter: logging.info(f"Filtering JD's: {args.jdfilter}") ref_inside_filter = (float(ref_jd) > args.jdfilter[0] and float(ref_jd) > args.jdfilter[1]) if ref_inside_filter: if not args.jdrefignore: assert not ref_inside_filter, "Reference frame JD is filtered" else: logging.info( "Reference frame JD is inside of the JD filter, but you indicated that's ok." ) if not os.path.isfile(wcs_file): full_ref_path = Path(args.fitsdir) / reference_frame_filename if not args.fitsdir and args.apikey: logging.error( "There is no plate-solved reference frame {wcs_file}, please specify both --apikey " "and --fitsdir.") sys.exit(0) rotation = reading.extract_reference_frame_rotation( vastdir, reference_frame_filename) assert (rotation == 0.0 ), f"Error: rotation is {rotation} and should always be 0.0" subprocess.Popen( f"python3 ./src/astrometry_api.py --apikey={args.apikey} " f"--upload={full_ref_path} --newfits={wcs_file} --private --no_commercial", shell=True, ) while not os.path.isfile(wcs_file): logging.info(f"Waiting for the astrometry.net plate solve...") time.sleep(10) wcs_file, wcs = reading.read_wcs_file(vastdir) star_descriptions = construct_star_descriptions(vastdir, resultdir, wcs, args) stardict = get_localid_to_sd_dict(star_descriptions) logging.debug( f"First (max) 10 star descriptions: " f"{star_descriptions[:10] if (len(star_descriptions) >= 10) else star_descriptions}" ) write_augmented_autocandidates(vastdir, resultdir, stardict) write_augmented_all_stars(vastdir, resultdir, stardict) candidate_stars = utils.get_stars_with_metadata(star_descriptions, "CANDIDATE", exclude=["VSX"]) if args.selectcandidates: tag_candidates_as_selected(candidate_stars) logging.info(f"There are {len(candidate_stars)} candidate stars") vsx_stars = utils.get_stars_with_metadata(star_descriptions, "VSX") logging.info(f"There are {len(vsx_stars)} vsx stars") selected_stars = utils.get_stars_with_metadata(star_descriptions, "SELECTEDTAG") # if args.selectvsx: # selected_stars = utils.concat_sd_lists(selected_stars, vsx_stars) logging.info(f"There are {len(selected_stars)} selected stars") compstar_needing_stars = utils.concat_sd_lists(selected_stars, vsx_stars, candidate_stars) comp_stars = set_comp_stars_and_ucac4(star_descriptions, selected_stars, args.checkstarfile, vastdir, stardict, ref_jd) # Set comp stars for all interesting stars (stars which are interesting enough to measure) logging.info("Setting per star comparison stars...") if args.checkstarfile: utils.add_metadata(star_descriptions, CompStarData(compstar_ids=comp_stars.ids)) else: do_compstars.add_closest_compstars(compstar_needing_stars, comp_stars, 10) logging.info( f"Using {thread_count} threads for phase plots, lightcurves, ...") if args.allstars: do_charts_vast.run( star_descriptions, comp_stars, vastdir, resultdir, "phase_all/", "light_all/", "aavso_all/", do_phase=do_phase, do_light=do_light, do_light_raw=do_light, do_aavso=do_aavso, nr_threads=thread_count, jdfilter=args.jdfilter, desc="Phase/light/aavso of ALL stars", ) else: if args.vsx: logging.info(f"Plotting {len(vsx_stars)} vsx stars...") do_charts_vast.run( vsx_stars, comp_stars, vastdir, resultdir, "phase_vsx/", "light_vsx/", "aavso_vsx/", do_phase=do_phase, do_light=do_light, do_light_raw=do_light, do_aavso=do_aavso, nr_threads=thread_count, jdfilter=args.jdfilter, desc="Phase/light/aavso of VSX stars", ) if args.radeccatalog or args.localidcatalog: do_charts_vast.run( selected_stars, comp_stars, vastdir, resultdir, "phase_selected/", "light_selected/", "aavso_selected", do_phase=do_phase, do_light=do_light, do_light_raw=do_light, do_aavso=do_aavso, nr_threads=thread_count, jdfilter=args.jdfilter, desc="Phase/light/aavso of selected stars", ) if args.candidates: logging.info(f"Plotting {len(candidate_stars)} candidates...") do_charts_vast.run( candidate_stars, comp_stars, vastdir, resultdir, "phase_candidates/", "light_candidates/", "aavso_candidates/", do_phase=do_phase, do_light=do_light, do_light_raw=do_light, do_aavso=do_aavso, nr_threads=thread_count, jdfilter=args.jdfilter, desc="Phase/light/aavso of candidates", ) # starfiledata is filled in during the phase plotting, so should come after it. Without phase it will be incomplete ids = [x.local_id for x in selected_stars] logging.info( f"Writing selected files with {len(selected_stars)} selected stars: {ids}" ) write_selected_files(resultdir, vastdir, selected_stars) if args.field: do_charts_field.run_standard_field_charts(star_descriptions, wcs, fieldchartsdir, wcs_file, comp_stars) if args.stats: do_charts_stats.plot_comparison_stars(fieldchartsdir, selected_stars, stardict, args.jdfilter) do_charts_stats.plot_aperture_vs_jd(fieldchartsdir, vastdir, args.jdfilter) do_charts_stats.plot_aperture_vs_airmass(fieldchartsdir, vastdir, wcs, args.jdfilter) do_charts_stats.plot_merr_vs_jd(fieldchartsdir, selected_stars, args.jdfilter) if args.site: ids = [x.local_id for x in selected_stars] logging.info( f"Creating HTML site with {len(selected_stars)} selected stars: {ids}" ) hugo_site.run(args.site, selected_stars, len(vsx_stars), referene_frame_path, resultdir, args.explore)