Exemplo n.º 1
0
    def search(self, query = {}, attributes=[]):

        #Copy the queyr into a dict where all values
        #are case insensitive regex
        regex_query = dict(zip(query.iterkeys(), [re.compile(p, re.IGNORECASE) for p in filter(utils.pattern_from_javascript, query.itervalues())]))

        #Can we save a table scan?
        #We have an index, but it will work only if there is no regex
        #Let's try it blindly
        try:
            dn = regex_query['dn'].pattern.lower()

            if dn in self.shelf.keys():
                rawobject = self.shelf[dn]
                #Found it! (or else we would be in the KeyError handler)
                #The object was not save with the metadata, so let's make sure its there
                utils.add_metadata(self.sanitize_result(rawobject, dn), dn)
                yield rawobject

        except KeyError:
            #Our only key is not in the query
            #no choice but to do a table scan
            for dn, rawobject in self.shelf.iteritems():
                #The object was not save with the metadata, so let's make sure its there
                utils.add_metadata(self.sanitize_result(rawobject, dn), dn)

                if self.ldapobjectMatchesQuery(rawobject, regex_query):
                    #If we get here, the objet matches the query
                    #we will copy the object in a new dict
                    yield rawobject
Exemplo n.º 2
0
    def search(self, query = {}, attributes=[]):
        base, query = self.convert_query(copy.deepcopy(query))

        #Async paged search
        result_generator = self.connection.paged_search_ext_s(base, ldap.SCOPE_SUBTREE, query, attrlist=attributes, serverctrls=self.serverctrls)

        for dn,entry in result_generator:
            # process dn and entry
            wtf = utils.add_metadata(self.sanitize_result(entry, dn), dn)
            yield wtf
Exemplo n.º 3
0
    def write(self, original, ldapobject):
        '''Saves the ldapobject in a Mongo database
        '''
        #If there is a DN (should be, but we don't force it)
        #we use it as a spec (criteria) key for update
     
        try:
            #We use a specification so that multiple imports does not
            #create multiple objects. Making the dn the _id also works
            #but it makes everybody in the pipeline aware of the hack.
            spec = { 'dn' : ldapobject['dn'][0] }
            #Insert private metadata that we will use later
            #The mongolito.parent field allows to sort results so that
            #parents are always created first
            
            utils.add_metadata(ldapobject, ldapobject['dn'][0])
            
            #I though there would be an implict conversion to dict, but
            #there is not. Let's make one.
            #This line works in most cases :
            #ldapobject = dict(zip(ldapobject.keys(), ldapobject.values()))
            #But Mongo will fail to insert any binary objects that are in 
            #ldapobject.values(). The zip statement above can be written in
            #a single line, but the following is clearer imo
            mongoobject = {}

            for attribute, value in ldapobject.iteritems():
                mongoobject[attribute] = self.convert_to_mongo_datatype(value)

            #Upsert the object
            self.collection.update(spec, mongoobject, upsert=True)

        except KeyError:
            raise SaveException(-1, 'dn', 'Object does not have a dn attribute')
        except bson.errors.InvalidDocument:
            raise SaveException(-1, ldapobject['dn'], 'Encoding the object returned an invalid BSON document')
        except bson.errors.InvalidStringData:
            raise SaveException(-1, ldapobject['dn'], 'Object has a value that is not properly encoded')
Exemplo n.º 4
0
def run_do_rest(args):
    thread_count = args.threads
    vastdir = utils.add_trailing_slash(args.datadir)
    resultdir = clean_and_create_resultdir(args.resultdir, vastdir)
    do_light = args.light
    do_phase = args.phase
    do_aavso = args.aavso
    logging.info(
        f"Dir with VaST files: '{vastdir}'\nResults dir: '{resultdir}'")
    # get wcs model from the reference header. Used in writing world positions and field charts (can fail)
    wcs_file, wcs = reading.read_wcs_file(vastdir)
    ref_jd, _, _, reference_frame = reading.extract_reference_frame(vastdir)
    _, _, _, first_frame = reading.extract_first_frame(vastdir)
    reference_frame_path = Path(reference_frame)
    reference_frame_filename = reference_frame_path.name
    wcs_file, wcs = perform_astrometry_net(args, vastdir, wcs_file, wcs,
                                           reference_frame_filename)
    logging.info(f"The reference frame is '{reference_frame}' at JD: {ref_jd}")
    logging.info(f"The first frame is '{first_frame}'")
    logging.info(f"Reference header is '{wcs_file}'")
    check_that_reference_image_not_within_jdfilter(ref_jd, args.jdfilter,
                                                   args.jdrefignore)

    star_descriptions = construct_star_descriptions(vastdir, resultdir, wcs,
                                                    args)
    stardict = get_localid_to_sd_dict(star_descriptions)
    logging.debug(
        f"First (max) 10 star descriptions: "
        f"{star_descriptions[:10] if (len(star_descriptions) >= 10) else star_descriptions}"
    )
    write_augmented_autocandidates(vastdir, resultdir, stardict)
    write_augmented_all_stars(vastdir, resultdir, stardict)
    candidate_stars = utils.get_stars_with_metadata(star_descriptions,
                                                    "CANDIDATE",
                                                    exclude=["VSX"])
    if args.selectcandidates:
        tag_candidates_as_selected(candidate_stars)
    logging.info(f"There are {len(candidate_stars)} candidate stars")

    vsx_stars = utils.get_stars_with_metadata(star_descriptions, "VSX")
    logging.info(f"There are {len(vsx_stars)} vsx stars")
    if args.allstars:
        tag_all_stars_as_selected(star_descriptions)
    selected_stars = utils.get_stars_with_metadata(star_descriptions,
                                                   "SELECTEDTAG")
    logging.info(f"There are {len(selected_stars)} selected stars")
    compstar_needing_stars = utils.concat_sd_lists(selected_stars, vsx_stars,
                                                   candidate_stars)
    comp_stars = set_comp_stars_and_ucac4(star_descriptions, selected_stars,
                                          args.checkstarfile, vastdir,
                                          stardict, ref_jd)
    # Set comp stars for all interesting stars (stars which are interesting enough to measure)
    logging.info("Setting per star comparison stars...")
    if args.checkstarfile:
        utils.add_metadata(star_descriptions,
                           CompStarData(compstar_ids=comp_stars.ids))
    else:
        do_compstars.add_closest_compstars(compstar_needing_stars, comp_stars,
                                           10)

    logging.info(
        f"Using {thread_count} threads for phase plots, lightcurves, ...")
    if args.allstars:
        do_charts_vast.run(
            star_descriptions,
            comp_stars,
            vastdir,
            resultdir,
            "phase_all/",
            "light_all/",
            "aavso_all/",
            do_phase=do_phase,
            do_light=do_light,
            do_light_raw=do_light,
            do_aavso=do_aavso,
            nr_threads=thread_count,
            jdfilter=args.jdfilter,
            desc="Phase/light/aavso of ALL stars",
        )
    else:
        if args.vsx:
            logging.info(f"Plotting {len(vsx_stars)} vsx stars...")
            do_charts_vast.run(
                vsx_stars,
                comp_stars,
                vastdir,
                resultdir,
                "phase_vsx/",
                "light_vsx/",
                "aavso_vsx/",
                do_phase=do_phase,
                do_light=do_light,
                do_light_raw=do_light,
                do_aavso=do_aavso,
                nr_threads=thread_count,
                jdfilter=args.jdfilter,
                desc="Phase/light/aavso of VSX stars",
            )
        if args.radeccatalog or args.localidcatalog:
            do_charts_vast.run(
                selected_stars,
                comp_stars,
                vastdir,
                resultdir,
                "phase_selected/",
                "light_selected/",
                "aavso_selected",
                do_phase=do_phase,
                do_light=do_light,
                do_light_raw=do_light,
                do_aavso=do_aavso,
                nr_threads=thread_count,
                jdfilter=args.jdfilter,
                desc="Phase/light/aavso of selected stars",
            )
        if args.candidates:
            logging.info(f"Plotting {len(candidate_stars)} candidates...")
            do_charts_vast.run(
                candidate_stars,
                comp_stars,
                vastdir,
                resultdir,
                "phase_candidates/",
                "light_candidates/",
                "aavso_candidates/",
                do_phase=do_phase,
                do_light=do_light,
                do_light_raw=do_light,
                do_aavso=do_aavso,
                nr_threads=thread_count,
                jdfilter=args.jdfilter,
                desc="Phase/light/aavso of candidates",
            )

    # starfiledata is filled in during the phase plotting, so should come after it. Without phase it will be incomplete
    ids = [x.local_id for x in selected_stars]
    logging.info(
        f"Writing selected files with {len(selected_stars)}  selected stars: {ids}"
    )
    write_selected_files(resultdir, vastdir, selected_stars)

    # only create fieldcharts dir if we use it
    fieldchartsdir = resultdir + "fieldcharts/"
    if args.field or args.stats:
        trash_and_recreate_dir(fieldchartsdir)

    if args.field:
        do_charts_field.run_standard_field_charts(star_descriptions, wcs,
                                                  fieldchartsdir, wcs_file,
                                                  comp_stars)

    if args.stats:
        do_charts_stats.plot_comparison_stars(fieldchartsdir, selected_stars,
                                              stardict, args.jdfilter)
        do_charts_stats.plot_aperture_vs_jd(fieldchartsdir, vastdir,
                                            args.jdfilter)
        do_charts_stats.plot_aperture_vs_airmass(fieldchartsdir, vastdir, wcs,
                                                 args.jdfilter)
        do_charts_stats.plot_merr_vs_jd(fieldchartsdir, selected_stars,
                                        args.jdfilter)

    if args.site:
        ids = [x.local_id for x in selected_stars]
        logging.info(
            f"Creating HTML site with {len(selected_stars)} selected stars: {ids}"
        )
        hugo_site.run(args.site, selected_stars, len(vsx_stars),
                      reference_frame_path, resultdir, args.explore)
Exemplo n.º 5
0
def run_do_rest(args):
    thread_count = args.threads
    vastdir = utils.add_trailing_slash(args.datadir)
    resultdir = clean_and_create_resultdir(args.resultdir, vastdir)
    fieldchartsdir = resultdir + "fieldcharts/"
    do_light = args.light
    do_phase = args.phase
    do_aavso = args.aavso
    logging.info(
        f"Dir with VaST files: '{vastdir}'\nResults dir: '{resultdir}'")
    # get wcs model from the reference header. Used in writing world positions and field charts (can fail)
    wcs_file, wcs = reading.read_wcs_file(vastdir)
    ref_jd, _, _, reference_frame = reading.extract_reference_frame(vastdir)
    _, _, _, first_frame = reading.extract_first_frame(vastdir)
    referene_frame_path = Path(reference_frame)
    reference_frame_filename = referene_frame_path.name
    logging.info(f"The reference frame is '{reference_frame}' at JD: {ref_jd}")
    logging.info(f"The first frame is '{first_frame}'")
    logging.info(f"Reference header is '{wcs_file}'")
    # Log filtering settings + check that reference frame is not inside of filter
    if args.jdfilter:
        logging.info(f"Filtering JD's: {args.jdfilter}")
        ref_inside_filter = (float(ref_jd) > args.jdfilter[0]
                             and float(ref_jd) > args.jdfilter[1])
        if ref_inside_filter:
            if not args.jdrefignore:
                assert not ref_inside_filter, "Reference frame JD is filtered"
            else:
                logging.info(
                    "Reference frame JD is inside of the JD filter, but you indicated that's ok."
                )

    if not os.path.isfile(wcs_file):
        full_ref_path = Path(args.fitsdir) / reference_frame_filename
        if not args.fitsdir and args.apikey:
            logging.error(
                "There is no plate-solved reference frame {wcs_file}, please specify both --apikey "
                "and --fitsdir.")
            sys.exit(0)
        rotation = reading.extract_reference_frame_rotation(
            vastdir, reference_frame_filename)
        assert (rotation == 0.0
                ), f"Error: rotation is {rotation} and should always be 0.0"
        subprocess.Popen(
            f"python3 ./src/astrometry_api.py --apikey={args.apikey} "
            f"--upload={full_ref_path} --newfits={wcs_file} --private --no_commercial",
            shell=True,
        )
        while not os.path.isfile(wcs_file):
            logging.info(f"Waiting for the astrometry.net plate solve...")
            time.sleep(10)
        wcs_file, wcs = reading.read_wcs_file(vastdir)

    star_descriptions = construct_star_descriptions(vastdir, resultdir, wcs,
                                                    args)
    stardict = get_localid_to_sd_dict(star_descriptions)
    logging.debug(
        f"First (max) 10 star descriptions: "
        f"{star_descriptions[:10] if (len(star_descriptions) >= 10) else star_descriptions}"
    )
    write_augmented_autocandidates(vastdir, resultdir, stardict)
    write_augmented_all_stars(vastdir, resultdir, stardict)
    candidate_stars = utils.get_stars_with_metadata(star_descriptions,
                                                    "CANDIDATE",
                                                    exclude=["VSX"])
    if args.selectcandidates:
        tag_candidates_as_selected(candidate_stars)
    logging.info(f"There are {len(candidate_stars)} candidate stars")

    vsx_stars = utils.get_stars_with_metadata(star_descriptions, "VSX")
    logging.info(f"There are {len(vsx_stars)} vsx stars")
    selected_stars = utils.get_stars_with_metadata(star_descriptions,
                                                   "SELECTEDTAG")
    # if args.selectvsx:
    #     selected_stars = utils.concat_sd_lists(selected_stars, vsx_stars)
    logging.info(f"There are {len(selected_stars)} selected stars")
    compstar_needing_stars = utils.concat_sd_lists(selected_stars, vsx_stars,
                                                   candidate_stars)
    comp_stars = set_comp_stars_and_ucac4(star_descriptions, selected_stars,
                                          args.checkstarfile, vastdir,
                                          stardict, ref_jd)
    # Set comp stars for all interesting stars (stars which are interesting enough to measure)
    logging.info("Setting per star comparison stars...")
    if args.checkstarfile:
        utils.add_metadata(star_descriptions,
                           CompStarData(compstar_ids=comp_stars.ids))
    else:
        do_compstars.add_closest_compstars(compstar_needing_stars, comp_stars,
                                           10)

    logging.info(
        f"Using {thread_count} threads for phase plots, lightcurves, ...")
    if args.allstars:
        do_charts_vast.run(
            star_descriptions,
            comp_stars,
            vastdir,
            resultdir,
            "phase_all/",
            "light_all/",
            "aavso_all/",
            do_phase=do_phase,
            do_light=do_light,
            do_light_raw=do_light,
            do_aavso=do_aavso,
            nr_threads=thread_count,
            jdfilter=args.jdfilter,
            desc="Phase/light/aavso of ALL stars",
        )
    else:
        if args.vsx:
            logging.info(f"Plotting {len(vsx_stars)} vsx stars...")
            do_charts_vast.run(
                vsx_stars,
                comp_stars,
                vastdir,
                resultdir,
                "phase_vsx/",
                "light_vsx/",
                "aavso_vsx/",
                do_phase=do_phase,
                do_light=do_light,
                do_light_raw=do_light,
                do_aavso=do_aavso,
                nr_threads=thread_count,
                jdfilter=args.jdfilter,
                desc="Phase/light/aavso of VSX stars",
            )
        if args.radeccatalog or args.localidcatalog:
            do_charts_vast.run(
                selected_stars,
                comp_stars,
                vastdir,
                resultdir,
                "phase_selected/",
                "light_selected/",
                "aavso_selected",
                do_phase=do_phase,
                do_light=do_light,
                do_light_raw=do_light,
                do_aavso=do_aavso,
                nr_threads=thread_count,
                jdfilter=args.jdfilter,
                desc="Phase/light/aavso of selected stars",
            )
        if args.candidates:
            logging.info(f"Plotting {len(candidate_stars)} candidates...")
            do_charts_vast.run(
                candidate_stars,
                comp_stars,
                vastdir,
                resultdir,
                "phase_candidates/",
                "light_candidates/",
                "aavso_candidates/",
                do_phase=do_phase,
                do_light=do_light,
                do_light_raw=do_light,
                do_aavso=do_aavso,
                nr_threads=thread_count,
                jdfilter=args.jdfilter,
                desc="Phase/light/aavso of candidates",
            )

    # starfiledata is filled in during the phase plotting, so should come after it. Without phase it will be incomplete
    ids = [x.local_id for x in selected_stars]
    logging.info(
        f"Writing selected files with {len(selected_stars)}  selected stars: {ids}"
    )
    write_selected_files(resultdir, vastdir, selected_stars)
    if args.field:
        do_charts_field.run_standard_field_charts(star_descriptions, wcs,
                                                  fieldchartsdir, wcs_file,
                                                  comp_stars)

    if args.stats:
        do_charts_stats.plot_comparison_stars(fieldchartsdir, selected_stars,
                                              stardict, args.jdfilter)
        do_charts_stats.plot_aperture_vs_jd(fieldchartsdir, vastdir,
                                            args.jdfilter)
        do_charts_stats.plot_aperture_vs_airmass(fieldchartsdir, vastdir, wcs,
                                                 args.jdfilter)
        do_charts_stats.plot_merr_vs_jd(fieldchartsdir, selected_stars,
                                        args.jdfilter)

    if args.site:
        ids = [x.local_id for x in selected_stars]
        logging.info(
            f"Creating HTML site with {len(selected_stars)} selected stars: {ids}"
        )
        hugo_site.run(args.site, selected_stars, len(vsx_stars),
                      referene_frame_path, resultdir, args.explore)