Ejemplo n.º 1
0
    def pull(self):
        # return cached file if it exists
        if os.path.isfile(self.cached_fname):
            if (self.verbose):
                print_info("Found cached file at %s" % self.cached_fname)
            return self.cached_fname

        from boutiques.searcher import Searcher
        searcher = Searcher(self.zid,
                            self.verbose,
                            self.sandbox,
                            exact_match=True)
        r = searcher.zenodo_search()

        for hit in r.json()["hits"]["hits"]:
            file_path = hit["files"][0]["links"]["self"]
            file_name = file_path.split(os.sep)[-1]
            if hit["id"] == int(self.zid):
                if not os.path.exists(self.cache_dir):
                    os.makedirs(self.cache_dir)
                if (self.verbose):
                    print_info("Downloading descriptor %s" % file_name)
                downloaded = urlretrieve(file_path, self.cached_fname)
                print("Downloaded descriptor to " + downloaded[0])
                return downloaded[0]
        raise_error(ZenodoError, "Descriptor not found")
Ejemplo n.º 2
0
def search(*params):
    parser = ArgumentParser("Search Zenodo for Boutiques descriptors. "
                            "When no term is supplied, will search for "
                            "all descriptors.")
    parser.add_argument("query", nargs="?", default="boutiques",
                        action="store", help="Search query")
    parser.add_argument("-v", "--verbose", action="store_true",
                        help="Print information messages")
    parser.add_argument("--sandbox", action="store_true",
                        help="search Zenodo's sandbox instead of "
                        "production server. Recommended for tests.")
    parser.add_argument("-m", "--max", action="store", type=int,
                        help="Specify the maximum number of results "
                        "to be returned. Default is 10.")
    parser.add_argument("-nt", "--no-trunc", action="store_true",
                        help="Do not truncate long tool descriptions.")
    parser.add_argument("-e", "--exact", action="store_true",
                        help="Only return results containing the exact query.")

    result = parser.parse_args(params)

    from boutiques.searcher import Searcher
    searcher = Searcher(result.query, result.verbose, result.sandbox,
                        result.max, result.no_trunc, result.exact)

    return searcher.search()
Ejemplo n.º 3
0
    def run(self):
        try:
            boutique_cache_dir = os.path.join(os.path.expanduser('~'),
                                              ".cache", "boutiques",
                                              "production")
            # first search for all descriptors
            searcher = Searcher(query=None, max_results=100, no_trunc=True)
            all_descriptors = searcher.search()
            # then pull every single descriptor
            all_descriptor_ids = list(map(lambda x: x["ID"], all_descriptors))
            files = Puller(all_descriptor_ids).pull()

            # fetch every single descriptor into one file
            detailed_all_descriptors = list(
                map(lambda f: json.load(open(f, 'r')), files))

            # store data in cache
            with open(os.path.join(boutique_cache_dir, "all_descriptors.json"),
                      "w") as f:
                json.dump(all_descriptors, f, indent=4)

            with open(
                    os.path.join(boutique_cache_dir,
                                 "detailed_all_descriptors.json"), "w") as f:
                json.dump(detailed_all_descriptors, f, indent=4)

        except Exception as e:
            logging.exception(
                "An exception occurred in the thread:{0}.".format(e))
Ejemplo n.º 4
0
    def publish(self):
        if (not self.no_int):
            prompt = ("The descriptor will be published to Zenodo, "
                      "this cannot be undone. Are you sure? (Y/n) ")
            try:
                ret = raw_input(prompt)  # Python 2
            except NameError:
                ret = input(prompt)  # Python 3
            if ret.upper() != "Y":
                return

        if self.id_to_update is not None:
            publish_update = True
        else:
            # perform a search to check if descriptor is an updated version
            # of an existing one
            from boutiques.searcher import Searcher
            searcher = Searcher(self.descriptor.get("name"),
                                self.verbose,
                                self.sandbox,
                                exact_match=True)
            r = searcher.zenodo_search()

            publish_update = False
            for hit in r.json()["hits"]["hits"]:
                title = hit["metadata"]["title"]
                if title == self.descriptor.get("name"):
                    self.id_to_update = hit["id"]
                    break

            if self.id_to_update is not None:
                if (not self.no_int):
                    prompt = ("Found an existing record with the same name, "
                              "would you like to update it? "
                              "(Y:Update existing / n:Publish new entry with "
                              "name {}) ".format(self.descriptor.get("name")))
                    try:
                        ret = raw_input(prompt)  # Python 2
                    except NameError:
                        ret = input(prompt)  # Python 3
                    if ret.upper() == "Y":
                        publish_update = True
                else:
                    publish_update = True

        if publish_update:
            deposition_id = self.zenodo_helper.zenodo_deposit_updated_version(
                self.create_metadata(), self.zenodo_access_token,
                self.id_to_update)
        else:
            deposition_id = self.zenodo_helper.zenodo_deposit(
                self.create_metadata(), self.zenodo_access_token)

        self.zenodo_upload_descriptor(deposition_id)
        self.doi = self.zenodo_helper.zenodo_publish(self.zenodo_access_token,
                                                     deposition_id,
                                                     "Descriptor")
        self.descriptor['doi'] = self.doi
        with open(self.descriptor_file_name, "w") as f:
            f.write(json.dumps(self.descriptor, indent=4, sort_keys=True))
Ejemplo n.º 5
0
    def run(self):
        try:
            # if cache directory doesn't exist then create it
            if not os.path.exists(self.cache_dir):
                os.makedirs(self.cache_dir)

            # first search for all descriptors
            searcher = Searcher(query="", max_results=100, no_trunc=True)
            all_descriptors = searcher.search()

            # then pull every single descriptor
            all_descriptor_ids = list(map(lambda x: x["ID"], all_descriptors))
            Puller(all_descriptor_ids).pull()

            # fetch every single descriptor into one file
            detailed_all_descriptors = [
                json.load(open(os.path.join(self.cache_dir,
                                            descriptor["ID"].replace(".", "-") + ".json"),
                          "r"))
                for descriptor in all_descriptors
            ]

            # store data in cache
            with open(os.path.join(self.cache_dir,
                                   "all_descriptors.json"),
                      "w") as f:
                json.dump(all_descriptors, f, indent=4)

            with open(os.path.join(self.cache_dir,
                                   "detailed_all_descriptors.json"),
                      "w") as f:
                json.dump(detailed_all_descriptors, f, indent=4)

        except Exception as e:
            logging.exception("An exception occurred in the thread.")
Ejemplo n.º 6
0
    def pull(self):
        from boutiques.searcher import Searcher
        searcher = Searcher(self.zid, self.verbose, self.sandbox, None)
        r = searcher.zenodo_search()

        for hit in r.json()["hits"]["hits"]:
            file_path = hit["files"][0]["links"]["self"]
            file_name = file_path.split(os.sep)[-1]
            if hit["id"] == int(self.zid):
                if self.download:
                    cache_dir = os.path.join(os.path.expanduser('~'), ".cache",
                                             "boutiques")
                    if not os.path.exists(cache_dir):
                        os.makedirs(cache_dir)
                    if (self.verbose):
                        print_info("Downloading descriptor %s" % file_name)
                    downloaded = urlretrieve(
                        file_path, os.path.join(cache_dir, file_name))
                    print_info("Downloaded descriptor to " + cache_dir)
                    return downloaded
                if (self.verbose):
                    print_info("Opening descriptor %s" % file_name)
                return urlopen(file_path)

        raise_error(ZenodoError, "Descriptor not found")
Ejemplo n.º 7
0
def search(*params):
    parser = parser_search()
    results = parser.parse_args(params)

    from boutiques.searcher import Searcher
    searcher = Searcher(results.query, results.verbose, results.sandbox,
                        results.max, results.no_trunc, results.exact)

    return searcher.search()
Ejemplo n.º 8
0
    def publish(self):
        if (not self.no_int):
            prompt = ("The descriptor will be published to Zenodo, "
                      "this cannot be undone. Are you sure? (Y/n) ")
            try:
                ret = raw_input(prompt)  # Python 2
            except NameError:
                ret = input(prompt)  # Python 3
            if ret.upper() != "Y":
                return
        self.zenodo_test_api()

        # perform a search to check if descriptor is an updated version
        # of an existing one
        from boutiques.searcher import Searcher
        searcher = Searcher(self.descriptor.get("name"), self.verbose,
                            self.sandbox, None)
        r = searcher.zenodo_search()

        id_to_update = 0
        publish_update = False
        for hit in r.json()["hits"]["hits"]:
            title = hit["metadata"]["title"]
            if title == self.descriptor.get("name"):
                id_to_update = hit["id"]
                break

        if id_to_update:
            if (not self.no_int):
                prompt = ("Found an existing record with the same name, "
                          "would you like to update it? "
                          "(Y:Update existing / n:Publish new entry with "
                          "name {}) ".format(self.descriptor.get("name")))
                try:
                    ret = raw_input(prompt)  # Python 2
                except NameError:
                    ret = input(prompt)  # Python 3
                if ret.upper() == "Y":
                    publish_update = True
            else:
                publish_update = True

        if publish_update:
            self.publish_updated_version(id_to_update)
        else:
            self.publish_new_entry()
Ejemplo n.º 9
0
    def _download_spec(self, zenodo_id):
        """
        usind boutiques Searcher to find url of zenodo file for a specific id,
        and download the file to self.cache_dir
        """
        from boutiques.searcher import Searcher

        searcher = Searcher(zenodo_id, exact_match=True)
        hits = searcher.zenodo_search().json()["hits"]["hits"]
        if len(hits) == 0:
            raise Exception(f"can't find zenodo spec for {zenodo_id}")
        elif len(hits) > 1:
            raise Exception(f"too many hits for {zenodo_id}")
        else:
            zenodo_url = hits[0]["files"][0]["links"]["self"]
            zenodo_file = self.cache_dir / f"zenodo.{zenodo_id}.json"
            urlretrieve(zenodo_url, zenodo_file)
            return zenodo_file
Ejemplo n.º 10
0
    def pull(self):
        # return cached file if it exists
        json_files = []
        for entry in self.zenodo_entries:
            if os.path.isfile(entry["fname"]):
                if (self.verbose):
                    print_info("Found cached file at %s" % entry["fname"])
                json_files.append(entry["fname"])
                continue

            searcher = Searcher(entry["zid"],
                                self.verbose,
                                self.sandbox,
                                exact_match=True)
            r = searcher.zenodo_search()

            if not len(r.json()["hits"]["hits"]):
                raise_error(
                    ZenodoError, "Descriptor \"{0}\" "
                    "not found".format(entry["zid"]))
            for hit in r.json()["hits"]["hits"]:
                file_path = hit["files"][0]["links"]["self"]
                file_name = file_path.split(os.sep)[-1]
                if hit["id"] == int(entry["zid"]):
                    if not os.path.exists(self.cache_dir):
                        os.makedirs(self.cache_dir)
                    if (self.verbose):
                        print_info("Downloading descriptor %s" % file_name)
                    downloaded = urlretrieve(file_path, entry["fname"])
                    if (self.verbose):
                        print_info("Downloaded descriptor to " + downloaded[0])
                    json_files.append(downloaded[0])
                else:
                    raise_error(
                        ZenodoError, "Searched-for descriptor \"{0}\" "
                        "does not match descriptor \"{1}\" returned "
                        "from Zenodo".format(entry["zid"], hit["id"]))

        return json_files