예제 #1
0
    def collect_individual_checksums(self):
        """This step grabs all of the small checksums files for the release,
        filters out any unwanted files from within them, and adds the remainder
        to self.checksums for subsequent steps to use."""
        bucket = self._get_bucket()
        self.info("File prefix is: {}".format(self.file_prefix))

        # Temporary holding place for checksums
        raw_checksums = []

        def worker(item):
            self.debug("Downloading {}".format(item))
            # TODO: It would be nice to download the associated .asc file
            # and verify against it.
            sums = bucket.get_key(item).get_contents_as_string()
            raw_checksums.append(sums)

        def find_checksums_files():
            self.info("Getting key names from bucket")
            checksum_files = {"beets": [], "checksums": []}
            for key in bucket.list(prefix=self.file_prefix):
                if key.key.endswith(".checksums"):
                    self.debug("Found checksums file: {}".format(key.key))
                    checksum_files["checksums"].append(key.key)
                elif key.key.endswith(".beet"):
                    self.debug("Found beet file: {}".format(key.key))
                    checksum_files["beets"].append(key.key)
                else:
                    self.debug("Ignoring non-checksums file: {}".format(
                        key.key))
            if checksum_files["beets"]:
                self.log("Using beet format")
                return checksum_files["beets"]
            else:
                self.log("Using checksums format")
                return checksum_files["checksums"]

        pool = ThreadPool(self.config["parallelization"])
        pool.map(worker, find_checksums_files())

        for c in raw_checksums:
            for f, info in parse_checksums_file(c).iteritems():
                for pattern in self.config["includes"]:
                    if re.search(pattern, f):
                        if f in self.checksums:
                            self.fatal(
                                "Found duplicate checksum entry for {}, don't know which one to pick."
                                .format(f))
                        if not set(self.config["formats"]) <= set(
                                info["hashes"]):
                            self.fatal(
                                "Missing necessary format for file {}".format(
                                    f))
                        self.debug("Adding checksums for file: {}".format(f))
                        self.checksums[f] = info
                        break
                else:
                    self.debug("Ignoring checksums for file: {}".format(f))
예제 #2
0
    def collect_individual_checksums(self):
        """This step grabs all of the small checksums files for the release,
        filters out any unwanted files from within them, and adds the remainder
        to self.checksums for subsequent steps to use."""
        from boto.s3.connection import S3Connection

        bucket_name = self._get_bucket_name()
        file_prefix = self._get_file_prefix()
        self.info("Bucket name is: {}".format(bucket_name))
        self.info("File prefix is: {}".format(file_prefix))

        self.info("Connecting to S3")
        conn = S3Connection(anon=True)
        self.debug("Successfully connected to S3")
        candidates = conn.get_bucket(bucket_name)

        # Temporary holding place for checksums
        raw_checksums = []

        def worker(item):
            self.debug("Downloading {}".format(item))
            # TODO: It would be nice to download the associated .asc file
            # and verify against it.
            sums = candidates.get_key(item).get_contents_as_string()
            raw_checksums.append(sums)

        def find_checksums_files():
            self.info("Getting key names from bucket")
            for key in candidates.list(prefix=file_prefix):
                if key.key.endswith(".checksums"):
                    self.debug("Found checksums file: {}".format(key.key))
                    yield key.key
                else:
                    self.debug("Ignoring non-checksums file: {}".format(
                        key.key))

        pool = ThreadPool(self.config["parallelization"])
        pool.map(worker, find_checksums_files())

        for c in raw_checksums:
            for f, info in parse_checksums_file(c).iteritems():
                for pattern in self.config["includes"]:
                    if re.search(pattern, f):
                        if f in self.checksums:
                            self.fatal(
                                "Found duplicate checksum entry for {}, don't know which one to pick."
                                .format(f))
                        if not set(self.config["formats"]) <= set(
                                info["hashes"]):
                            self.fatal(
                                "Missing necessary format for file {}".format(
                                    f))
                        self.debug("Adding checksums for file: {}".format(f))
                        self.checksums[f] = info
                        break
                else:
                    self.debug("Ignoring checksums for file: {}".format(f))
예제 #3
0
    def collect_individual_checksums(self):
        """This step grabs all of the small checksums files for the release,
        filters out any unwanted files from within them, and adds the remainder
        to self.checksums for subsequent steps to use."""
        bucket = self._get_bucket()
        self.info("File prefix is: {}".format(self.file_prefix))

        # Temporary holding place for checksums
        raw_checksums = []

        def worker(item):
            self.debug("Downloading {}".format(item))
            # TODO: It would be nice to download the associated .asc file
            # and verify against it.
            sums = bucket.get_key(item).get_contents_as_string()
            raw_checksums.append(sums)

        def find_checksums_files():
            self.info("Getting key names from bucket")
            checksum_files = {"beets": [], "checksums": []}
            for key in bucket.list(prefix=self.file_prefix):
                if key.key.endswith(".checksums"):
                    self.debug("Found checksums file: {}".format(key.key))
                    checksum_files["checksums"].append(key.key)
                elif key.key.endswith(".beet"):
                    self.debug("Found beet file: {}".format(key.key))
                    checksum_files["beets"].append(key.key)
                else:
                    self.debug("Ignoring non-checksums file: {}".format(key.key))
            if checksum_files["beets"]:
                self.log("Using beet format")
                return checksum_files["beets"]
            else:
                self.log("Using checksums format")
                return checksum_files["checksums"]

        pool = ThreadPool(self.config["parallelization"])
        pool.map(worker, find_checksums_files())

        for c in raw_checksums:
            for f, info in parse_checksums_file(c).iteritems():
                for pattern in self.config["includes"]:
                    if re.search(pattern, f):
                        if f in self.checksums:
                            self.fatal("Found duplicate checksum entry for {}, "
                                       "don't know which one to pick.".format(f))
                        if not set(self.config["formats"]) <= set(info["hashes"]):
                            self.fatal("Missing necessary format for file {}".format(f))
                        self.debug("Adding checksums for file: {}".format(f))
                        self.checksums[f] = info
                        break
                else:
                    self.debug("Ignoring checksums for file: {}".format(f))
예제 #4
0
    def collect_individual_checksums(self):
        """This step grabs all of the small checksums files for the release,
        filters out any unwanted files from within them, and adds the remainder
        to self.checksums for subsequent steps to use."""
        from boto.s3.connection import S3Connection

        bucket_name = self._get_bucket_name()
        file_prefix = self._get_file_prefix()
        self.info("Bucket name is: {}".format(bucket_name))
        self.info("File prefix is: {}".format(file_prefix))

        self.info("Connecting to S3")
        conn = S3Connection(anon=True)
        self.debug("Successfully connected to S3")
        candidates = conn.get_bucket(bucket_name)

        # Temporary holding place for checksums
        raw_checksums = []
        def worker(item):
            self.debug("Downloading {}".format(item))
            # TODO: It would be nice to download the associated .asc file
            # and verify against it.
            sums = candidates.get_key(item).get_contents_as_string()
            raw_checksums.append(sums)

        def find_checksums_files():
            self.info("Getting key names from bucket")
            for key in candidates.list(prefix=file_prefix):
                if key.key.endswith(".checksums"):
                    self.debug("Found checksums file: {}".format(key.key))
                    yield key.key
                else:
                    self.debug("Ignoring non-checksums file: {}".format(key.key))

        pool = ThreadPool(self.config["parallelization"])
        pool.map(worker, find_checksums_files())

        for c in raw_checksums:
            for f, info in parse_checksums_file(c).iteritems():
                for pattern in self.config["includes"]:
                    if re.search(pattern, f):
                        if f in self.checksums:
                            self.fatal("Found duplicate checksum entry for {}, don't know which one to pick.".format(f))
                        if not set(self.config["formats"]) <= set(info["hashes"]):
                            self.fatal("Missing necessary format for file {}".format(f))
                        self.debug("Adding checksums for file: {}".format(f))
                        self.checksums[f] = info
                        break
                else:
                    self.debug("Ignoring checksums for file: {}".format(f))