def commit(): if pf.tell() == 0: pf.close() return None pf.seek(0) p = PackData(pf.name, pf) entries = p.sorted_entries() basename = iter_sha1(entry[0] for entry in entries).decode('ascii') idxf = tempfile.SpooledTemporaryFile() checksum = p.get_stored_checksum() write_pack_index_v2(idxf, entries, checksum) idxf.seek(0) idx = load_pack_index_file(basename + '.idx', idxf) for pack in self.packs: if pack.get_stored_checksum() == p.get_stored_checksum(): p.close() idx.close() return pack pf.seek(0) idxf.seek(0) self._upload_pack(basename, pf, idxf) final_pack = Pack.from_objects(p, idx) self._add_cached_pack(basename, final_pack) return final_pack
def _update_pack_cache(self): pack_files = set(self._pack_names()) new_packs = [] for basename in pack_files: pack_name = basename + ".pack" if basename not in self._pack_cache: try: size = self.pack_transport.stat(pack_name).st_size except TransportNotPossible: f = self.pack_transport.get(pack_name) # TODO(jelmer): Don't read entire file into memory? f = BytesIO(f.read()) pd = PackData(pack_name, f) else: pd = PackData( pack_name, self.pack_transport.get(pack_name), size=size) idxname = basename + ".idx" idx = load_pack_index_file( idxname, self.pack_transport.get(idxname)) pack = Pack.from_objects(pd, idx) pack._basename = basename self._pack_cache[basename] = pack new_packs.append(pack) # Remove disappeared pack files for f in set(self._pack_cache) - pack_files: self._pack_cache.pop(f).close() return new_packs
def move_in_pack(self, f): """Move a specific file containing a pack into the pack directory. :note: The file should be on the same file system as the packs directory. :param path: Path to the pack file. """ f.seek(0) p = PackData("", f, len(f.getvalue())) entries = p.sorted_entries() basename = "pack-%s" % iter_sha1(entry[0] for entry in entries).decode('ascii') p._filename = basename + ".pack" f.seek(0) self.pack_transport.put_file(basename + ".pack", f) idxfile = self.pack_transport.open_write_stream(basename + ".idx") try: write_pack_index_v2(idxfile, entries, p.get_stored_checksum()) finally: idxfile.close() idxfile = self.pack_transport.get(basename + ".idx") idx = load_pack_index_file(basename + ".idx", idxfile) final_pack = Pack.from_objects(p, idx) final_pack._basename = basename self._add_known_pack(basename, final_pack) return final_pack
def _update_pack_cache(self): pack_files = set() pack_dir_contents = self._pack_names() for name in pack_dir_contents: if name.startswith("pack-") and name.endswith(".pack"): # verify that idx exists first (otherwise the pack was not yet # fully written) idx_name = os.path.splitext(name)[0] + ".idx" if idx_name in pack_dir_contents: pack_files.add(os.path.splitext(name)[0]) new_packs = [] for basename in pack_files: pack_name = basename + ".pack" if basename not in self._pack_cache: try: size = self.pack_transport.stat(pack_name).st_size except TransportNotPossible: f = self.pack_transport.get(pack_name) pd = PackData(pack_name, f) else: pd = PackData(pack_name, self.pack_transport.get(pack_name), size=size) idxname = basename + ".idx" idx = load_pack_index_file(idxname, self.pack_transport.get(idxname)) pack = Pack.from_objects(pd, idx) pack._basename = basename self._pack_cache[basename] = pack new_packs.append(pack) # Remove disappeared pack files for f in set(self._pack_cache) - pack_files: self._pack_cache.pop(f).close() return new_packs
def _complete_thin_pack(self, f, path, copier, indexer): entries = list(indexer) # Update the header with the new number of objects. f.seek(0) write_pack_header(f, len(entries) + len(indexer.ext_refs())) # Must flush before reading (http://bugs.python.org/issue3207) f.flush() # Rescan the rest of the pack, computing the SHA with the new header. new_sha = compute_file_sha(f, end_ofs=-20) # Must reposition before writing (http://bugs.python.org/issue3207) f.seek(0, os.SEEK_CUR) # Complete the pack. for ext_sha in indexer.ext_refs(): assert len(ext_sha) == 20 type_num, data = self.get_raw(ext_sha) offset = f.tell() crc32 = write_pack_object(f, type_num, data, sha=new_sha) entries.append((ext_sha, offset, crc32)) pack_sha = new_sha.digest() f.write(pack_sha) f.flush() # Move the pack in. entries.sort() pack_base_name = posixpath.join( self.pack_dir, "pack-" + os.fsdecode(iter_sha1(e[0] for e in entries)), ) self.scon.put_object(pack_base_name + ".pack", f) # Write the index. filename = pack_base_name + ".idx" index_file = BytesIO() write_pack_index_v2(index_file, entries, pack_sha) self.scon.put_object(filename, index_file) # Write pack info. f.seek(0) pack_data = PackData(filename="", file=f) index_file.seek(0) pack_index = load_pack_index_file("", index_file) serialized_pack_info = pack_info_create(pack_data, pack_index) f.close() index_file.close() pack_info_file = BytesIO(serialized_pack_info) filename = pack_base_name + ".info" self.scon.put_object(filename, pack_info_file) pack_info_file.close() # Add the pack to the store and return it. final_pack = SwiftPack(pack_base_name, scon=self.scon) final_pack.check_length_and_checksum() self._add_cached_pack(pack_base_name, final_pack) return final_pack
def _complete_thin_pack(self, f, path, copier, indexer): entries = list(indexer) # Update the header with the new number of objects. f.seek(0) write_pack_header(f, len(entries) + len(indexer.ext_refs())) # Must flush before reading (http://bugs.python.org/issue3207) f.flush() # Rescan the rest of the pack, computing the SHA with the new header. new_sha = compute_file_sha(f, end_ofs=-20) # Must reposition before writing (http://bugs.python.org/issue3207) f.seek(0, os.SEEK_CUR) # Complete the pack. for ext_sha in indexer.ext_refs(): assert len(ext_sha) == 20 type_num, data = self.get_raw(ext_sha) offset = f.tell() crc32 = write_pack_object(f, type_num, data, sha=new_sha) entries.append((ext_sha, offset, crc32)) pack_sha = new_sha.digest() f.write(pack_sha) f.flush() # Move the pack in. entries.sort() pack_base_name = posixpath.join( self.pack_dir, 'pack-' + iter_sha1(e[0] for e in entries).decode( sys.getfilesystemencoding())) self.scon.put_object(pack_base_name + '.pack', f) # Write the index. filename = pack_base_name + '.idx' index_file = BytesIO() write_pack_index_v2(index_file, entries, pack_sha) self.scon.put_object(filename, index_file) # Write pack info. f.seek(0) pack_data = PackData(filename="", file=f) index_file.seek(0) pack_index = load_pack_index_file('', index_file) serialized_pack_info = pack_info_create(pack_data, pack_index) f.close() index_file.close() pack_info_file = BytesIO(serialized_pack_info) filename = pack_base_name + '.info' self.scon.put_object(filename, pack_info_file) pack_info_file.close() # Add the pack to the store and return it. final_pack = SwiftPack(pack_base_name, scon=self.scon) final_pack.check_length_and_checksum() self._add_cached_pack(pack_base_name, final_pack) return final_pack
def swift_load_pack_index(scon, filename): """Read a pack index file from Swift :param scon: a `SwiftConnector` instance :param filename: Path to the index file objectise :return: a `PackIndexer` instance """ with scon.get_object(filename) as f: return load_pack_index_file(filename, f)
def idx_loader(): index_tmpfile = tempfile.NamedTemporaryFile() log.debug('Downloading pack index %s into %s' % (path, index_tmpfile)) index_key = self.bucket.new_key('%s.idx' % path) index_key.get_contents_to_file(index_tmpfile) log.debug('Rewinding...') index_tmpfile.flush() index_tmpfile.seek(0) return load_pack_index_file(index_tmpfile.name, index_tmpfile)
def setUp(self): conf = swift.load_conf(file=StringIO(config_file % def_config_file)) sos = swift.SwiftObjectStore(FakeSwiftConnector("fakerepo", conf=conf)) commit_amount = 10 self.commits = create_commits(length=commit_amount, marker="m") data = [(d.type_num, d.as_raw_string()) for d in self.commits] f = StringIO() fi = StringIO() expected = build_pack(f, data, store=sos) entries = [(sha, ofs, checksum) for ofs, _, _, sha, checksum in expected] self.pack_data = PackData.from_file(file=f, size=None) write_pack_index_v2(fi, entries, self.pack_data.calculate_checksum()) fi.seek(0) self.pack_index = load_pack_index_file("", fi)
def setUp(self): conf = swift.load_conf(file=BytesIO(config_file % def_config_file)) sos = swift.SwiftObjectStore(FakeSwiftConnector('fakerepo', conf=conf)) commit_amount = 10 self.commits = create_commits(length=commit_amount, marker="m") data = [(d.type_num, d.as_raw_string()) for d in self.commits] f = BytesIO() fi = BytesIO() expected = build_pack(f, data, store=sos) entries = [(sha, ofs, checksum) for ofs, _, _, sha, checksum in expected] self.pack_data = PackData.from_file(file=f, size=None) write_pack_index_v2(fi, entries, self.pack_data.calculate_checksum()) fi.seek(0) self.pack_index = load_pack_index_file('', fi)
def _load_packs(self): ret = [] for name in self._pack_names(): if name.startswith("pack-") and name.endswith(".pack"): try: size = self.pack_transport.stat(name).st_size except TransportNotPossible: f = self.pack_transport.get(name) pd = PackData(name, f) else: pd = PackData(name, self.pack_transport.get(name), size=size) idxname = name.replace(".pack", ".idx") idx = load_pack_index_file(idxname, self.pack_transport.get(idxname)) pack = Pack.from_objects(pd, idx) pack._basename = idxname[:-4] ret.append(pack) return ret
def _pack_idx() -> PackIndex: pack_idx_bytes = self._http_get(f"objects/pack/{pack_idx_name}") return load_pack_index_file(pack_idx_name, pack_idx_bytes)