def _import_path(self, src_path): src_path_sha = sha1_for_path(src_path) dst_path = self._path_for_file_with_sha(src_path, src_path_sha) dst_path_gz = dst_path+'.gz' # TODO: this is lame if os.path.exists(dst_path): return (dst_path, os.path.getsize(dst_path)) elif os.path.exists(dst_path_gz): return (dst_path_gz, os.path.getsize(dst_path_gz)) # gzip the file first, and see if it passes the compression # threshhold makedirs(os.path.dirname(dst_path)) mygzip(src_path, dst_path_gz) src_size = os.path.getsize(src_path) dst_gz_size = os.path.getsize(dst_path_gz) if float(dst_gz_size) / src_size <= self.config.gzip_threshhold: final_dst_path = dst_path_gz final_dst_size = dst_gz_size else: final_dst_path = dst_path final_dst_size = src_size copyfile(src_path, dst_path) os.remove(dst_path_gz) logging.info("Imported %s --> %s" % (src_path, final_dst_path)) return (final_dst_path, final_dst_size)
def run(self): version = self._next_version_for_bundle(self.bundle_name) # Create a new manifest outside of the catalog new_manifest = ZincManifest(self.bundle_name, version) # Process all the paths and add them to the manifest for root, dirs, files in os.walk(self.src_dir): for f in files: if f in IGNORE: continue # TODO: real ignore full_path = pjoin(root, f) rel_dir = root[len(self.src_dir)+1:] rel_path = pjoin(rel_dir, f) sha = sha1_for_path(full_path) new_manifest.add_file(rel_path, sha) existing_manifest = self.catalog.manifest_for_bundle(self.bundle_name) if existing_manifest is None or not new_manifest.files_are_equivalent(existing_manifest): tar_file_name = self.catalog._path_for_archive_for_bundle_version(self.bundle_name, version) tar = tarfile.open(tar_file_name, 'w') print tar_file_name for file in new_manifest.files.keys(): full_path = pjoin(self.src_dir, file) (catalog_path, size) = self.catalog._import_path(full_path) if catalog_path[-3:] == '.gz': format = 'gz' else: format = 'raw' new_manifest.add_format_for_file(file, format, size) tar.add(catalog_path, os.path.basename(catalog_path)) tar.close() self.catalog.index.add_version_for_bundle(self.bundle_name, version) self.catalog._write_manifest(new_manifest) self.catalog.save() return new_manifest return existing_manifest