def _import_path(self, src_path):
src_path_sha = sha1_for_path(src_path)
dst_path = self._path_for_file_with_sha(src_path, src_path_sha)
dst_path_gz = dst_path+'.gz'
# TODO: this is lame
if os.path.exists(dst_path):
return (dst_path, os.path.getsize(dst_path))
elif os.path.exists(dst_path_gz):
return (dst_path_gz, os.path.getsize(dst_path_gz))
# gzip the file first, and see if it passes the compression
# threshhold
makedirs(os.path.dirname(dst_path))
mygzip(src_path, dst_path_gz)
src_size = os.path.getsize(src_path)
dst_gz_size = os.path.getsize(dst_path_gz)
if float(dst_gz_size) / src_size <= self.config.gzip_threshhold:
final_dst_path = dst_path_gz
final_dst_size = dst_gz_size
else:
final_dst_path = dst_path
final_dst_size = src_size
copyfile(src_path, dst_path)
os.remove(dst_path_gz)
logging.info("Imported %s --> %s" % (src_path, final_dst_path))
return (final_dst_path, final_dst_size)
def run(self):
version = self._next_version_for_bundle(self.bundle_name)
# Create a new manifest outside of the catalog
new_manifest = ZincManifest(self.bundle_name, version)
# Process all the paths and add them to the manifest
for root, dirs, files in os.walk(self.src_dir):
for f in files:
if f in IGNORE: continue # TODO: real ignore
full_path = pjoin(root, f)
rel_dir = root[len(self.src_dir)+1:]
rel_path = pjoin(rel_dir, f)
sha = sha1_for_path(full_path)
new_manifest.add_file(rel_path, sha)
existing_manifest = self.catalog.manifest_for_bundle(self.bundle_name)
if existing_manifest is None or not new_manifest.files_are_equivalent(existing_manifest):
tar_file_name = self.catalog._path_for_archive_for_bundle_version(self.bundle_name, version)
tar = tarfile.open(tar_file_name, 'w')
print tar_file_name
for file in new_manifest.files.keys():
full_path = pjoin(self.src_dir, file)
(catalog_path, size) = self.catalog._import_path(full_path)
if catalog_path[-3:] == '.gz':
format = 'gz'
else:
format = 'raw'
new_manifest.add_format_for_file(file, format, size)
tar.add(catalog_path, os.path.basename(catalog_path))
tar.close()
self.catalog.index.add_version_for_bundle(self.bundle_name, version)
self.catalog._write_manifest(new_manifest)
self.catalog.save()
return new_manifest
return existing_manifest
