def _import_path(self, src_path):
src_path_sha = sha1_for_path(src_path)
dst_path = self._path_for_file_with_sha(src_path, src_path_sha)
dst_path_gz = dst_path+'.gz'
# TODO: this is lame
if os.path.exists(dst_path):
return (dst_path, os.path.getsize(dst_path))
elif os.path.exists(dst_path_gz):
return (dst_path_gz, os.path.getsize(dst_path_gz))
# gzip the file first, and see if it passes the compression
# threshhold
makedirs(os.path.dirname(dst_path))
mygzip(src_path, dst_path_gz)
src_size = os.path.getsize(src_path)
dst_gz_size = os.path.getsize(dst_path_gz)
if float(dst_gz_size) / src_size <= self.config.gzip_threshhold:
final_dst_path = dst_path_gz
final_dst_size = dst_gz_size
else:
final_dst_path = dst_path
final_dst_size = src_size
copyfile(src_path, dst_path)
os.remove(dst_path_gz)
logging.info("Imported %s --> %s" % (src_path, final_dst_path))
return (final_dst_path, final_dst_size)
def write(self, path, gzip=False):
manifest_file = open(path, 'w')
dict = self.to_json()
manifest_file.write(json.dumps(dict))
manifest_file.close()
if gzip:
gzpath = path + '.gz'
mygzip(path, gzpath)
def write(self, path, gzip=False):
if self.id is None:
raise ValueError("catalog id is None") # TODO: better exception?
index_file = open(path, 'w')
dict = self.to_json()
index_file.write(json.dumps(dict))
index_file.close()
if gzip:
gzpath = path + '.gz'
mygzip(path, gzpath)