def check(self):
'''
Check repository for unreferenced and missing files
'''
# Check if the repo is local
if not self.local:
raise ISError(u"Repository must be local")
local_files = set(listdir(self.config.path))
local_files.remove(self.config.dbname)
local_files.remove(self.config.lastname)
db_files = set(self.getallmd5())
# check missing files
arrow("Checking missing files")
missing_files = db_files - local_files
if len(missing_files) > 0:
out(linesep.join(missing_files))
# check unreferenced files
arrow("Checking unreferenced files")
unref_files = local_files - db_files
if len(unref_files) > 0:
out(linesep.join(unref_files))
# check corruption of local files
arrow("Checking corrupted files")
for f in local_files:
fo = PipeFile(join(self.config.path, f))
fo.consume()
fo.close()
if fo.md5 != f:
out(f)
def get(self, name, version=None):
'''
Return an image from a name and version
'''
# is no version take the last
if version is None:
version = self.last(name)
if version is None:
raise ISError(u"Unable to find image %s in %s" % (name,
self.config.name))
# get file md5 from db
r = self.db.ask("select md5 from image where name = ? and version = ? limit 1",
(name, version)).fetchone()
if r is None:
raise ISError(u"Unable to find image %s v%s in %s" % (name, version,
self.config.name))
path = join(self.config.path, r[0])
# getting the file
arrow(u"Loading image %s v%s from repository %s" % (name,
version,
self.config.name))
memfile = StringIO()
try:
fo = PipeFile(path, "r")
fo.consume(memfile)
fo.close()
except Exception as e:
raise ISError(u"Loading image %s v%s failed" % (name, version), e)
memfile.seek(0)
pkg = PackageImage(path, fileobj=memfile, md5name=True)
if pkg.md5 != r[0]:
raise ISError(u"Image MD5 verification failure")
return pkg
def create_payload_tarball(self, tar_path, data_path, compressor):
'''
Create a payload tarball
'''
try:
# get compressor argv (first to escape file creation if not found)
a_comp = get_compressor_path(compressor, compress=True)
a_tar = ["tar", "--create", "--numeric-owner", "--directory",
data_path, "."]
# create destination file
f_dst = PipeFile(tar_path, "w", progressbar=True)
# run tar process
p_tar = Popen(a_tar, shell=False, close_fds=True,
stdout=PIPE)
# run compressor process
p_comp = Popen(a_comp, shell=False, close_fds=True,
stdin=p_tar.stdout, stdout=PIPE)
# write data from compressor to tar_path
f_dst.consume(p_comp.stdout)
# close all fd
p_tar.stdout.close()
p_comp.stdout.close()
f_dst.close()
# check tar return 0
if p_tar.wait() != 0:
raise ISError("Tar return is not zero")
# check compressor return 0
if p_comp.wait() != 0:
raise ISError(u"Compressor %s return is not zero" % a_comp[0])
except (SystemExit, KeyboardInterrupt):
if exists(tar_path):
unlink(tar_path)
raise
def create_payload_file(self, dest, source, compressor):
'''
Create a payload file
'''
try:
# get compressor argv (first to escape file creation if not found)
a_comp = get_compressor_path(compressor, compress=True)
# open source file
f_src = open(source, "r")
# create destination file
f_dst = PipeFile(dest, "w", progressbar=True)
# run compressor
p_comp = Popen(a_comp, shell=False, close_fds=True,
stdin=f_src, stdout=PIPE)
# close source file fd
f_src.close()
# write data from compressor to dest file
f_dst.consume(p_comp.stdout)
# close compressor stdin and destination file
p_comp.stdout.close()
f_dst.close()
# check compressor return 0
if p_comp.wait() != 0:
raise ISError(u"Compressor %s return is not zero" % a_comp[0])
except (SystemExit, KeyboardInterrupt):
if exists(dest):
unlink(dest)
raise
def download(self, directory, force=False, image=True, payload=False):
'''
Download image in directory
Doesn't use in memory image because we cannot access it
This is done to don't parasitize self._tarfile access to memfile
'''
# check if destination exists
directory = abspath(directory)
if image:
dest = join(directory, self.filename)
if not force and exists(dest):
raise ISError(u"Image destination already exists: %s" % dest)
# some display
arrow(u"Downloading image in %s" % directory)
debug(u"Downloading %s from %s" % (self.filename, self.path))
# open source
fs = PipeFile(self.path, progressbar=True)
# check if announced file size is good
if fs.size is not None and self.size != fs.size:
raise ISError(u"Downloading image %s failed: Invalid announced size" % self.name)
# open destination
fd = open(self.filename, "wb")
fs.consume(fd)
fs.close()
fd.close()
if self.size != fs.consumed_size:
raise ISError(u"Download image %s failed: Invalid size" % self.name)
if self.md5 != fs.md5:
raise ISError(u"Download image %s failed: Invalid MD5" % self.name)
if payload:
for payname in self.payload:
arrow(u"Downloading payload %s in %s" % (payname, directory))
self.payload[payname].info
self.payload[payname].download(directory, force=force)
def download(self, dest, force=False):
'''
Download payload in directory
'''
# if dest is a directory try to create file inside
if isdir(dest):
dest = join(dest, self.filename)
# try to create leading directories
elif not exists(dirname(dest)):
mkdir(dirname(dest))
# check validity of dest
if exists(dest):
if isdir(dest):
raise ISError(u"Destination %s is a directory" % dest)
if not force:
raise ISError(u"File %s already exists" % dest)
# open remote file
debug(u"Downloading payload %s from %s" % (self.filename, self.path))
fs = PipeFile(self.path, progressbar=True)
# check if announced file size is good
if fs.size is not None and self.size != fs.size:
raise ISError(u"Downloading payload %s failed: Invalid announced size" %
self.name)
fd = open(dest, "wb")
fs.consume(fd)
# closing fo
fs.close()
fd.close()
# checking download size
if self.size != fs.read_size:
raise ISError(u"Downloading payload %s failed: Invalid size" % self.name)
if self.md5 != fs.md5:
raise ISError(u"Downloading payload %s failed: Invalid MD5" % self.name)
def checksummize(self):
'''
Fill missing md5/size about payload
'''
fileobj = PipeFile(self.path, "r")
fileobj.consume()
fileobj.close()
if self._size is None:
self._size = fileobj.read_size
if self._md5 is None:
self._md5 = fileobj.md5
def extract_file(self, dest, force=False):
'''
Copy a payload directly to a file
Check md5 on the fly
'''
# if dest is a directory try to create file inside
if isdir(dest):
dest = join(dest, self.name)
# try to create leading directories
elif not exists(dirname(dest)):
mkdir(dirname(dest))
# check validity of dest
if exists(dest):
if isdir(dest):
raise ISError(u"Destination %s is a directory" % dest)
if not force:
raise ISError(u"File %s already exists" % dest)
# get compressor argv (first to escape file creation if not found)
a_comp = get_compressor_path(self.compressor, compress=False)
# try to open payload file (source)
try:
f_src = PipeFile(self.path, "r", progressbar=True)
except Exception as e:
raise ISError(u"Unable to open payload file %s" % self.path, e)
# check if announced file size is good
if f_src.size is not None and self.size != f_src.size:
raise ISError(u"Invalid announced size on %s" % self.path)
# opening destination
try:
f_dst = open(dest, "wb")
except Exception as e:
raise ISError(u"Unable to open destination file %s" % dest, e)
# run compressor process
p_comp = Popen(a_comp, shell=False, close_fds=True,
stdin=PIPE, stdout=f_dst)
# close destination file
f_dst.close()
# push data into compressor
f_src.consume(p_comp.stdin)
# closing source fo
f_src.close()
# checking download size
if self.size != f_src.read_size:
raise ISError("Invalid size")
# checking downloaded md5
if self.md5 != f_src.md5:
raise ISError("Invalid MD5")
# close compressor pipe
p_comp.stdin.close()
# check compressor return 0
if p_comp.wait() != 0:
raise ISError(u"Compressor %s return is not zero" % a_comp[0])
# settings file orginal rights
chrights(dest, self.uid, self.gid, self.mode, self.mtime)
def extract_tar(self, dest, force=False, filelist=None):
'''
Extract a payload which is a tarball.
This is used mainly to extract payload from a directory
'''
# check validity of dest
if exists(dest):
if not isdir(dest):
raise ISError(u"Destination %s is not a directory" % dest)
if not force and len(listdir(dest)) > 0:
raise ISError(u"Directory %s is not empty (need force)" % dest)
else:
mkdir(dest)
# try to open payload file
try:
fo = PipeFile(self.path, progressbar=True)
except Exception as e:
raise ISError(u"Unable to open %s" % self.path)
# check if announced file size is good
if fo.size is not None and self.size != fo.size:
raise ISError(u"Invalid announced size on %s" % self.path)
# get compressor argv (first to escape file creation if not found)
a_comp = get_compressor_path(self.compressor, compress=False)
a_tar = ["tar", "--extract", "--numeric-owner", "--ignore-zeros",
"--preserve-permissions", "--directory", dest]
# add optionnal selected filename for decompression
if filelist is not None:
a_tar += filelist
p_tar = Popen(a_tar, shell=False, close_fds=True,
stdin=PIPE)
p_comp = Popen(a_comp, shell=False, close_fds=True,
stdin=PIPE, stdout=p_tar.stdin)
# close tar fd
p_tar.stdin.close()
# push data into compressor
fo.consume(p_comp.stdin)
# close source fd
fo.close()
# checking downloaded size
if self.size != fo.read_size:
raise ISError("Invalid size")
# checking downloaded md5
if self.md5 != fo.md5:
raise ISError("Invalid MD5")
# close compressor pipe
p_comp.stdin.close()
# check compressor return 0
if p_comp.wait() != 0:
raise ISError(u"Compressor %s return is not zero" % a_comp[0])
# check tar return 0
if p_tar.wait() != 0:
raise ISError("Tar return is not zero")
def check(self):
'''
Check that path correspond to current md5 and size
'''
if self._size is None or self._md5 is None:
debug("Check is called on payload with nothing to check")
return True
fileobj = PipeFile(self.path, "r")
fileobj.consume()
fileobj.close()
if self._size != fileobj.read_size:
raise ISError(u"Invalid size of payload %s" % self.name)
if self._md5 != fileobj.md5:
raise ISError(u"Invalid MD5 of payload %s" % self._md5)
def generate_json_description(self):
'''
Generate a JSON description file
'''
arrow("Generating JSON description")
arrowlevel(1)
# copy description
desc = self.description.copy()
# only store compressor patterns
desc["compressor"] = desc["compressor"]["patterns"]
# timestamp image
arrow("Timestamping")
desc["date"] = int(time())
# watermark
desc["is_build_version"] = VERSION
# append payload infos
arrow("Checksumming payloads")
desc["payload"] = {}
for payload_name in self.select_payloads():
arrow(payload_name, 1)
# getting payload info
payload_desc = self.describe_payload(payload_name)
# compute md5 and size
fileobj = PipeFile(payload_desc["link_path"], "r")
fileobj.consume()
fileobj.close()
# create payload entry
desc["payload"][payload_name] = {
"md5": fileobj.md5,
"size": fileobj.size,
"isdir": payload_desc["isdir"],
"uid": payload_desc["uid"],
"gid": payload_desc["gid"],
"mode": payload_desc["mode"],
"mtime": payload_desc["mtime"],
"compressor": payload_desc["compressor"]
}
arrowlevel(-1)
# check md5 are uniq
md5s = [v["md5"] for v in desc["payload"].values()]
if len(md5s) != len(set(md5s)):
raise ISError("Two payloads cannot have the same md5")
# serialize
return dumps(desc)
def check(self, message="Check MD5"):
'''
Check md5 and size of tarballs are correct
Download tarball from path and compare the loaded md5 and remote
'''
arrow(message)
arrowlevel(1)
# check image
fo = PipeFile(self.path, "r")
fo.consume()
fo.close()
if self.size != fo.read_size:
raise ISError(u"Invalid size of image %s" % self.name)
if self.md5 != fo.md5:
raise ISError(u"Invalid MD5 of image %s" % self.name)
# check payloads
for pay_name, pay_obj in self.payload.items():
arrow(pay_name)
pay_obj.check()
arrowlevel(-1)
def add(self, image, delete=False):
'''
Add a packaged image to repository
if delete is true, remove original files
'''
# check local repository
if not self.local:
raise ISError(u"Repository addition must be local")
# cannot add already existant image
if self.has(image.name, image.version):
raise ISError(u"Image already in database, delete first!")
# adding file to repository
arrow("Copying images and payload")
for obj in [ image ] + image.payload.values():
dest = join(self.config.path, obj.md5)
basesrc = basename(obj.path)
if exists(dest):
arrow(u"Skipping %s: already exists" % basesrc, 1)
else:
arrow(u"Adding %s (%s)" % (basesrc, obj.md5), 1)
dfo = open(dest, "wb")
sfo = PipeFile(obj.path, "r", progressbar=True)
sfo.consume(dfo)
sfo.close()
dfo.close()
chrights(dest, self.config.uid,
self.config.gid, self.config.fmod)
# copy is done. create a image inside repo
r_image = PackageImage(join(self.config.path, image.md5),
md5name=True)
# checking must be done with original md5
r_image.md5 = image.md5
# checking image and payload after copy
r_image.check("Check image and payload")
self._add(image)
# removing orginal files
if delete:
arrow("Removing original files")
for obj in [ image ] + image.payload.values():
arrow(basename(obj.path), 1)
unlink(obj.path)
def __init__(self, path, fileobj=None, md5name=False):
'''
Initialize a package image
fileobj must be a seekable fileobj
'''
Image.__init__(self)
self.path = abspath(path)
self.base_path = dirname(self.path)
# tarball are named by md5 and not by real name
self.md5name = md5name
try:
if fileobj is None:
fileobj = PipeFile(self.path, "r")
else:
fileobj = PipeFile(mode="r", fileobj=fileobj)
memfile = StringIO()
fileobj.consume(memfile)
# close source
fileobj.close()
# get downloaded size and md5
self.size = fileobj.read_size
self.md5 = fileobj.md5
memfile.seek(0)
self._tarball = Tarball.open(fileobj=memfile, mode='r:gz')
except Exception as e:
raise ISError(u"Unable to open image %s" % path, e)
self._metadata = self.read_metadata()
# print info
arrow(u"Image %s v%s loaded" % (self.name, self.version))
arrow(u"Author: %s" % self.author, 1)
arrow(u"Date: %s" % time_rfc2822(self.date), 1)
# build payloads info
self.payload = {}
for pname, pval in self._metadata["payload"].items():
pfilename = u"%s-%s%s" % (self.filename[:-len(Image.extension)],
pname, Payload.extension)
if self.md5name:
ppath = join(self.base_path,
self._metadata["payload"][pname]["md5"])
else:
ppath = join(self.base_path, pfilename)
self.payload[pname] = Payload(pname, pfilename, ppath, **pval)
def _cachify(self, config, temp=False, nosync=False):
'''
Return a config of a cached repository from an orignal config file
:param config: repository configuration
:param temp: repository db should be stored in a temporary location
:param nosync: if a cache exists, don't try to update it
'''
# if cache is disable => temp =True
if self.cache_path is None:
temp = True
try:
original_dbpath = config.dbpath
if temp and nosync:
raise ISError("sync is disabled")
elif temp:
# this is a temporary cached repository
tempfd, config.dbpath = tempfile.mkstemp()
os.close(tempfd)
self.tempfiles.append(config.dbpath)
else:
config.dbpath = os.path.join(self.cache_path, config.name)
if not nosync:
# Open remote database
rdb = PipeFile(original_dbpath, timeout=self.timeout)
# get remote last modification
if rdb.mtime is None:
# We doesn't have modification time, we use the last file
try:
rlast = int(PipeFile(config.lastpath, mode='r',
timeout=self.timeout).read().strip())
except ISError:
rlast = -1
else:
rlast = rdb.mtime
# get local last value
if os.path.exists(config.dbpath):
llast = int(os.stat(config.dbpath).st_mtime)
else:
llast = -2
# if repo is out of date, download it
if rlast != llast:
try:
arrow(u"Downloading %s" % original_dbpath)
rdb.progressbar = True
ldb = open(config.dbpath, "wb")
rdb.consume(ldb)
ldb.close()
rdb.close()
istools.chrights(config.dbpath,
uid=config.uid,
gid=config.gid,
mode=config.fmod,
mtime=rlast)
except:
if os.path.exists(config.dbpath):
os.unlink(config.dbpath)
raise
except ISError as e :
# if something append bad during caching, we mark repo as offline
debug(u"Unable to cache repository %s: %s" % (config.name, e))
config.offline = True
return self.factory.create(config)
