def export_package(manifest, dest):
abs_dest = os.path.abspath(dest)
print "[INFO] exporting into directory %s" % abs_dest
# Remove and create the base directory where we are going to put all package files.
if os.path.exists(abs_dest):
shutil.rmtree(abs_dest)
os.makedirs(abs_dest)
files = list(expand(manifest))
files = [(x, unsymlink(y % defines)) for (x, y) in files]
for name, hostname in files:
name = name[1:] if name.startswith("/") else name
name = os.path.join(abs_dest, name)
if hostname.startswith("->"):
link_source = hostname[2:]
target_dir = os.path.dirname(name)
if link_source.startswith("/"):
link_source = os.path.join(abs_dest, link_source[1:])
else:
link_source = os.path.abspath(
os.path.join(target_dir, link_source))
link_source = os.path.relpath(link_source, target_dir)
if not os.path.exists(target_dir):
os.makedirs(target_dir)
os.symlink(link_source, name)
print "[INFO] added link %s -> %s" % (name, link_source)
else:
# If it is a file, copy it to the target directory.
if os.path.isfile(hostname):
# Make sure the target dir exists
dirname = os.path.dirname(name)
if not os.path.exists(dirname):
os.makedirs(dirname)
if hostname.endswith("-stripped.so"):
continue
hostname = strip_file(hostname)
shutil.copy(hostname, name)
print "[INFO] exported %s" % name
elif os.path.isdir(hostname):
# If hostname is a dir, it is only a request to create the folder on guest. Nothing to copy.
if not os.path.exists(name):
os.makedirs(name)
print "[INFO] created dir %s" % name
else:
# Inform the user that the rule cannot be applied. For example, this happens for links in OSv.
print "[ERR] unable to export %s" % hostname
def parse_manifest(manifest):
manifest = [(x, y % defines) for (x, y) in manifest]
files = list(expand(manifest))
files = [(x, unsymlink(y)) for (x, y) in files]
file_dict = {}
def populate_with_directory_path(path, directory):
tokens = path.rstrip('/').split('/')
dictionary = directory
for token in tokens[:-1]:
dictionary = dictionary.setdefault(token, {})
return (dictionary, tokens)
for name, hostname in files:
p = file_dict
if hostname.startswith('->'):
p, tokens = populate_with_directory_path(name, p)
entry = tokens[len(tokens) - 1]
p[entry] = hostname
else:
if hostname.endswith('-stripped.so'):
continue
hostname = strip_file(hostname)
if os.path.islink(hostname):
p, tokens = populate_with_directory_path(name, p)
entry = tokens[len(tokens) - 1]
link = os.readlink(hostname)
p[entry] = '->%s' % link
elif os.path.isdir(hostname):
for token in name.rstrip('/').split('/'):
p = p.setdefault(token, {})
else:
p, tokens = populate_with_directory_path(name, p)
entry = tokens[len(tokens) - 1]
p[entry] = hostname
return file_dict
def parse_manifest(manifest):
manifest = [(x, y % defines) for (x, y) in manifest]
files = list(expand(manifest))
files = [(x, unsymlink(y)) for (x, y) in files]
file_dict = {}
def populate_with_directory_path(path,directory):
tokens = path.rstrip('/').split('/')
dictionary = directory
for token in tokens[:-1]:
dictionary = dictionary.setdefault(token, {})
return (dictionary,tokens)
for name, hostname in files:
p = file_dict
if hostname.startswith('->'):
p, tokens = populate_with_directory_path(name,p)
entry = tokens[len(tokens)-1]
p[entry] = hostname
else:
if hostname.endswith('-stripped.so'):
continue
hostname = strip_file(hostname)
if os.path.islink(hostname):
p, tokens = populate_with_directory_path(name,p)
entry = tokens[len(tokens)-1]
link = os.readlink(hostname)
p[entry] = '->%s' % link
elif os.path.isdir(hostname):
for token in name.rstrip('/').split('/'):
p = p.setdefault(token, {})
else:
p, tokens = populate_with_directory_path(name,p)
entry = tokens[len(tokens)-1]
p[entry] = hostname
return file_dict
def upload(osv, manifest, depends, port):
manifest = [(x, y % defines) for (x, y) in manifest]
files = list(expand(manifest))
files = [(x, unsymlink(y)) for (x, y) in files]
# Wait for the guest to come up and tell us it's listening
while True:
line = osv.stdout.readline()
if not line or line.find(b"Waiting for connection") >= 0:
break
os.write(sys.stdout.fileno(), line)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(("127.0.0.1", port))
# We'll want to read the rest of the guest's output, so that it doesn't
# hang, and so the user can see what's happening. Easiest to do this with
# a thread.
stop_output = False
def consumeoutput(file, silent):
for line in iter(lambda: file.readline(), b''):
if not silent():
os.write(sys.stdout.fileno(), line)
threading.Thread(target=consumeoutput,
args=(osv.stdout, lambda: stop_output)).start()
# Send a CPIO header or file, padded to multiple of 4 bytes
def cpio_send(data):
s.sendall(data)
partial = len(data) % 4
if partial > 0:
s.sendall(b'\0' * (4 - partial))
def cpio_field(number, length):
return ("%.*x" % (length, number)).encode()
def cpio_header(filename, mode, filesize):
if sys.version_info >= (3, 0, 0):
filename = filename.encode("utf-8")
return (b"070701" # magic
+ cpio_field(0, 8) # inode
+ cpio_field(mode, 8) # mode
+ cpio_field(0, 8) # uid
+ cpio_field(0, 8) # gid
+ cpio_field(0, 8) # nlink
+ cpio_field(0, 8) # mtime
+ cpio_field(filesize, 8) # filesize
+ cpio_field(0, 8) # devmajor
+ cpio_field(0, 8) # devminor
+ cpio_field(0, 8) # rdevmajor
+ cpio_field(0, 8) # rdevminor
+ cpio_field(len(filename) + 1, 8) # namesize
+ cpio_field(0, 8) # check
+ filename + b'\0')
# Send the files to the guest
try:
for name, hostname in files:
if hostname.startswith("->"):
link = hostname[2:]
cpio_send(cpio_header(name, stat.S_IFLNK, len(link)))
cpio_send(link.encode())
else:
depends.write('\t%s \\\n' % (hostname, ))
if hostname.endswith("-stripped.so"):
continue
hostname = strip_file(hostname)
if os.path.islink(hostname):
perm = os.lstat(hostname).st_mode & 0o777
link = os.readlink(hostname)
cpio_send(cpio_header(name, perm | stat.S_IFLNK,
len(link)))
cpio_send(link.encode())
elif os.path.isdir(hostname):
perm = os.stat(hostname).st_mode & 0o777
cpio_send(cpio_header(name, perm | stat.S_IFDIR, 0))
else:
perm = os.stat(hostname).st_mode & 0o777
cpio_send(
cpio_header(name, perm | stat.S_IFREG,
os.stat(hostname).st_size))
with open(hostname, 'rb') as f:
cpio_send(f.read())
except:
# We had an error uploading one of the files (e.g., one of the files
# in the manifest does not exist). Let's print an error message, but
# first stop the consumeoutput thread displaying the guest's output
# so its messages do not get mixed with ours. Then, do NOT exit -
# continue the code below to stop the guest normally.
import time
time.sleep(1) # give consumeoutput thread a chance to print some more
stop_output = True
print("\nERROR: file upload failed:")
print(sys.exc_info()[1])
cpio_send(cpio_header("TRAILER!!!", 0, 0))
s.shutdown(socket.SHUT_WR)
# Wait for the guest to actually finish writing and syncing
s.recv(1)
s.close()
if stop_output:
# stop_output is set when we failed during the upload, so let's
# have the upload fail.
sys.exit(1)
def main():
make_option = optparse.make_option
opt = optparse.OptionParser(option_list=[
make_option('-o', dest='output', help='write to FILE', metavar='FILE'),
make_option('-d',
dest='depends',
help='write dependencies to FILE',
metavar='FILE',
default=None),
make_option('-m',
dest='manifest',
help='read manifest from FILE',
metavar='FILE'),
make_option('-D',
type='string',
help='define VAR=DATA',
metavar='VAR=DATA',
action='callback',
callback=add_var),
])
(options, args) = opt.parse_args()
# See unpack_bootfs() as the reference to this ad-hoc packing format.
metadata_size = 128
depends = io.StringIO()
if options.depends:
depends = open(options.depends, 'w')
out = open(options.output, 'wb')
depends.write(u'%s: \\\n' % (options.output, ))
files = read_manifest(options.manifest)
files = [(x, y % defines) for (x, y) in files]
files = list(expand(files))
files = [(x, unsymlink(y)) for (x, y) in files]
files = [(x, y) for (x, y) in files if not x.endswith("-stripped.so")]
files = [(x, strip_file(y)) for (x, y) in files]
pos = (len(files) + 1) * metadata_size
for name, hostname in files:
type = 0
if hostname.startswith("->"):
link = hostname[2:]
type = 1
size = len(link.encode()) + 1
elif os.path.isdir(hostname):
size = 0
if not name.endswith("/"):
name += "/"
else:
size = os.stat(hostname).st_size
# FIXME: check if name.encode()'s length is more than 110 (111
# minus the necessary null byte) and fail if it is.
metadata = struct.pack('QQb111s', size, pos, type, name.encode())
out.write(metadata)
pos += size
depends.write(u'\t%s \\\n' % (hostname, ))
out.write(struct.pack('128s', b''))
for name, hostname in files:
if os.path.isdir(hostname):
continue
if hostname.startswith("->"):
link = hostname[2:]
out.write(link.encode())
out.write('\0')
else:
out.write(open(hostname, 'rb').read())
depends.write(u'\n\n')
out.close()
depends.close()
def upload(osv, manifest, depends):
manifest = [(x, y % defines) for (x, y) in manifest]
files = list(expand(manifest))
files = [(x, unsymlink(y)) for (x, y) in files]
# Wait for the guest to come up and tell us it's listening
while True:
line = osv.stdout.readline()
if not line or line.find(b"Waiting for connection") >= 0:
break
os.write(sys.stdout.fileno(), line)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(("127.0.0.1", 10000))
# We'll want to read the rest of the guest's output, so that it doesn't
# hang, and so the user can see what's happening. Easiest to do this with
# a thread.
stop_output = False
def consumeoutput(file, silent):
for line in iter(lambda: file.readline(), b''):
if not silent():
os.write(sys.stdout.fileno(), line)
threading.Thread(target=consumeoutput, args=(osv.stdout, lambda: stop_output)).start()
# Send a CPIO header or file, padded to multiple of 4 bytes
def cpio_send(data):
s.sendall(data)
partial = len(data)%4
if partial > 0:
s.sendall(b'\0'*(4-partial))
def cpio_field(number, length):
return ("%.*x" % (length, number)).encode()
def cpio_header(filename, mode, filesize):
if sys.version_info >= (3, 0, 0):
filename = filename.encode("utf-8")
return (b"070701" # magic
+ cpio_field(0, 8) # inode
+ cpio_field(mode, 8) # mode
+ cpio_field(0, 8) # uid
+ cpio_field(0, 8) # gid
+ cpio_field(0, 8) # nlink
+ cpio_field(0, 8) # mtime
+ cpio_field(filesize, 8) # filesize
+ cpio_field(0, 8) # devmajor
+ cpio_field(0, 8) # devminor
+ cpio_field(0, 8) # rdevmajor
+ cpio_field(0, 8) # rdevminor
+ cpio_field(len(filename)+1, 8) # namesize
+ cpio_field(0, 8) # check
+ filename + b'\0')
# Send the files to the guest
try:
for name, hostname in files:
if hostname.startswith("->"):
link = hostname[2:]
cpio_send(cpio_header(name, stat.S_IFLNK, len(link)))
cpio_send(link.encode())
else:
depends.write('\t%s \\\n' % (hostname,))
if hostname.endswith("-stripped.so"):
continue
hostname = strip_file(hostname)
if os.path.islink(hostname):
perm = os.lstat(hostname).st_mode & 0o777
link = os.readlink(hostname)
cpio_send(cpio_header(name, perm | stat.S_IFLNK, len(link)))
cpio_send(link.encode())
elif os.path.isdir(hostname):
perm = os.stat(hostname).st_mode & 0o777
cpio_send(cpio_header(name, perm | stat.S_IFDIR, 0))
else:
perm = os.stat(hostname).st_mode & 0o777
cpio_send(cpio_header(name, perm | stat.S_IFREG, os.stat(hostname).st_size))
with open(hostname, 'rb') as f:
cpio_send(f.read())
except:
# We had an error uploading one of the files (e.g., one of the files
# in the manifest does not exist). Let's print an error message, but
# first stop the consumeoutput thread displaying the guest's output
# so its messages do not get mixed with ours. Then, do NOT exit -
# continue the code below to stop the guest normally.
import time
time.sleep(1) # give consumeoutput thread a chance to print some more
stop_output = True
print("\nERROR: file upload failed:")
print(sys.exc_info()[1])
cpio_send(cpio_header("TRAILER!!!", 0, 0))
s.shutdown(socket.SHUT_WR)
# Wait for the guest to actually finish writing and syncing
s.recv(1)
s.close()
if stop_output:
# stop_output is set when we failed during the upload, so let's
# have the upload fail.
sys.exit(1)
def upload(osv, manifest, depends):
manifest = [(x, y % defines) for (x, y) in manifest]
files = list(expand(manifest))
files = [(x, unsymlink(y)) for (x, y) in files]
# Wait for the guest to come up and tell us it's listening
while True:
line = osv.stdout.readline()
if not line or line.find(b"Waiting for connection") >= 0:
break
os.write(sys.stdout.fileno(), line)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(("127.0.0.1", 10000))
# We'll want to read the rest of the guest's output, so that it doesn't
# hang, and so the user can see what's happening. Easiest to do this with
# a thread.
def consumeoutput(file):
for line in iter(lambda: file.readline(), b''):
os.write(sys.stdout.fileno(), line)
threading.Thread(target=consumeoutput, args=(osv.stdout,)).start()
# Send a CPIO header or file, padded to multiple of 4 bytes
def cpio_send(data):
s.sendall(data)
partial = len(data)%4
if partial > 0:
s.sendall(b'\0'*(4-partial))
def cpio_field(number, length):
return ("%.*x" % (length, number)).encode()
def cpio_header(filename, mode, filesize):
if sys.version_info >= (3, 0, 0):
filename = filename.encode("utf-8")
return (b"070701" # magic
+ cpio_field(0, 8) # inode
+ cpio_field(mode, 8) # mode
+ cpio_field(0, 8) # uid
+ cpio_field(0, 8) # gid
+ cpio_field(0, 8) # nlink
+ cpio_field(0, 8) # mtime
+ cpio_field(filesize, 8) # filesize
+ cpio_field(0, 8) # devmajor
+ cpio_field(0, 8) # devminor
+ cpio_field(0, 8) # rdevmajor
+ cpio_field(0, 8) # rdevminor
+ cpio_field(len(filename)+1, 8) # namesize
+ cpio_field(0, 8) # check
+ filename + b'\0')
# Send the files to the guest
for name, hostname in files:
if hostname.startswith("->"):
link = hostname[2:]
cpio_send(cpio_header(name, stat.S_IFLNK, len(link)))
cpio_send(link.encode())
else:
depends.write('\t%s \\\n' % (hostname,))
if hostname.endswith("-stripped.so"):
continue
hostname = strip_file(hostname)
if os.path.islink(hostname):
perm = os.lstat(hostname).st_mode & 0o777
link = os.readlink(hostname)
cpio_send(cpio_header(name, perm | stat.S_IFLNK, len(link)))
cpio_send(link.encode())
elif os.path.isdir(hostname):
perm = os.stat(hostname).st_mode & 0o777
cpio_send(cpio_header(name, perm | stat.S_IFDIR, 0))
else:
perm = os.stat(hostname).st_mode & 0o777
cpio_send(cpio_header(name, perm | stat.S_IFREG, os.stat(hostname).st_size))
with open(hostname, 'rb') as f:
cpio_send(f.read())
cpio_send(cpio_header("TRAILER!!!", 0, 0))
s.shutdown(socket.SHUT_WR)
# Wait for the guest to actually finish writing and syncing
s.recv(1)
s.close()
def main():
make_option = optparse.make_option
opt = optparse.OptionParser(option_list=[
make_option('-o',
dest='output',
help='write to FILE',
metavar='FILE'),
make_option('-d',
dest='depends',
help='write dependencies to FILE',
metavar='FILE',
default=None),
make_option('-m',
dest='manifest',
help='read manifest from FILE',
metavar='FILE'),
make_option('-D',
type='string',
help='define VAR=DATA',
metavar='VAR=DATA',
action='callback',
callback=add_var),
])
(options, args) = opt.parse_args()
# See unpack_bootfs() as the reference to this ad-hoc packing format.
metadata_size = 128
depends = io.StringIO()
if options.depends:
depends = open(options.depends, 'w')
out = open(options.output, 'wb')
depends.write(u'%s: \\\n' % (options.output,))
files = read_manifest(options.manifest)
files = [(x, y % defines) for (x, y) in files]
files = list(expand(files))
files = [(x, unsymlink(y)) for (x, y) in files]
files = [(x, y) for (x, y) in files if not x.endswith("-stripped.so")]
files = [(x, strip_file(y)) for (x, y) in files]
pos = (len(files) + 1) * metadata_size
for name, hostname in files:
type = 0
if hostname.startswith("->"):
link = hostname[2:]
type = 1
size = len(link.encode())+1
elif os.path.isdir(hostname):
size = 0;
if not name.endswith("/"):
name += "/"
else:
size = os.stat(hostname).st_size
# FIXME: check if name.encode()'s length is more than 110 (111
# minus the necessary null byte) and fail if it is.
metadata = struct.pack('QQb111s', size, pos, type, name.encode())
out.write(metadata)
pos += size
depends.write(u'\t%s \\\n' % (hostname,))
out.write(struct.pack('128s', b''))
for name, hostname in files:
if os.path.isdir(hostname):
continue
if hostname.startswith("->"):
link = hostname[2:]
out.write(link.encode())
out.write('\0')
else:
out.write(open(hostname, 'rb').read())
depends.write(u'\n\n')
out.close()
depends.close()
def upload(osv, manifest, depends):
manifest = [(x, y % defines) for (x, y) in manifest]
files = list(expand(manifest))
files = [(x, unsymlink(y)) for (x, y) in files]
# Wait for the guest to come up and tell us it's listening
while True:
line = osv.stdout.readline()
if not line or line.find(b"Waiting for connection") >= 0:
break
os.write(sys.stdout.fileno(), line)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(("127.0.0.1", 10000))
# We'll want to read the rest of the guest's output, so that it doesn't
# hang, and so the user can see what's happening. Easiest to do this with
# a thread.
def consumeoutput(file):
for line in iter(lambda: file.readline(), b''):
os.write(sys.stdout.fileno(), line)
threading.Thread(target=consumeoutput, args=(osv.stdout,)).start()
# Send a CPIO header or file, padded to multiple of 4 bytes
def cpio_send(data):
s.sendall(data)
partial = len(data)%4
if partial > 0:
s.sendall(b'\0'*(4-partial))
def cpio_field(number, length):
return ("%.*x" % (length, number)).encode()
def cpio_header(filename, mode, filesize):
if sys.version_info >= (3, 0, 0):
filename = filename.encode("utf-8")
return (b"070701" # magic
+ cpio_field(0, 8) # inode
+ cpio_field(mode, 8) # mode
+ cpio_field(0, 8) # uid
+ cpio_field(0, 8) # gid
+ cpio_field(0, 8) # nlink
+ cpio_field(0, 8) # mtime
+ cpio_field(filesize, 8) # filesize
+ cpio_field(0, 8) # devmajor
+ cpio_field(0, 8) # devminor
+ cpio_field(0, 8) # rdevmajor
+ cpio_field(0, 8) # rdevminor
+ cpio_field(len(filename)+1, 8) # namesize
+ cpio_field(0, 8) # check
+ filename + b'\0')
# Send the files to the guest
for name, hostname in files:
if hostname.startswith("->"):
link = hostname[2:]
cpio_send(cpio_header(name, stat.S_IFLNK, len(link)))
cpio_send(link.encode())
else:
depends.write('\t%s \\\n' % (hostname,))
if hostname.endswith("-stripped.so"):
continue
hostname = strip_file(hostname)
if os.path.islink(hostname):
perm = os.lstat(hostname).st_mode & 0o777
link = os.readlink(hostname)
cpio_send(cpio_header(name, perm | stat.S_IFLNK, len(link)))
cpio_send(link.encode())
elif os.path.isdir(hostname):
perm = os.stat(hostname).st_mode & 0o777
cpio_send(cpio_header(name, perm | stat.S_IFDIR, 0))
else:
perm = os.stat(hostname).st_mode & 0o777
cpio_send(cpio_header(name, perm | stat.S_IFREG, os.stat(hostname).st_size))
with open(hostname, 'rb') as f:
cpio_send(f.read())
cpio_send(cpio_header("TRAILER!!!", 0, 0))
s.shutdown(socket.SHUT_WR)
# Wait for the guest to actually finish writing and syncing
s.recv(1)
s.close()
