def export_package(manifest, dest): abs_dest = os.path.abspath(dest) print "[INFO] exporting into directory %s" % abs_dest # Remove and create the base directory where we are going to put all package files. if os.path.exists(abs_dest): shutil.rmtree(abs_dest) os.makedirs(abs_dest) files = list(expand(manifest)) files = [(x, unsymlink(y % defines)) for (x, y) in files] for name, hostname in files: name = name[1:] if name.startswith("/") else name name = os.path.join(abs_dest, name) if hostname.startswith("->"): link_source = hostname[2:] target_dir = os.path.dirname(name) if link_source.startswith("/"): link_source = os.path.join(abs_dest, link_source[1:]) else: link_source = os.path.abspath( os.path.join(target_dir, link_source)) link_source = os.path.relpath(link_source, target_dir) if not os.path.exists(target_dir): os.makedirs(target_dir) os.symlink(link_source, name) print "[INFO] added link %s -> %s" % (name, link_source) else: # If it is a file, copy it to the target directory. if os.path.isfile(hostname): # Make sure the target dir exists dirname = os.path.dirname(name) if not os.path.exists(dirname): os.makedirs(dirname) if hostname.endswith("-stripped.so"): continue hostname = strip_file(hostname) shutil.copy(hostname, name) print "[INFO] exported %s" % name elif os.path.isdir(hostname): # If hostname is a dir, it is only a request to create the folder on guest. Nothing to copy. if not os.path.exists(name): os.makedirs(name) print "[INFO] created dir %s" % name else: # Inform the user that the rule cannot be applied. For example, this happens for links in OSv. print "[ERR] unable to export %s" % hostname
def parse_manifest(manifest): manifest = [(x, y % defines) for (x, y) in manifest] files = list(expand(manifest)) files = [(x, unsymlink(y)) for (x, y) in files] file_dict = {} def populate_with_directory_path(path, directory): tokens = path.rstrip('/').split('/') dictionary = directory for token in tokens[:-1]: dictionary = dictionary.setdefault(token, {}) return (dictionary, tokens) for name, hostname in files: p = file_dict if hostname.startswith('->'): p, tokens = populate_with_directory_path(name, p) entry = tokens[len(tokens) - 1] p[entry] = hostname else: if hostname.endswith('-stripped.so'): continue hostname = strip_file(hostname) if os.path.islink(hostname): p, tokens = populate_with_directory_path(name, p) entry = tokens[len(tokens) - 1] link = os.readlink(hostname) p[entry] = '->%s' % link elif os.path.isdir(hostname): for token in name.rstrip('/').split('/'): p = p.setdefault(token, {}) else: p, tokens = populate_with_directory_path(name, p) entry = tokens[len(tokens) - 1] p[entry] = hostname return file_dict
def parse_manifest(manifest): manifest = [(x, y % defines) for (x, y) in manifest] files = list(expand(manifest)) files = [(x, unsymlink(y)) for (x, y) in files] file_dict = {} def populate_with_directory_path(path,directory): tokens = path.rstrip('/').split('/') dictionary = directory for token in tokens[:-1]: dictionary = dictionary.setdefault(token, {}) return (dictionary,tokens) for name, hostname in files: p = file_dict if hostname.startswith('->'): p, tokens = populate_with_directory_path(name,p) entry = tokens[len(tokens)-1] p[entry] = hostname else: if hostname.endswith('-stripped.so'): continue hostname = strip_file(hostname) if os.path.islink(hostname): p, tokens = populate_with_directory_path(name,p) entry = tokens[len(tokens)-1] link = os.readlink(hostname) p[entry] = '->%s' % link elif os.path.isdir(hostname): for token in name.rstrip('/').split('/'): p = p.setdefault(token, {}) else: p, tokens = populate_with_directory_path(name,p) entry = tokens[len(tokens)-1] p[entry] = hostname return file_dict
def upload(osv, manifest, depends, port): manifest = [(x, y % defines) for (x, y) in manifest] files = list(expand(manifest)) files = [(x, unsymlink(y)) for (x, y) in files] # Wait for the guest to come up and tell us it's listening while True: line = osv.stdout.readline() if not line or line.find(b"Waiting for connection") >= 0: break os.write(sys.stdout.fileno(), line) s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect(("127.0.0.1", port)) # We'll want to read the rest of the guest's output, so that it doesn't # hang, and so the user can see what's happening. Easiest to do this with # a thread. stop_output = False def consumeoutput(file, silent): for line in iter(lambda: file.readline(), b''): if not silent(): os.write(sys.stdout.fileno(), line) threading.Thread(target=consumeoutput, args=(osv.stdout, lambda: stop_output)).start() # Send a CPIO header or file, padded to multiple of 4 bytes def cpio_send(data): s.sendall(data) partial = len(data) % 4 if partial > 0: s.sendall(b'\0' * (4 - partial)) def cpio_field(number, length): return ("%.*x" % (length, number)).encode() def cpio_header(filename, mode, filesize): if sys.version_info >= (3, 0, 0): filename = filename.encode("utf-8") return (b"070701" # magic + cpio_field(0, 8) # inode + cpio_field(mode, 8) # mode + cpio_field(0, 8) # uid + cpio_field(0, 8) # gid + cpio_field(0, 8) # nlink + cpio_field(0, 8) # mtime + cpio_field(filesize, 8) # filesize + cpio_field(0, 8) # devmajor + cpio_field(0, 8) # devminor + cpio_field(0, 8) # rdevmajor + cpio_field(0, 8) # rdevminor + cpio_field(len(filename) + 1, 8) # namesize + cpio_field(0, 8) # check + filename + b'\0') # Send the files to the guest try: for name, hostname in files: if hostname.startswith("->"): link = hostname[2:] cpio_send(cpio_header(name, stat.S_IFLNK, len(link))) cpio_send(link.encode()) else: depends.write('\t%s \\\n' % (hostname, )) if hostname.endswith("-stripped.so"): continue hostname = strip_file(hostname) if os.path.islink(hostname): perm = os.lstat(hostname).st_mode & 0o777 link = os.readlink(hostname) cpio_send(cpio_header(name, perm | stat.S_IFLNK, len(link))) cpio_send(link.encode()) elif os.path.isdir(hostname): perm = os.stat(hostname).st_mode & 0o777 cpio_send(cpio_header(name, perm | stat.S_IFDIR, 0)) else: perm = os.stat(hostname).st_mode & 0o777 cpio_send( cpio_header(name, perm | stat.S_IFREG, os.stat(hostname).st_size)) with open(hostname, 'rb') as f: cpio_send(f.read()) except: # We had an error uploading one of the files (e.g., one of the files # in the manifest does not exist). Let's print an error message, but # first stop the consumeoutput thread displaying the guest's output # so its messages do not get mixed with ours. Then, do NOT exit - # continue the code below to stop the guest normally. import time time.sleep(1) # give consumeoutput thread a chance to print some more stop_output = True print("\nERROR: file upload failed:") print(sys.exc_info()[1]) cpio_send(cpio_header("TRAILER!!!", 0, 0)) s.shutdown(socket.SHUT_WR) # Wait for the guest to actually finish writing and syncing s.recv(1) s.close() if stop_output: # stop_output is set when we failed during the upload, so let's # have the upload fail. sys.exit(1)
def main(): make_option = optparse.make_option opt = optparse.OptionParser(option_list=[ make_option('-o', dest='output', help='write to FILE', metavar='FILE'), make_option('-d', dest='depends', help='write dependencies to FILE', metavar='FILE', default=None), make_option('-m', dest='manifest', help='read manifest from FILE', metavar='FILE'), make_option('-D', type='string', help='define VAR=DATA', metavar='VAR=DATA', action='callback', callback=add_var), ]) (options, args) = opt.parse_args() # See unpack_bootfs() as the reference to this ad-hoc packing format. metadata_size = 128 depends = io.StringIO() if options.depends: depends = open(options.depends, 'w') out = open(options.output, 'wb') depends.write(u'%s: \\\n' % (options.output, )) files = read_manifest(options.manifest) files = [(x, y % defines) for (x, y) in files] files = list(expand(files)) files = [(x, unsymlink(y)) for (x, y) in files] files = [(x, y) for (x, y) in files if not x.endswith("-stripped.so")] files = [(x, strip_file(y)) for (x, y) in files] pos = (len(files) + 1) * metadata_size for name, hostname in files: type = 0 if hostname.startswith("->"): link = hostname[2:] type = 1 size = len(link.encode()) + 1 elif os.path.isdir(hostname): size = 0 if not name.endswith("/"): name += "/" else: size = os.stat(hostname).st_size # FIXME: check if name.encode()'s length is more than 110 (111 # minus the necessary null byte) and fail if it is. metadata = struct.pack('QQb111s', size, pos, type, name.encode()) out.write(metadata) pos += size depends.write(u'\t%s \\\n' % (hostname, )) out.write(struct.pack('128s', b'')) for name, hostname in files: if os.path.isdir(hostname): continue if hostname.startswith("->"): link = hostname[2:] out.write(link.encode()) out.write('\0') else: out.write(open(hostname, 'rb').read()) depends.write(u'\n\n') out.close() depends.close()
def upload(osv, manifest, depends): manifest = [(x, y % defines) for (x, y) in manifest] files = list(expand(manifest)) files = [(x, unsymlink(y)) for (x, y) in files] # Wait for the guest to come up and tell us it's listening while True: line = osv.stdout.readline() if not line or line.find(b"Waiting for connection") >= 0: break os.write(sys.stdout.fileno(), line) s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect(("127.0.0.1", 10000)) # We'll want to read the rest of the guest's output, so that it doesn't # hang, and so the user can see what's happening. Easiest to do this with # a thread. stop_output = False def consumeoutput(file, silent): for line in iter(lambda: file.readline(), b''): if not silent(): os.write(sys.stdout.fileno(), line) threading.Thread(target=consumeoutput, args=(osv.stdout, lambda: stop_output)).start() # Send a CPIO header or file, padded to multiple of 4 bytes def cpio_send(data): s.sendall(data) partial = len(data)%4 if partial > 0: s.sendall(b'\0'*(4-partial)) def cpio_field(number, length): return ("%.*x" % (length, number)).encode() def cpio_header(filename, mode, filesize): if sys.version_info >= (3, 0, 0): filename = filename.encode("utf-8") return (b"070701" # magic + cpio_field(0, 8) # inode + cpio_field(mode, 8) # mode + cpio_field(0, 8) # uid + cpio_field(0, 8) # gid + cpio_field(0, 8) # nlink + cpio_field(0, 8) # mtime + cpio_field(filesize, 8) # filesize + cpio_field(0, 8) # devmajor + cpio_field(0, 8) # devminor + cpio_field(0, 8) # rdevmajor + cpio_field(0, 8) # rdevminor + cpio_field(len(filename)+1, 8) # namesize + cpio_field(0, 8) # check + filename + b'\0') # Send the files to the guest try: for name, hostname in files: if hostname.startswith("->"): link = hostname[2:] cpio_send(cpio_header(name, stat.S_IFLNK, len(link))) cpio_send(link.encode()) else: depends.write('\t%s \\\n' % (hostname,)) if hostname.endswith("-stripped.so"): continue hostname = strip_file(hostname) if os.path.islink(hostname): perm = os.lstat(hostname).st_mode & 0o777 link = os.readlink(hostname) cpio_send(cpio_header(name, perm | stat.S_IFLNK, len(link))) cpio_send(link.encode()) elif os.path.isdir(hostname): perm = os.stat(hostname).st_mode & 0o777 cpio_send(cpio_header(name, perm | stat.S_IFDIR, 0)) else: perm = os.stat(hostname).st_mode & 0o777 cpio_send(cpio_header(name, perm | stat.S_IFREG, os.stat(hostname).st_size)) with open(hostname, 'rb') as f: cpio_send(f.read()) except: # We had an error uploading one of the files (e.g., one of the files # in the manifest does not exist). Let's print an error message, but # first stop the consumeoutput thread displaying the guest's output # so its messages do not get mixed with ours. Then, do NOT exit - # continue the code below to stop the guest normally. import time time.sleep(1) # give consumeoutput thread a chance to print some more stop_output = True print("\nERROR: file upload failed:") print(sys.exc_info()[1]) cpio_send(cpio_header("TRAILER!!!", 0, 0)) s.shutdown(socket.SHUT_WR) # Wait for the guest to actually finish writing and syncing s.recv(1) s.close() if stop_output: # stop_output is set when we failed during the upload, so let's # have the upload fail. sys.exit(1)
def upload(osv, manifest, depends): manifest = [(x, y % defines) for (x, y) in manifest] files = list(expand(manifest)) files = [(x, unsymlink(y)) for (x, y) in files] # Wait for the guest to come up and tell us it's listening while True: line = osv.stdout.readline() if not line or line.find(b"Waiting for connection") >= 0: break os.write(sys.stdout.fileno(), line) s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect(("127.0.0.1", 10000)) # We'll want to read the rest of the guest's output, so that it doesn't # hang, and so the user can see what's happening. Easiest to do this with # a thread. def consumeoutput(file): for line in iter(lambda: file.readline(), b''): os.write(sys.stdout.fileno(), line) threading.Thread(target=consumeoutput, args=(osv.stdout,)).start() # Send a CPIO header or file, padded to multiple of 4 bytes def cpio_send(data): s.sendall(data) partial = len(data)%4 if partial > 0: s.sendall(b'\0'*(4-partial)) def cpio_field(number, length): return ("%.*x" % (length, number)).encode() def cpio_header(filename, mode, filesize): if sys.version_info >= (3, 0, 0): filename = filename.encode("utf-8") return (b"070701" # magic + cpio_field(0, 8) # inode + cpio_field(mode, 8) # mode + cpio_field(0, 8) # uid + cpio_field(0, 8) # gid + cpio_field(0, 8) # nlink + cpio_field(0, 8) # mtime + cpio_field(filesize, 8) # filesize + cpio_field(0, 8) # devmajor + cpio_field(0, 8) # devminor + cpio_field(0, 8) # rdevmajor + cpio_field(0, 8) # rdevminor + cpio_field(len(filename)+1, 8) # namesize + cpio_field(0, 8) # check + filename + b'\0') # Send the files to the guest for name, hostname in files: if hostname.startswith("->"): link = hostname[2:] cpio_send(cpio_header(name, stat.S_IFLNK, len(link))) cpio_send(link.encode()) else: depends.write('\t%s \\\n' % (hostname,)) if hostname.endswith("-stripped.so"): continue hostname = strip_file(hostname) if os.path.islink(hostname): perm = os.lstat(hostname).st_mode & 0o777 link = os.readlink(hostname) cpio_send(cpio_header(name, perm | stat.S_IFLNK, len(link))) cpio_send(link.encode()) elif os.path.isdir(hostname): perm = os.stat(hostname).st_mode & 0o777 cpio_send(cpio_header(name, perm | stat.S_IFDIR, 0)) else: perm = os.stat(hostname).st_mode & 0o777 cpio_send(cpio_header(name, perm | stat.S_IFREG, os.stat(hostname).st_size)) with open(hostname, 'rb') as f: cpio_send(f.read()) cpio_send(cpio_header("TRAILER!!!", 0, 0)) s.shutdown(socket.SHUT_WR) # Wait for the guest to actually finish writing and syncing s.recv(1) s.close()
def main(): make_option = optparse.make_option opt = optparse.OptionParser(option_list=[ make_option('-o', dest='output', help='write to FILE', metavar='FILE'), make_option('-d', dest='depends', help='write dependencies to FILE', metavar='FILE', default=None), make_option('-m', dest='manifest', help='read manifest from FILE', metavar='FILE'), make_option('-D', type='string', help='define VAR=DATA', metavar='VAR=DATA', action='callback', callback=add_var), ]) (options, args) = opt.parse_args() # See unpack_bootfs() as the reference to this ad-hoc packing format. metadata_size = 128 depends = io.StringIO() if options.depends: depends = open(options.depends, 'w') out = open(options.output, 'wb') depends.write(u'%s: \\\n' % (options.output,)) files = read_manifest(options.manifest) files = [(x, y % defines) for (x, y) in files] files = list(expand(files)) files = [(x, unsymlink(y)) for (x, y) in files] files = [(x, y) for (x, y) in files if not x.endswith("-stripped.so")] files = [(x, strip_file(y)) for (x, y) in files] pos = (len(files) + 1) * metadata_size for name, hostname in files: type = 0 if hostname.startswith("->"): link = hostname[2:] type = 1 size = len(link.encode())+1 elif os.path.isdir(hostname): size = 0; if not name.endswith("/"): name += "/" else: size = os.stat(hostname).st_size # FIXME: check if name.encode()'s length is more than 110 (111 # minus the necessary null byte) and fail if it is. metadata = struct.pack('QQb111s', size, pos, type, name.encode()) out.write(metadata) pos += size depends.write(u'\t%s \\\n' % (hostname,)) out.write(struct.pack('128s', b'')) for name, hostname in files: if os.path.isdir(hostname): continue if hostname.startswith("->"): link = hostname[2:] out.write(link.encode()) out.write('\0') else: out.write(open(hostname, 'rb').read()) depends.write(u'\n\n') out.close() depends.close()