def get(self, ticket_id): # TODO: cancel copy if ticket expired or revoked if not ticket_id: raise HTTPBadRequest("Ticket id is required") # TODO: support partial range (e.g. bytes=0-*) if self.request.range: offset = self.request.range.start if self.request.range.end is None: size = tickets.get(ticket_id).size - offset else: size = self.request.range.end - offset status = 206 else: offset = 0 size = tickets.get(ticket_id).size status = 200 ticket = tickets.authorize(ticket_id, "read", offset + size) self.log.info("Reading %d bytes at offset %d from %s for ticket %s", size, offset, ticket.url.path, ticket_id) op = directio.Send(ticket.url.path, None, size, offset=offset, buffersize=self.config.daemon.buffer_size) filename = os.path.split(ticket.url.path)[1] with open(os.path.join("/tmp", filename), "a+") as f: f.seek(offset) for data in op: f.write(data) op = directio.Send(ticket.url.path, None, size, offset=offset, buffersize=self.config.daemon.buffer_size) ticket.add_operation(op) content_disposition = "attachment" if ticket.filename: filename = ticket.filename.encode("utf-8") content_disposition += "; filename=%s" % filename resp = webob.Response( status=status, app_iter=op, content_type="application/octet-stream", content_length=str(size), content_disposition=content_disposition, ) if self.request.range: content_range = self.request.range.content_range(size) resp.headers["content_range"] = str(content_range) return resp
def send(tmpdir, data, size, offset=0): src = tmpdir.join("src") src.write(data) dst = io.BytesIO() op = directio.Send(str(src), dst, size, offset=offset) op.run() return dst.getvalue()
def test_send_no_size(tmpdir, data, offset): src = tmpdir.join("src") src.write(data) dst = io.BytesIO() op = directio.Send(str(src), dst, offset=offset) op.run() assert dst.getvalue() == data[offset:]
def test_send_iterate_and_close(tmpfile): # Used when passing operation as app_iter on GET request. dst = io.BytesIO() op = directio.Send(str(tmpfile), dst, tmpfile.size()) for chunk in op: dst.write(chunk) op.close() assert not op.active
def backup(self, ticket_id, path, dest, size, buffer_size): op = directio.Send(path, None, size, buffersize=buffer_size) total = 0 print('Executing task id {0.id}, args: {0.args!r} kwargs: {0.kwargs!r}'. format(self.request)) gigs = 0 with open(dest, "w+") as f: #rdb.set_trace() for data in op: total += len(data) f.write(data) if total / 1024 / 1024 / 1024 > gigs: gigs = total / 1024 / 1024 / 1024 self.update_state(state='PENDING', meta={ 'size': size, 'total': total })
def get(self, ticket_id): # TODO: cancel copy if ticket expired or revoked if not ticket_id: raise HTTPBadRequest("Ticket id is required") # TODO: support partial range (e.g. bytes=0-*) if self.request.range: offset = self.request.range.start if self.request.range.end is None: size = tickets.get(ticket_id)["size"] - offset else: size = self.request.range.end - offset status = 206 else: offset = 0 size = tickets.get(ticket_id)["size"] status = 200 ticket = tickets.authorize(ticket_id, "read", offset + size) self.log.info("Reading %d bytes at offset %d from %s for ticket %s", size, offset, ticket["url"].path, ticket_id) op = directio.Send(ticket["url"].path, None, size, offset=offset, buffersize=self.config.daemon.buffer_size) content_disposition = "attachment" if "filename" in ticket: filename = ticket["filename"].encode("utf-8") content_disposition += "; filename=%s" % filename resp = webob.Response( status=status, app_iter=op, content_type="application/octet-stream", content_length=str(size), content_disposition=content_disposition, ) if self.request.range: content_range = self.request.range.content_range(size) resp.headers["content_range"] = str(content_range) return resp
def get(self, ticket_id): # TODO: cancel copy if ticket expired or revoked if not ticket_id: raise HTTPBadRequest("Ticket id is required") # TODO: support partial range (e.g. bytes=0-*) offset = 0 size = None if self.request.range: offset = self.request.range.start if self.request.range.end is not None: size = self.request.range.end - offset ticket = tickets.authorize(ticket_id, "read", offset, size) if size is None: size = ticket.size - offset self.log.info("[%s] READ size=%d offset=%d ticket=%s", web.client_address(self.request), size, offset, ticket_id) op = directio.Send(ticket.url.path, None, size, offset=offset, buffersize=self.config.daemon.buffer_size, clock=self.clock) content_disposition = "attachment" if ticket.filename: filename = ticket.filename.encode("utf-8") content_disposition += "; filename=%s" % filename resp = webob.Response( status=206 if self.request.range else 200, app_iter=ticket.bind(op), content_type="application/octet-stream", content_length=str(size), content_disposition=content_disposition, ) if self.request.range: content_range = self.request.range.content_range(ticket.size) resp.headers["content-range"] = str(content_range) return resp
def extract_disk(ova_path, pos, disk_size, image_path): send = directio.Send(ova_path, None, offset=pos, size=disk_size, buffersize=BUF_SIZE) op = directio.Receive(image_path, SendAdapter(send), size=disk_size, buffersize=BUF_SIZE) op.run()
def backup(self, ticket_id, path, dest, size, type, buffer_size, recent_snap_id): if type == "full": cmdspec = [ 'qemu-img', 'convert', '-p', ] cmdspec += ['-O', 'qcow2', path, dest] cmd = " ".join(cmdspec) print('Take a full snapshot with qemu-img convert cmd: %s ' % cmd) process = subprocess.Popen(cmdspec, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=-1, close_fds=True, shell=False) queue = Queue() read_thread = Thread(target=enqueue_output, args=(process.stdout, queue)) read_thread.daemon = True # thread dies with the program read_thread.start() percentage = 0.0 while process.poll() is None: try: try: output = queue.get(timeout=300) except Empty: continue except Exception as ex: print(ex) percentage = re.search(r'\d+\.\d+', output).group(0) print(("copying from %(path)s to " "%(dest)s %(percentage)s %% completed\n") % {'path': path, 'dest': dest, 'percentage': str(percentage)}) percentage = float(percentage) self.update_state(state='PENDING', meta={'percentage': percentage}) except Exception as ex: pass '''qemu_cmd = ["qemu-img", "info", "--output", "json", dest] temp_process = subprocess.Popen(qemu_cmd, stdout=subprocess.PIPE) data, err = temp_process.communicate() data = json.loads(data) size = data["actual-size"] process.stdin.close() self.update_state(state='PENDING', meta={'actual-size': size})''' _returncode = process.returncode # pylint: disable=E1101 if _returncode: print(('Result was %s' % _returncode)) raise Exception("Execution error %(exit_code)d (%(stderr)s). " "cmd %(cmd)s" % {'exit_code': _returncode, 'stderr': process.stderr.read(), 'cmd': cmd}) else: process = subprocess.Popen('qemu-img info --backing-chain --output json ' + path, stdout=subprocess.PIPE, shell=True) stdout, stderr = process.communicate() if stderr: print(('Result was %s' % stderr)) raise Exception("Execution error %(exit_code)d (%(stderr)s). " "cmd %(cmd)s" % {'exit_code': 1, 'stderr': stderr, 'cmd': 'qemu-img info --backing-chain --output json ' + path}) result = json.loads(stdout) first_record = result[0] first_record_backing_file = first_record.get('backing-filename', None) recent_snap_path = recent_snap_id.get(str(first_record_backing_file), None) if first_record_backing_file and recent_snap_path: op = directio.Send(path, None, size, buffersize=buffer_size) total = 0 print('Executing task id {0.id}, args: {0.args!r} kwargs: {0.kwargs!r}'.format( self.request)) gigs = 0 with open(dest, "w+") as f: for data in op: total += len(data) f.write(data) if total/1024/1024/1024 > gigs: gigs = total/1024/1024/1024 percentage = (total/size) * 100 self.update_state(state='PENDING', meta={'percentage': percentage}) process = subprocess.Popen('qemu-img rebase -u -b ' + recent_snap_path + ' ' + dest, stdout=subprocess.PIPE, shell=True) stdout, stderr = process.communicate() if stderr: log.error("Unable to change the backing file", dest, stderr) else: temp_random_id = generate_random_string(5) tempdir = '/var/triliovault-mounts/staging/' + temp_random_id os.makedirs(tempdir) commands = [] for record in result: filename = os.path.basename(str(record.get('filename', None))) recent_snap_path = recent_snap_id.get(str(record.get('backing-filename')), None) if record.get('backing-filename', None) and str(record.get('backing-filename', None)) and not recent_snap_path: try: shutil.copy(path, tempdir) backing_file = os.path.basename(str(record.get('backing-filename', None))) command = 'qemu-img rebase -u -b ' + backing_file + ' ' + filename commands.append(command) except IOError as e: print("Unable to copy file. %s" % e) except: print("Unexpected error:", sys.exc_info()) else: try: shutil.copy(path, tempdir) command = 'qemu-img rebase -u ' + filename commands.append(command) except IOError as e: print("Unable to copy file. %s" % e) except: print("Unexpected error:", sys.exc_info()) break path = str(record.get('full-backing-filename')) string_commands = ";".join(str(x) for x in commands) process = subprocess.Popen(string_commands, stdin=subprocess.PIPE, stdout=subprocess.PIPE , cwd=tempdir, shell=True) stdout, stderr = process.communicate() if stderr: raise Exception(stdout) cmdspec = [ 'qemu-img', 'convert', '-p', ] filename = os.path.basename(str(first_record.get('filename', None))) path = os.path.join(tempdir, filename) cmdspec += ['-O', 'qcow2', path, dest] process = subprocess.Popen(cmdspec, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=-1, close_fds=True, shell=False) queue = Queue() read_thread = Thread(target=enqueue_output, args=(process.stdout, queue)) read_thread.daemon = True # thread dies with the program read_thread.start() percentage = 0.0 while process.poll() is None: try: try: output = queue.get(timeout=300) except Empty: continue except Exception as ex: print(ex) percentage = re.search(r'\d+\.\d+', output).group(0) print(("copying from %(path)s to " "%(dest)s %(percentage)s %% completed\n") % {'path': path, 'dest': dest, 'percentage': str(percentage)}) percentage = float(percentage) self.update_state(state='PENDING', meta={'percentage': percentage}) except Exception as ex: pass if recent_snap_path: process = subprocess.Popen('qemu-img rebase -u -b ' + recent_snap_path + ' ' + dest, stdout=subprocess.PIPE, shell=True) stdout, stderr = process.communicate() if stderr: log.error("Unable to change the backing file", dest, stderr) del_command = 'rm -rf ' + tempdir delete_process = subprocess.Popen(del_command, shell=True, stdout=subprocess.PIPE) delete_process.communicate()
def test_send_repr(): op = directio.Send("/path", None, 200, offset=24) rep = repr(op) assert "Send" in rep assert "path='/path' size=200 offset=24 buffersize=512 done=0" in rep
def test_send_busy(tmpfile): op = directio.Send(str(tmpfile), io.BytesIO(), tmpfile.size()) next(iter(op)) assert op.active
def test_send_repr_active(): op = directio.Send("/path", None) op.close() assert "active" not in repr(op)
def test_send_close_twice(tmpfile): op = directio.Send(str(tmpfile), io.BytesIO(), tmpfile.size()) op.run() op.close() # Should do nothing assert not op.active
def test_send_close_on_error(tmpfile): op = directio.Send(str(tmpfile), io.BytesIO(), tmpfile.size() + 1) with pytest.raises(errors.PartialContent): op.run() assert not op.active
def test_send_close_on_success(tmpfile): op = directio.Send(str(tmpfile), io.BytesIO(), tmpfile.size()) op.run() assert not op.active