def get(self, ticket_id):
# TODO: cancel copy if ticket expired or revoked
if not ticket_id:
raise HTTPBadRequest("Ticket id is required")
# TODO: support partial range (e.g. bytes=0-*)
if self.request.range:
offset = self.request.range.start
if self.request.range.end is None:
size = tickets.get(ticket_id).size - offset
else:
size = self.request.range.end - offset
status = 206
else:
offset = 0
size = tickets.get(ticket_id).size
status = 200
ticket = tickets.authorize(ticket_id, "read", offset + size)
self.log.info("Reading %d bytes at offset %d from %s for ticket %s",
size, offset, ticket.url.path, ticket_id)
op = directio.Send(ticket.url.path,
None,
size,
offset=offset,
buffersize=self.config.daemon.buffer_size)
filename = os.path.split(ticket.url.path)[1]
with open(os.path.join("/tmp", filename), "a+") as f:
f.seek(offset)
for data in op:
f.write(data)
op = directio.Send(ticket.url.path,
None,
size,
offset=offset,
buffersize=self.config.daemon.buffer_size)
ticket.add_operation(op)
content_disposition = "attachment"
if ticket.filename:
filename = ticket.filename.encode("utf-8")
content_disposition += "; filename=%s" % filename
resp = webob.Response(
status=status,
app_iter=op,
content_type="application/octet-stream",
content_length=str(size),
content_disposition=content_disposition,
)
if self.request.range:
content_range = self.request.range.content_range(size)
resp.headers["content_range"] = str(content_range)
return resp
def send(tmpdir, data, size, offset=0):
src = tmpdir.join("src")
src.write(data)
dst = io.BytesIO()
op = directio.Send(str(src), dst, size, offset=offset)
op.run()
return dst.getvalue()
def test_send_no_size(tmpdir, data, offset):
src = tmpdir.join("src")
src.write(data)
dst = io.BytesIO()
op = directio.Send(str(src), dst, offset=offset)
op.run()
assert dst.getvalue() == data[offset:]
def test_send_iterate_and_close(tmpfile):
# Used when passing operation as app_iter on GET request.
dst = io.BytesIO()
op = directio.Send(str(tmpfile), dst, tmpfile.size())
for chunk in op:
dst.write(chunk)
op.close()
assert not op.active
def backup(self, ticket_id, path, dest, size, buffer_size):
op = directio.Send(path, None, size, buffersize=buffer_size)
total = 0
print('Executing task id {0.id}, args: {0.args!r} kwargs: {0.kwargs!r}'.
format(self.request))
gigs = 0
with open(dest, "w+") as f:
#rdb.set_trace()
for data in op:
total += len(data)
f.write(data)
if total / 1024 / 1024 / 1024 > gigs:
gigs = total / 1024 / 1024 / 1024
self.update_state(state='PENDING',
meta={
'size': size,
'total': total
})
def get(self, ticket_id):
# TODO: cancel copy if ticket expired or revoked
if not ticket_id:
raise HTTPBadRequest("Ticket id is required")
# TODO: support partial range (e.g. bytes=0-*)
if self.request.range:
offset = self.request.range.start
if self.request.range.end is None:
size = tickets.get(ticket_id)["size"] - offset
else:
size = self.request.range.end - offset
status = 206
else:
offset = 0
size = tickets.get(ticket_id)["size"]
status = 200
ticket = tickets.authorize(ticket_id, "read", offset + size)
self.log.info("Reading %d bytes at offset %d from %s for ticket %s",
size, offset, ticket["url"].path, ticket_id)
op = directio.Send(ticket["url"].path,
None,
size,
offset=offset,
buffersize=self.config.daemon.buffer_size)
content_disposition = "attachment"
if "filename" in ticket:
filename = ticket["filename"].encode("utf-8")
content_disposition += "; filename=%s" % filename
resp = webob.Response(
status=status,
app_iter=op,
content_type="application/octet-stream",
content_length=str(size),
content_disposition=content_disposition,
)
if self.request.range:
content_range = self.request.range.content_range(size)
resp.headers["content_range"] = str(content_range)
return resp
def get(self, ticket_id):
# TODO: cancel copy if ticket expired or revoked
if not ticket_id:
raise HTTPBadRequest("Ticket id is required")
# TODO: support partial range (e.g. bytes=0-*)
offset = 0
size = None
if self.request.range:
offset = self.request.range.start
if self.request.range.end is not None:
size = self.request.range.end - offset
ticket = tickets.authorize(ticket_id, "read", offset, size)
if size is None:
size = ticket.size - offset
self.log.info("[%s] READ size=%d offset=%d ticket=%s",
web.client_address(self.request), size, offset,
ticket_id)
op = directio.Send(ticket.url.path,
None,
size,
offset=offset,
buffersize=self.config.daemon.buffer_size,
clock=self.clock)
content_disposition = "attachment"
if ticket.filename:
filename = ticket.filename.encode("utf-8")
content_disposition += "; filename=%s" % filename
resp = webob.Response(
status=206 if self.request.range else 200,
app_iter=ticket.bind(op),
content_type="application/octet-stream",
content_length=str(size),
content_disposition=content_disposition,
)
if self.request.range:
content_range = self.request.range.content_range(ticket.size)
resp.headers["content-range"] = str(content_range)
return resp
def extract_disk(ova_path, pos, disk_size, image_path):
send = directio.Send(ova_path, None, offset=pos, size=disk_size,
buffersize=BUF_SIZE)
op = directio.Receive(image_path, SendAdapter(send), size=disk_size,
buffersize=BUF_SIZE)
op.run()
def backup(self, ticket_id, path, dest, size, type, buffer_size, recent_snap_id):
if type == "full":
cmdspec = [
'qemu-img',
'convert',
'-p',
]
cmdspec += ['-O', 'qcow2', path, dest]
cmd = " ".join(cmdspec)
print('Take a full snapshot with qemu-img convert cmd: %s ' % cmd)
process = subprocess.Popen(cmdspec,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=-1,
close_fds=True,
shell=False)
queue = Queue()
read_thread = Thread(target=enqueue_output,
args=(process.stdout, queue))
read_thread.daemon = True # thread dies with the program
read_thread.start()
percentage = 0.0
while process.poll() is None:
try:
try:
output = queue.get(timeout=300)
except Empty:
continue
except Exception as ex:
print(ex)
percentage = re.search(r'\d+\.\d+', output).group(0)
print(("copying from %(path)s to "
"%(dest)s %(percentage)s %% completed\n") %
{'path': path,
'dest': dest,
'percentage': str(percentage)})
percentage = float(percentage)
self.update_state(state='PENDING',
meta={'percentage': percentage})
except Exception as ex:
pass
'''qemu_cmd = ["qemu-img", "info", "--output", "json", dest]
temp_process = subprocess.Popen(qemu_cmd, stdout=subprocess.PIPE)
data, err = temp_process.communicate()
data = json.loads(data)
size = data["actual-size"]
process.stdin.close()
self.update_state(state='PENDING',
meta={'actual-size': size})'''
_returncode = process.returncode # pylint: disable=E1101
if _returncode:
print(('Result was %s' % _returncode))
raise Exception("Execution error %(exit_code)d (%(stderr)s). "
"cmd %(cmd)s" %
{'exit_code': _returncode,
'stderr': process.stderr.read(),
'cmd': cmd})
else:
process = subprocess.Popen('qemu-img info --backing-chain --output json ' + path, stdout=subprocess.PIPE, shell=True)
stdout, stderr = process.communicate()
if stderr:
print(('Result was %s' % stderr))
raise Exception("Execution error %(exit_code)d (%(stderr)s). "
"cmd %(cmd)s" %
{'exit_code': 1,
'stderr': stderr,
'cmd': 'qemu-img info --backing-chain --output json ' + path})
result = json.loads(stdout)
first_record = result[0]
first_record_backing_file = first_record.get('backing-filename', None)
recent_snap_path = recent_snap_id.get(str(first_record_backing_file), None)
if first_record_backing_file and recent_snap_path:
op = directio.Send(path,
None,
size,
buffersize=buffer_size)
total = 0
print('Executing task id {0.id}, args: {0.args!r} kwargs: {0.kwargs!r}'.format(
self.request))
gigs = 0
with open(dest, "w+") as f:
for data in op:
total += len(data)
f.write(data)
if total/1024/1024/1024 > gigs:
gigs = total/1024/1024/1024
percentage = (total/size) * 100
self.update_state(state='PENDING',
meta={'percentage': percentage})
process = subprocess.Popen('qemu-img rebase -u -b ' + recent_snap_path + ' ' + dest, stdout=subprocess.PIPE, shell=True)
stdout, stderr = process.communicate()
if stderr:
log.error("Unable to change the backing file", dest, stderr)
else:
temp_random_id = generate_random_string(5)
tempdir = '/var/triliovault-mounts/staging/' + temp_random_id
os.makedirs(tempdir)
commands = []
for record in result:
filename = os.path.basename(str(record.get('filename', None)))
recent_snap_path = recent_snap_id.get(str(record.get('backing-filename')), None)
if record.get('backing-filename', None) and str(record.get('backing-filename', None)) and not recent_snap_path:
try:
shutil.copy(path, tempdir)
backing_file = os.path.basename(str(record.get('backing-filename', None)))
command = 'qemu-img rebase -u -b ' + backing_file + ' ' + filename
commands.append(command)
except IOError as e:
print("Unable to copy file. %s" % e)
except:
print("Unexpected error:", sys.exc_info())
else:
try:
shutil.copy(path, tempdir)
command = 'qemu-img rebase -u ' + filename
commands.append(command)
except IOError as e:
print("Unable to copy file. %s" % e)
except:
print("Unexpected error:", sys.exc_info())
break
path = str(record.get('full-backing-filename'))
string_commands = ";".join(str(x) for x in commands)
process = subprocess.Popen(string_commands, stdin=subprocess.PIPE, stdout=subprocess.PIPE
, cwd=tempdir, shell=True)
stdout, stderr = process.communicate()
if stderr:
raise Exception(stdout)
cmdspec = [
'qemu-img',
'convert',
'-p',
]
filename = os.path.basename(str(first_record.get('filename', None)))
path = os.path.join(tempdir, filename)
cmdspec += ['-O', 'qcow2', path, dest]
process = subprocess.Popen(cmdspec,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=-1,
close_fds=True,
shell=False)
queue = Queue()
read_thread = Thread(target=enqueue_output,
args=(process.stdout, queue))
read_thread.daemon = True # thread dies with the program
read_thread.start()
percentage = 0.0
while process.poll() is None:
try:
try:
output = queue.get(timeout=300)
except Empty:
continue
except Exception as ex:
print(ex)
percentage = re.search(r'\d+\.\d+', output).group(0)
print(("copying from %(path)s to "
"%(dest)s %(percentage)s %% completed\n") %
{'path': path,
'dest': dest,
'percentage': str(percentage)})
percentage = float(percentage)
self.update_state(state='PENDING',
meta={'percentage': percentage})
except Exception as ex:
pass
if recent_snap_path:
process = subprocess.Popen('qemu-img rebase -u -b ' + recent_snap_path + ' ' + dest, stdout=subprocess.PIPE, shell=True)
stdout, stderr = process.communicate()
if stderr:
log.error("Unable to change the backing file", dest, stderr)
del_command = 'rm -rf ' + tempdir
delete_process = subprocess.Popen(del_command, shell=True, stdout=subprocess.PIPE)
delete_process.communicate()
def test_send_repr():
op = directio.Send("/path", None, 200, offset=24)
rep = repr(op)
assert "Send" in rep
assert "path='/path' size=200 offset=24 buffersize=512 done=0" in rep
def test_send_busy(tmpfile):
op = directio.Send(str(tmpfile), io.BytesIO(), tmpfile.size())
next(iter(op))
assert op.active
def test_send_repr_active():
op = directio.Send("/path", None)
op.close()
assert "active" not in repr(op)
def test_send_close_twice(tmpfile):
op = directio.Send(str(tmpfile), io.BytesIO(), tmpfile.size())
op.run()
op.close() # Should do nothing
assert not op.active
def test_send_close_on_error(tmpfile):
op = directio.Send(str(tmpfile), io.BytesIO(), tmpfile.size() + 1)
with pytest.raises(errors.PartialContent):
op.run()
assert not op.active
def test_send_close_on_success(tmpfile):
op = directio.Send(str(tmpfile), io.BytesIO(), tmpfile.size())
op.run()
assert not op.active
