def test_putcontent_unlinked(self):
"""Try to put content in an unlinked file."""
empty_hash = content_hash_factory().content_hash()
data = "*"
size = 1
hash_object = content_hash_factory()
hash_object.update(data)
hash_value = hash_object.content_hash()
crc32_value = crc32(data)
def auth(client):
# setup
d = client.dummy_authenticate("open sesame")
d.addCallback(lambda r: client.get_root())
# create file and remove it
d.addCallback(lambda r: client.make_file(request.ROOT, r, "hola"))
d.addCallback(lambda req: self._save_state("file", req.new_id))
d.addCallback(
lambda _: client.unlink(request.ROOT, self._state.file))
# try to put content
d.addCallback(lambda _: client.put_content(
request.ROOT, self._state.file, empty_hash, hash_value,
crc32_value, size, StringIO(data)))
d.addCallbacks(client.test_fail, lambda x: client.test_done("ok"))
return self.callback_test(auth)
def test_proxy_producer_not_streaming(self):
"""Test ProxyHashingProducer."""
data = os.urandom(1024 * 10)
message = zlib.compress(data)
ds = diskstorage.DiskStorage(os.path.join(self.tmpdir, "testfile"))
consumer = ds.put("somenode")
producer = upload.ProxyHashingProducer(consumer, False)
# add chunks, but see that nothing is really being calculated
chunk_sz = 10
for part in range(0, len(message), chunk_sz):
yield producer.dataReceived(message[part:part + chunk_sz])
self.assertEqual(producer.deflated_size, 0)
self.assertEqual(producer.inflated_size, 0)
self.assertEqual(producer.crc32, 0)
# stop and re-check
producer.stopProducing()
consumer.commit()
yield producer.flush_decompressor()
with open(consumer.filepath, "rb") as fh:
self.assertEqual(fh.read(), message)
hasher = content_hash_factory()
hasher.update(data)
self.assertEqual(producer.hash_object.content_hash(),
hasher.content_hash())
magic_hasher = magic_hash_factory()
magic_hasher.update(data)
self.assertEqual(producer.magic_hash_object.content_hash()._magic_hash,
magic_hasher.content_hash()._magic_hash)
self.assertEqual(producer.inflated_size, len(data))
self.assertEqual(producer.crc32, crc32(data))
def test_get_content_on_share(self):
"""read a file on a share."""
data = ""
deflated_data = zlib.compress(data)
hash_object = content_hash_factory()
hash_object.update(data)
hash_value = hash_object.content_hash()
crc32_value = crc32(data)
size = len(data)
deflated_size = len(deflated_data)
def auth(client):
"""auth"""
d = client.dummy_authenticate("open sesame")
# need to put data to be able to retrieve it!
d.addCallback(
lambda r: client.put_content(
self.share_modify, self.filerw, NO_CONTENT_HASH,
hash_value, crc32_value, size, deflated_size,
StringIO(deflated_data)))
d.addCallback(
lambda r: client.get_content(
self.share_modify, self.filerw, EMPTY_HASH))
d.addCallbacks(client.test_done, client.test_fail)
return self.callback_test(auth)
def test_getcontent_file_slow(self):
"""Get content from a file with very low BW and fail with timeout."""
data = os.urandom(300000)
deflated_data = zlib.compress(data)
hash_object = content_hash_factory()
hash_object.update(data)
hash_value = hash_object.content_hash()
crc32_value = crc32(data)
size = len(data)
deflated_size = len(deflated_data)
@defer.inlineCallbacks
def auth(client):
"""Test."""
yield client.dummy_authenticate("open sesame")
root = yield client.get_root()
# make a file and put content in it
mkfile_req = yield client.make_file(request.ROOT, root, "hola")
yield client.put_content(request.ROOT, mkfile_req.new_id,
NO_CONTENT_HASH, hash_value, crc32_value,
size, deflated_size,
StringIO(deflated_data))
# set the read limit, and get content
client.factory.factory.readLimit = 1000
yield client.get_content(request.ROOT, mkfile_req.new_id,
hash_value)
d = self.callback_test(auth, add_default_callbacks=True, timeout=0.1)
err = yield self.assertFailure(d, Exception)
self.assertEqual(str(err), "timeout")
def test_getcontent_file(self, check_file_content=True):
"""Get the content from a file."""
data = os.urandom(300000)
deflated_data = zlib.compress(data)
hash_object = content_hash_factory()
hash_object.update(data)
hash_value = hash_object.content_hash()
crc32_value = crc32(data)
size = len(data)
deflated_size = len(deflated_data)
def check_file(req):
if req.data != deflated_data:
raise Exception("data does not match")
def auth(client):
d = client.dummy_authenticate("open sesame")
d.addCallbacks(lambda _: client.get_root(), client.test_fail)
d.addCallbacks(
lambda root: client.make_file(request.ROOT, root, "hola"),
client.test_fail)
d.addCallback(self.save_req, 'req')
d.addCallbacks(
lambda mkfile_req: client.
put_content(request.ROOT, mkfile_req.new_id, NO_CONTENT_HASH,
hash_value, crc32_value, size, deflated_size,
StringIO(deflated_data)), client.test_fail)
d.addCallback(lambda _: client.get_content(
request.ROOT, self._state.req.new_id, hash_value))
if check_file_content:
d.addCallback(check_file)
d.addCallbacks(client.test_done, client.test_fail)
return self.callback_test(auth, timeout=5)
def test_put_content(self):
"""Write a file."""
data = "*" * 10000
deflated_data = zlib.compress(data)
hash_object = content_hash_factory()
hash_object.update(data)
hash_value = hash_object.content_hash()
crc32_value = crc32(data)
size = len(data)
deflated_size = len(deflated_data)
def auth(client):
"""Authenticate and test."""
d = client.dummy_authenticate("open sesame")
d.addCallback(lambda _: client.create_udf(u"~", u"myudf"))
d.addCallback(self.save_req, "udf")
# create a file with content
d.addCallback(lambda r: client.make_file(
self._state.udf.volume_id, self._state.udf.node_id, "foo"))
# put content
d.addCallback(lambda req: client.put_content(
self._state.udf.volume_id, req.new_id, NO_CONTENT_HASH,
hash_value, crc32_value, size, deflated_size,
StringIO(deflated_data)))
d.addCallbacks(client.test_done, client.test_fail)
return self.callback_test(auth)
def test_upload(self):
"""Hiccup the network in the middle of an upload."""
data = os.urandom(1000)
hash_object = content_hash_factory()
hash_object.update(data)
hash_value = hash_object.content_hash()
crc32_value = crc32(data)
size = len(data)
self.patch(self.main.fs, 'open_file', lambda mdid: StringIO(data))
mdid, node_id = yield self._mkfile('hola')
def worker():
"""Async worker."""
self.aq.upload('', node_id, NO_CONTENT_HASH, hash_value,
crc32_value, size, mdid)
return self.hiccup()
yield self.wait_for_nirvana()
yield self.nuke_client_method(
'put_content_request', worker, lambda: self.connlost_deferred)
self.assertInListenerEvents(
'AQ_UPLOAD_FINISHED',
{'share_id': '', 'hash': hash_value, 'node_id': anUUID,
'new_generation': 2})
def _get_data(self, data_len=1000):
"""Get the hash, crc and size of a chunk of data."""
data = os.urandom(data_len) # not terribly compressible
hash_object = content_hash_factory()
hash_object.update(data)
hash_value = hash_object.content_hash()
crc32_value = crc32(data)
size = len(data)
return NoCloseStringIO(data), data, hash_value, crc32_value, size
def __init__(self, consumer, streaming):
self.decompressor = zlib.decompressobj()
self.hash_object = content_hash_factory()
self.magic_hash_object = magic_hash_factory()
self.crc32 = 0
self.inflated_size = 0
self.deflated_size = 0
self.consumer = consumer
self.streaming = streaming
def test_putcontent_slow(self, num_files=1):
"""Test putting content to a file with very low bandwidth and fail
with timeout.
"""
data = os.urandom(30000)
deflated_data = zlib.compress(data)
hash_object = content_hash_factory()
hash_object.update(data)
hash_value = hash_object.content_hash()
crc32_value = crc32(data)
size = len(data)
deflated_size = len(deflated_data)
def auth(client):
def check_file(result):
@transaction.atomic
def _check_file():
content_blob = services.get_object_or_none(ContentBlob,
hash=hash_value)
if not content_blob:
raise ValueError("content blob is not there")
d = threads.deferToThread(_check_file)
return d
d = client.dummy_authenticate("open sesame")
filename = 'hola_1'
d.addCallbacks(lambda _: client.get_root(), client.test_fail)
d.addCallbacks(
lambda root: client.make_file(request.ROOT, root, filename),
client.test_fail)
def set_write_limit(r):
client.factory.factory.writeLimit = 100
return r
d.addCallback(set_write_limit)
d.addCallbacks(
lambda mkfile_req: client.
put_content(request.ROOT, mkfile_req.new_id, NO_CONTENT_HASH,
hash_value, crc32_value, size, deflated_size,
StringIO(deflated_data)), client.test_fail)
return d
d1 = defer.Deferred()
test_d = self.callback_test(auth, timeout=1)
test_d.addCallbacks(d1.errback, lambda r: d1.callback(None))
return d1
def test_shutdown_upload(self):
"""Stop and restart the server."""
service = yield self.create_service()
client = yield self.connect_client(service)
root = yield client.get_root()
mkfile_req = yield client.make_file(request.ROOT, root, "hola")
# try to upload something that will fail when sending data
empty_hash = content_hash_factory().content_hash()
# lose the connection if something wrong
try:
yield client.put_content(request.ROOT, mkfile_req.new_id,
empty_hash, "fake_hash", 1234, 1000, None)
except Exception:
client.transport.loseConnection()
ujobs = self.usr0.get_uploadjobs(node_id=mkfile_req.new_id)
self.assertEqual(ujobs, [])
def put(self, local, remote):
"""Put local file into remote file."""
try:
node_id = self.get_id_from_filename(remote)
except ValueError:
parent_id = self.get_cwd_id()
r = self.defer_from_thread(self.factory.current_protocol.make_file,
self.volume, parent_id,
remote.split("/")[-1])
node_id = r.new_id
old_hash = self.get_hash(node_id)
ho = content_hash_factory()
zipper = zlib.compressobj()
crc32_value = 0
size = 0
deflated_size = 0
temp_file_name = None
with open(local) as fh:
with tempfile.NamedTemporaryFile(mode='w',
prefix='cmd_client-',
delete=False) as dest:
temp_file_name = dest.name
while True:
cont = fh.read(1024**2)
if not cont:
dest.write(zipper.flush())
deflated_size = dest.tell()
break
ho.update(cont)
crc32_value = crc32(cont, crc32_value)
size += len(cont)
dest.write(zipper.compress(cont))
hash_value = ho.content_hash()
try:
self.defer_from_thread(self.factory.current_protocol.put_content,
self.volume, node_id, old_hash, hash_value,
crc32_value, size, deflated_size,
open(temp_file_name, 'r'))
finally:
if os.path.exists(temp_file_name):
os.unlink(temp_file_name)
def test_put_content_on_share_ro(self):
"""Write a file on a share thats read only."""
data = "*" * 100000
hash_object = content_hash_factory()
hash_object.update(data)
hash_value = hash_object.content_hash()
crc32_value = crc32(data)
size = len(data)
def auth(client):
"""auth"""
d = client.dummy_authenticate("open sesame")
d.addCallback(
lambda r: client.put_content(
self.share, self.filero, NO_CONTENT_HASH, hash_value,
crc32_value, size, StringIO(data)))
d.addCallbacks(client.test_fail, lambda x: client.test_done())
return self.callback_test(auth)
def test_putcontent(self, num_files=1):
"""Test putting content to a file."""
data = os.urandom(300000)
deflated_data = zlib.compress(data)
hash_object = content_hash_factory()
hash_object.update(data)
hash_value = hash_object.content_hash()
crc32_value = crc32(data)
size = len(data)
deflated_size = len(deflated_data)
def auth(client):
def check_file(result):
@transaction.atomic
def _check_file():
content_blob = services.get_object_or_none(ContentBlob,
hash=hash_value)
if not content_blob:
raise ValueError("content blob is not there")
d = threads.deferToThread(_check_file)
return d
d = client.dummy_authenticate("open sesame")
filenames = iter('hola_%d' % i for i in range(num_files))
for i in range(num_files):
d.addCallbacks(lambda _: client.get_root(), client.test_fail)
d.addCallbacks(
lambda root: client.make_file(request.ROOT, root,
filenames.next()),
client.test_fail)
d.addCallbacks(
lambda mkfile_req: client.put_content(
request.ROOT, mkfile_req.new_id, NO_CONTENT_HASH,
hash_value, crc32_value, size, deflated_size,
StringIO(deflated_data)), client.test_fail)
d.addCallback(check_file)
d.addCallbacks(client.test_done, client.test_fail)
return d
return self.callback_test(auth, timeout=1)
def do_create_lots_of_files(self, suffix=''):
"""A helper that creates N files."""
# data for putcontent
ho = content_hash_factory()
hash_value = ho.content_hash()
crc32_value = crc32("")
deflated_content = zlib.compress("")
deflated_size = len(deflated_content)
mk = yield self.client.make_file(request.ROOT, self.root_id,
"test_first" + suffix)
yield self.client.put_content(request.ROOT, mk.new_id, NO_CONTENT_HASH,
hash_value, crc32_value,
0, deflated_size,
StringIO(deflated_content))
for i in range(self.N):
mk = yield self.client.make_file(request.ROOT, self.root_id,
"test_%03x%s" % (i, suffix))
yield self.client.put_content(request.ROOT, mk.new_id,
NO_CONTENT_HASH, hash_value,
crc32_value, 0, deflated_size,
StringIO(deflated_content))
def test_mkfile_already_exists_content(self):
"""Create a file on a file that already exists and have content."""
data = "*" * 100
deflated_data = zlib.compress(data)
hash_object = content_hash_factory()
hash_object.update(data)
hash_value = hash_object.content_hash()
crc32_value = crc32(data)
size = len(data)
deflated_size = len(deflated_data)
def auth(client):
d = client.dummy_authenticate("open sesame")
d.addCallback(lambda r: client.get_root())
d.addCallback(self.save_req, "root")
d.addCallback(lambda r: client.make_file(request.ROOT, r, "hola"))
d.addCallback(lambda req: client.put_content(request.ROOT,
req.new_id, NO_CONTENT_HASH, hash_value, crc32_value,
size, deflated_size, StringIO(deflated_data)))
d.addCallback(lambda r: client.make_file(request.ROOT,
self._state.root, "hola"))
d.addCallbacks(
lambda x: client.test_done("ok"), client.test_fail)
return self.callback_test(auth)
# For further info, check http://launchpad.net/magicicada-server
"""Test User Defined Folder operations."""
import uuid
import zlib
from StringIO import StringIO
from magicicadaprotocol import request
from magicicadaprotocol.content_hash import content_hash_factory, crc32
from twisted.internet import reactor, defer
from magicicada.filesync.errors import DoesNotExist
from magicicada.server.testing.testcase import FactoryHelper, TestWithDatabase
EMPTY_HASH = content_hash_factory().content_hash()
NO_CONTENT_HASH = ""
class TestManageUDF(TestWithDatabase):
"""Test udf creation and deletion."""
def test_create_udf_simple(self):
"""Create an UDF."""
def auth(client):
"""Authenticate and test."""
d = client.dummy_authenticate("open sesame")
def check_result(req):
"""Check the request result."""
self.assertFalse(req.volume_id is None)
self.assertFalse(req.node_id is None)