class ResourceSetUp(TempDirFixture):
__test__ = False
def setUp(self):
self.dir_manager = DirManager(self.path)
self.node_name = str(uuid.uuid4())
self.task_id = str(uuid.uuid4())
self.resources_dir = self.dir_manager.get_task_resource_dir(
self.task_id)
self.test_file = os.path.join(self.resources_dir, 'test_file.one.2')
self.test_dir = os.path.join(self.resources_dir, 'test_dir.one.2')
self.test_dir_file = os.path.join(self.resources_dir, 'dir_file.one.2')
self.split_resources = [['test_file.one.two'],
['test_dir.one.two', 'dir_file.one.two']]
self.joined_resources = [
os.path.join(*r) for r in self.split_resources
]
self.target_resources = [
os.path.join(self.resources_dir, *self.split_resources[0]),
os.path.join(self.resources_dir, *self.split_resources[1])
]
if not os.path.isdir(self.test_dir):
os.makedirs(self.test_dir)
open(self.test_file, 'w').close()
with open(self.test_dir_file, 'w') as f:
f.write("test content")
def testGetTaskResourceDir(self):
dm = DirManager(self.path)
task_id = '12345'
resDir = dm.get_task_resource_dir(task_id)
expectedResDir = os.path.join(self.path, task_id, 'resources')
self.assertEquals(os.path.normpath(resDir), expectedResDir)
self.assertTrue(os.path.isdir(resDir))
resDir = dm.get_task_resource_dir(task_id)
self.assertTrue(os.path.isdir(resDir))
resDir = dm.get_task_resource_dir(task_id, create=False)
self.assertTrue(os.path.isdir(resDir))
self.assertEquals(os.path.normpath(resDir), expectedResDir)
shutil.rmtree(resDir)
resDir = dm.get_task_resource_dir(task_id, create=False)
self.assertFalse(os.path.isdir(resDir))
resDir = dm.get_task_resource_dir(task_id, create=True)
self.assertTrue(os.path.isdir(resDir))
class TestTaskResourceHeader(TempDirFixture):
def setUp(self):
TempDirFixture.setUp(self)
self.dir_manager = DirManager(self.path)
res_path = self.dir_manager.get_task_resource_dir('task2')
self.file1 = os.path.join(res_path, 'file1')
self.file2 = os.path.join(res_path, 'file2')
self.dir1 = os.path.join(res_path, 'dir1')
self.file3 = os.path.join(self.dir1, 'file3')
open(self.file1, 'w').close()
open(self.file2, 'w').close()
if not os.path.isdir(self.dir1):
os.mkdir(self.dir1)
open(self.file3, 'w').close()
def testBuild(self):
dir_name = self.dir_manager.get_task_resource_dir("task2")
header = TaskResourceHeader.build("resource", dir_name)
self.assertEqual(len(header.files_data), 2)
self.assertEqual(len(header.sub_dir_headers[0].files_data), 1)
def testBuildFromChosen(self):
dir_name = self.dir_manager.get_task_resource_dir('task2')
header = TaskResourceHeader.build_from_chosen("resource", dir_name, [self.file1, self.file3])
header2 = TaskResourceHeader.build_header_delta_from_header(TaskResourceHeader("resource"), dir_name,
[self.file1, self.file3])
self.assertTrue(header == header2)
self.assertEqual(header.dir_name, header2.dir_name)
self.assertEqual(header.files_data, header2.files_data)
with self.assertRaises(TypeError):
TaskResourceHeader.build_header_delta_from_chosen(None, None)
self.assertEqual(TaskResourceHeader.build_header_delta_from_chosen(header, self.path),
TaskResourceHeader(header.dir_name))
with self.assertRaises(TypeError):
TaskResourceHeader.build_parts_header_delta_from_chosen(None, None, None)
with self.assertRaises(TypeError):
TaskResourceHeader.build_header_delta_from_header(None, None, None)
def testClearResource(self):
dm = DirManager(self.path)
task_id = '67891'
resDir = dm.get_task_resource_dir(task_id)
self.assertTrue(os.path.isdir(resDir))
file1 = os.path.join(resDir, 'file1')
file2 = os.path.join(resDir, 'file2')
dir1 = os.path.join(resDir, 'dir1')
file3 = os.path.join(dir1, 'file3')
open(file1, 'w').close()
open(file2, 'w').close()
if not os.path.isdir(dir1):
os.mkdir(dir1)
open(file3, 'w').close()
self.assertTrue(os.path.isfile(file1))
self.assertTrue(os.path.isfile(file2))
self.assertTrue(os.path.isfile(file3))
self.assertTrue(os.path.isdir(dir1))
dm.clear_resource(task_id)
self.assertTrue(os.path.isdir(resDir))
self.assertFalse(os.path.isfile(file1))
self.assertFalse(os.path.isfile(file2))
self.assertFalse(os.path.isfile(file3))
self.assertFalse(os.path.isdir(dir1))
class TestResourcesManager(TestDirFixture):
def setUp(self):
TestDirFixture.setUp(self)
self.dir_manager = DirManager(self.path)
res_path = self.dir_manager.get_task_resource_dir('task2')
file1 = os.path.join(res_path, 'file1')
file2 = os.path.join(res_path, 'file2')
dir1 = os.path.join(res_path, 'dir1')
file3 = os.path.join(dir1, 'file3')
open(file1, 'w').close()
open(file2, 'w').close()
if not os.path.isdir(dir1):
os.mkdir(dir1)
open(file3, 'w').close()
def testInit(self):
self.assertIsNotNone(ResourcesManager(self.dir_manager, 'owner'))
def testGetResourceHeader(self):
rm = ResourcesManager(self.dir_manager, 'owner')
header = rm.get_resource_header('task2')
self.assertEquals(len(header.files_data), 2)
self.assertEquals(len(header.sub_dir_headers[0].files_data), 1)
header2 = rm.get_resource_header('task3')
self.assertEquals(len(header2.files_data), 0)
self.assertEquals(len(header2.sub_dir_headers), 0)
def testGetResourceDelta(self):
rm = ResourcesManager(self.dir_manager, 'owner')
header = rm.get_resource_header('task2')
delta = rm.get_resource_delta('task2', header)
self.assertEquals(len(delta.files_data), 0)
self.assertEquals(len(delta.sub_dir_resources[0].files_data), 0)
header2 = rm.get_resource_header('task3')
delta2 = rm.get_resource_delta('task2', header2)
self.assertEquals(len(delta2.files_data), 2)
self.assertEquals(len(delta2.sub_dir_resources[0].files_data), 1)
res_path = self.dir_manager.get_task_resource_dir('task2')
file5 = os.path.join(res_path, 'file5')
open(file5, 'w').close()
dir1 = os.path.join(res_path, 'dir1')
file4 = os.path.join(dir1, 'file4')
open(file4, 'w').close()
delta3 = rm.get_resource_delta('task2', header)
self.assertEquals(len(delta3.files_data), 1)
self.assertEquals(len(delta3.sub_dir_resources[0].files_data), 1)
os.remove(file4)
os.remove(file5)
#
# def testPrepareResourceDelta(self):
# assert False
#
# def testUpdateResource(self):
# assert False
#
def testGetResourceDir(self):
rm = ResourcesManager(self.dir_manager, 'owner')
resDir = rm.get_resource_dir('task2')
self.assertTrue(os.path.isdir(resDir))
self.assertEqual(resDir,
self.dir_manager.get_task_resource_dir('task2'))
def testGetTemporaryDir(self):
rm = ResourcesManager(self.dir_manager, 'owner')
tmp_dir = rm.get_temporary_dir('task2')
self.assertTrue(os.path.isdir(tmp_dir))
self.assertEqual(tmp_dir,
self.dir_manager.get_task_temporary_dir('task2'))
def testGetOutputDir(self):
rm = ResourcesManager(self.dir_manager, 'owner')
outDir = rm.get_output_dir('task2')
self.assertTrue(os.path.isdir(outDir))
self.assertEqual(outDir, self.dir_manager.get_task_output_dir('task2'))
class TestIPFSClient(TestDirFixture):
def setUp(self):
TestDirFixture.setUp(self)
task_id = str(uuid.uuid4())
self.node_name = str(uuid.uuid4())
self.dir_manager = DirManager(self.path)
res_path = self.dir_manager.get_task_resource_dir(task_id)
self.test_dir = os.path.join(res_path, 'test_dir')
self.test_dir_file = os.path.join(self.test_dir, 'dir_file')
if not os.path.isdir(self.test_dir):
os.mkdir(self.test_dir)
self._write_test_file(1)
def testAdd(self):
client = IPFSClient()
response = client.add([self.test_dir_file])
assert response['Name']
assert response['Hash']
def testGet(self):
self._write_test_file(102400)
client = IPFSClient()
response = client.add([self.test_dir_file])
client.get(response['Hash'], filepath=self.test_dir)
tmp_file_path = os.path.join(self.test_dir, response['Hash'])
assert os.stat(tmp_file_path).st_size == os.stat(
self.test_dir_file).st_size
assert self._md5sum(tmp_file_path) == self._md5sum(self.test_dir_file)
def testGetFile(self):
self._write_test_file(102400)
client = IPFSClient()
response = client.add([self.test_dir_file])
assert response
tmp_filename = 'tmp_file'
get_response = client.get_file(response['Hash'],
filepath=self.test_dir,
filename=tmp_filename)
assert get_response['Name'] == os.path.join(self.test_dir,
tmp_filename)
assert get_response['Hash'] == response['Hash']
tmp_file_path = os.path.join(self.test_dir, tmp_filename)
assert os.stat(tmp_file_path).st_size == os.stat(
self.test_dir_file).st_size
assert self._md5sum(tmp_file_path) == self._md5sum(self.test_dir_file)
def testPinAdd(self):
client = IPFSClient()
response = client.add([self.test_dir_file])
self.assertIsNotNone(response)
client.pin_add(response['Hash'])
def testPinRm(self):
client = IPFSClient()
response = client.add([self.test_dir_file])
self.assertIsNotNone(response)
client.pin_add(response['Hash'])
client.pin_rm(response['Hash'])
def testBuildOptions(self):
from golem.resource.client import ClientError
client = IPFSClient()
client_options = {
'options': {
'option1': 1,
'option2': 'abcd',
'option3': None
}
}
option = client.build_options("id", **client_options)
print option
assert option.client_id == client.CLIENT_ID
assert option.version == client.VERSION
with self.assertRaises(ClientError):
option.get("Incorrect_id", client.VERSION, None)
with self.assertRaises(ClientError):
option.get(client.CLIENT_ID, client.VERSION + 1, None)
assert option.get(client.CLIENT_ID, client.VERSION, 'option1') == 1
assert option.get(client.CLIENT_ID, client.VERSION,
'option2') == "abcd"
assert not option.get(client.CLIENT_ID, client.VERSION, 'option3')
def _write_test_file(self, n_entries):
with open(self.test_dir_file, 'w') as f:
for i in xrange(0, n_entries):
f.write(str(uuid.uuid4()) + "\n")
f.flush()
@staticmethod
def _md5sum(file_name):
hash_md5 = hashlib.md5()
with open(file_name, "rb") as f:
for chunk in iter(lambda: f.read(1024), b""):
hash_md5.update(chunk)
return hash_md5.hexdigest()
class TestIPFSResourceManager(TestDirFixture):
def setUp(self):
TestDirFixture.setUp(self)
self.node_name = str(uuid.uuid4())
self.task_id = str(uuid.uuid4())
self.dir_manager = DirManager(self.path)
res_path = self.dir_manager.get_task_resource_dir(self.task_id)
test_file = os.path.join(res_path, 'test_file.one.two')
test_dir = os.path.join(res_path, 'test_dir.one.two')
test_dir_file = os.path.join(test_dir, 'dir_file.one.two')
self.split_resources = [
['test_file.one.two'],
['test_dir.one.two', 'dir_file.one.two']
]
self.target_resources = [
os.path.join(res_path, *self.split_resources[0]),
os.path.join(res_path, *self.split_resources[1])
]
if not os.path.isdir(test_dir):
os.mkdir(test_dir)
open(test_file, 'w').close()
with open(test_dir_file, 'w') as f:
f.write("test content")
def test_new_client(self):
rm = IPFSResourceManager(self.dir_manager)
rm.storage.clear_cache()
from golem.network.ipfs.client import IPFSClient
self.assertIsInstance(rm.new_client(), IPFSClient)
def test_pin(self):
rm = IPFSResourceManager(self.dir_manager)
rm.storage.clear_cache()
rm.add_files(self.target_resources, self.task_id)
resources = rm.storage.get_resources(self.task_id)
assert resources
result = rm.pin_resource(resources[0].hash)
assert result
def test_unpin(self):
rm = IPFSResourceManager(self.dir_manager)
rm.storage.clear_cache()
rm.add_files(self.target_resources, self.task_id)
resources = rm.storage.get_resources(self.task_id)
assert resources
rm.pin_resource(resources[0].hash)
rm.unpin_resource(resources[0].hash)
def test_pull(self):
rm = IPFSResourceManager(self.dir_manager)
rm.storage.clear_cache()
rm.add_files(self.target_resources, self.task_id)
rls = rm.storage.get_resources(self.task_id)
assert rls
rl = rls[0]
multihash = rl.hash
# working, downloaded
status = [True, False]
async = False
def success(*args, **kwargs):
status[0] = False
status[1] = True
def error(*args, **kwargs):
status[0] = False
status[1] = False
raise ValueError("Invalid value downloaded %r" % args)
def wait():
while status[0]:
time.sleep(0.25)
self.assertTrue(status[1])
rm.pull_resource(('other_resource', multihash),
self.task_id,
success, error,
async=async)
class TestResourceServer(TestDirFixtureWithReactor):
def setUp(self):
TestDirFixtureWithReactor.setUp(self)
self.task_id = str(uuid.uuid4())
self.dir_manager = DirManager(self.path)
self.dir_manager_aux = DirManager(self.path)
self.config_desc = MockConfig()
self.target_resources = [
'test_file',
os.path.join('test_dir', 'dir_file'),
os.path.join('test_dir', 'dir_file_copy')
]
res_path = self.dir_manager.get_task_resource_dir(self.task_id)
test_file = os.path.join(res_path, 'test_file')
test_dir = os.path.join(res_path, 'test_dir')
test_dir_file = os.path.join(test_dir, 'dir_file')
test_dir_file_copy = os.path.join(test_dir, 'dir_file_copy')
open(test_file, 'w').close()
if not os.path.isdir(test_dir):
os.mkdir(test_dir)
with open(test_dir_file, 'w') as f:
f.write("test content")
if os.path.exists(test_dir_file_copy):
os.remove(test_dir_file_copy)
shutil.copy(test_dir_file, test_dir_file_copy)
def testStartAccepting(self):
keys_auth = EllipticalKeysAuth(self.path)
client = MockClient()
rs = BaseResourceServer(TestResourceManager(self.dir_manager),
self.dir_manager, keys_auth, client)
rs.start_accepting()
def testGetDistributedResourceRoot(self):
keys_auth = EllipticalKeysAuth(self.path)
client = MockClient()
rs = BaseResourceServer(TestResourceManager(self.dir_manager),
self.dir_manager, keys_auth, client)
resource_dir = self.dir_manager.get_node_dir()
assert rs.get_distributed_resource_root() == resource_dir
def testDecrypt(self):
keys_auth = EllipticalKeysAuth(self.path)
client = MockClient()
rs = BaseResourceServer(TestResourceManager(self.dir_manager),
self.dir_manager, keys_auth, client)
to_encrypt = "test string to enc"
encrypted = rs.encrypt(to_encrypt, keys_auth.get_public_key())
decrypted = rs.decrypt(encrypted)
self.assertEqual(decrypted, to_encrypt)
def _resources(self):
existing_dir = self.dir_manager.get_task_resource_dir(self.task_id)
existing_paths = []
for resource in self.target_resources:
resource_path = os.path.join(existing_dir, resource)
existing_paths.append(resource_path)
return existing_paths
def _add_task(self):
keys_auth = EllipticalKeysAuth(self.path)
client = MockClient()
new_config_desc = MockConfig(self.path, node_name)
dir_manager = DirManager(new_config_desc.root_path)
rs = BaseResourceServer(TestResourceManager(self.dir_manager),
dir_manager, keys_auth, client)
rm = rs.resource_manager
rm.storage.clear_cache()
existing_paths = self._resources()
return rm, rs, rs.add_task(existing_paths, self.task_id)
def testAddTask(self):
rm, rs, deferred = self._add_task()
def test(*_):
resources = rm.storage.get_resources(self.task_id)
assert resources
assert len(resources) == len(self.target_resources)
deferred.addCallbacks(
test,
lambda e: self.fail(e)
)
started = time.time()
while not deferred.called:
if time.time() - started > 10:
self.fail("Test timed out")
time.sleep(0.1)
def testChangeResourceDir(self):
keys_auth = EllipticalKeysAuth(self.path)
client = MockClient()
rm = TestResourceManager(self.dir_manager)
rs = BaseResourceServer(TestResourceManager(self.dir_manager),
self.dir_manager, keys_auth, client)
rm.add_files(self._resources(), self.task_id,
absolute_path=True)
resources = rm.storage.get_resources(self.task_id)
assert resources
new_path = self.path + '_' + str(uuid.uuid4())
new_config_desc = MockConfig(new_path, node_name + "-new")
rs.change_resource_dir(new_config_desc)
new_resources = rm.storage.get_resources(self.task_id)
assert len(resources) == len(new_resources)
for resource in resources:
assert resource in new_resources
if os.path.exists(new_path):
shutil.rmtree(new_path)
def testRemoveTask(self):
keys_auth = EllipticalKeysAuth(self.path)
client = MockClient()
rs = BaseResourceServer(TestResourceManager(self.dir_manager),
self.dir_manager, keys_auth, client)
rm = rs.resource_manager
rm.add_files(self._resources(), self.task_id,
absolute_path=True)
assert rm.storage.get_resources(self.task_id)
rs.remove_task(self.task_id)
resources = rm.storage.get_resources(self.task_id)
assert not resources
def testGetResources(self):
keys_auth = EllipticalKeysAuth(self.path)
client = MockClient()
rs = BaseResourceServer(TestResourceManager(self.dir_manager),
self.dir_manager, keys_auth, client)
rm = rs.resource_manager
rm.storage.clear_cache()
rm.add_files(self.target_resources, self.task_id)
common_path = common_dir(self.target_resources)
resources = rm.storage.get_resources(self.task_id)
assert len(resources) == len(self.target_resources)
assert len(rs.pending_resources) == 0
rs.download_resources(resources, self.task_id)
assert len(rs.pending_resources[self.task_id]) == len(resources)
rs_aux = BaseResourceServer(TestResourceManager(self.dir_manager),
self.dir_manager_aux, keys_auth, client)
relative_resources = []
for resource in resources:
relative_resources.append((resource.path.replace(common_path, '', 1),
resource.hash))
task_id_2 = str(uuid.uuid4())
assert len(rs_aux.pending_resources) == 0
rs_aux.download_resources(relative_resources, task_id_2)
assert len(rs_aux.pending_resources[task_id_2]) == len(resources)
rs_aux._download_resources(async=False)
for entry in relative_resources:
assert os.path.exists(entry[0])
assert client.downloaded
def testVerifySig(self):
keys_auth = EllipticalKeysAuth(self.path)
rs = BaseResourceServer(TestResourceManager(self.dir_manager),
self.dir_manager, keys_auth, MockClient())
test_str = "A test string to sign"
sig = rs.sign(test_str)
self.assertTrue(rs.verify_sig(sig, test_str, keys_auth.get_public_key()))
def testAddFilesToGet(self):
keys_auth = EllipticalKeysAuth(self.path)
rs = BaseResourceServer(TestResourceManager(self.dir_manager),
self.dir_manager, keys_auth, MockClient())
test_files = [
['file1.txt', '1'],
[os.path.join('tmp', 'file2.bin'), '2']
]
assert not rs.pending_resources
rs.download_resources(test_files, self.task_id)
assert len(rs.pending_resources[self.task_id]) == len(test_files)
return rs, test_files
def testDownloadSuccess(self):
rs, file_names = self.testAddFilesToGet()
resources = list(rs.pending_resources[self.task_id])
for entry in resources:
rs._download_success(entry.resource, self.task_id)
assert not rs.pending_resources
def testDownloadError(self):
rs, file_names = self.testAddFilesToGet()
resources = list(rs.pending_resources[self.task_id])
for entry in resources:
rs._download_error(Exception(), entry.resource, self.task_id)
assert not rs.pending_resources
