def test_task_result_output(remote):
rpc = create_rpc_component(remote=remote)
rpc.start()
TESTSTRING = 'test'
expected_results = set([GLAMBDA_RESULT_FILE, 'stdout.log', 'stderr.log'])
def test_task(args):
return TESTSTRING
_ = rpc.post_wait({
'type': 'CreateTask',
'task': {
'type': 'GLambda',
'options': {
'method': test_task
}
}
})
results = rpc.poll(timeout=None)['results']
result_directory = os.path.split(results[0])[0]
assert set(os.path.basename(r) for r in results) == expected_results
with open(os.path.join(result_directory, GLAMBDA_RESULT_FILE), 'r') as f:
assert f.read() == json.dumps({'data': TESTSTRING})
rpc.post_wait({
'type': 'Disconnect'
})
def test_task_timeout(remote):
rpc = create_rpc_component(remote=remote)
rpc.start()
# FIXME remove exception assertion when moving to test framework
def test_task(args):
import time
time.sleep(5.0)
try:
_ = rpc.post_wait({
'type': 'CreateTask',
'task': {
'type': 'GLambda',
'options': {
'method': test_task
},
'timeout': '00:00:00'
}
})
rpc.poll(timeout=None)
except RuntimeError as e:
pass
else:
assert False
def test_file_output(remote):
rpc = create_rpc_component(remote=remote)
rpc.start()
expected_results = set(
[GLAMBDA_RESULT_FILE, 'stdout.log', 'stderr.log', 'testfile'])
TESTSTRING = b'\xDA'
def test_task(args):
import os
with open('/golem/output/testfile', 'wb') as f:
f.write(TESTSTRING)
_ = rpc.post_wait({
'type': 'CreateTask',
'task': {
'type': 'GLambda',
'options': {
'method': test_task,
'outputs': list(expected_results)
}
}
})
results = rpc.poll(timeout=None)['results']
result_directory = os.path.split(results[0])[0]
assert set(os.path.basename(r) for r in results) == expected_results
with open(os.path.join(result_directory, 'testfile'), 'rb') as f:
assert f.read() == TESTSTRING
rpc.post_wait({
'type': 'Disconnect'
})
def test_implicit_no_verification():
rpc = create_rpc_component()
rpc.start()
TEST_STRING = 'test'
expected_results = set([GLAMBDA_RESULT_FILE, 'stdout.log', 'stderr.log'])
def test_task(args):
return 1 + 2
response = rpc.post_wait({
'type': 'CreateTask',
'task': {
'type': 'GLambda',
'options': {
'method': test_task
}
}
})
assert response['type'] == 'TaskCreatedEvent'
task_id = response['task_id']
response = rpc.poll(timeout=None)
assert response['type'] == 'TaskResults'
assert response['task_id'] == task_id
result = load_result(response['results'])
# Intentionally do not assert 'data' field
assert 'error' not in result
assert result['data'] == 3
rpc.post_wait({'type': 'Disconnect'})
def test_raise_exception(remote):
rpc = create_rpc_component(remote=remote)
rpc.start()
TEST_STRING = 'test'
expected_results = set([GLAMBDA_RESULT_FILE, 'stdout.log', 'stderr.log'])
def test_task(args):
raise RuntimeError(TEST_STRING)
_ = rpc.post_wait({
'type': 'CreateTask',
'task': {
'type': 'GLambda',
'options': {
'method': test_task
}
}
})
results = rpc.poll(timeout=None)['results']
result_directory = os.path.split(results[0])[0]
assert set(os.path.basename(r) for r in results) == expected_results
with open(os.path.join(result_directory, GLAMBDA_RESULT_FILE), 'r') as f:
result_json = json.loads(f.read())
assert 'error' in result_json
assert result_json['error'] == TEST_STRING
assert 'data' not in result_json
rpc.post_wait({'type': 'Disconnect'})
def test_task_too_big(remote):
rpc = create_rpc_component(remote=remote)
rpc.start()
# FIXME remove exception assertion when moving to test framework
def test_task(args):
import time
time.sleep(5.0)
try:
_ = rpc.post_wait({
'type': 'CreateTask',
'task': {
'type': 'GLambda',
'options': {
'method': test_task
},
'timeout': '00:00:00',
# This must be adjusted to Golem:golem/tools/uploadcontroller.py changes
'dummy_data': [0 for _ in range(20 * 1024 * 1024)]
}
})
rpc.poll(timeout=None)
except ValueError as e:
pass
else:
assert False
def test_non_serializable_task_result(remote):
rpc = create_rpc_component(remote=remote)
rpc.start()
expected_results = set([GLAMBDA_RESULT_FILE, 'stdout.log', 'stderr.log'])
def test_task(args):
return lambda x: print(x)
rpc.post({
'type': 'CreateTask',
'task': {
'type': 'GLambda',
'options': {
'method': test_task
},
'timeout': '00:10:00',
}
})
_ = rpc.poll(timeout=None)
results = rpc.poll(timeout=None)['results']
result_directory = os.path.split(results[0])[0]
assert set(os.path.basename(r) for r in results) == expected_results
with open(os.path.join(result_directory, GLAMBDA_RESULT_FILE), 'r') as f:
result_json = json.loads(f.read())
assert 'error' in result_json
assert 'data' not in result_json
rpc.post_wait({'type': 'Disconnect'})
def test_empty_resource(remote):
rpc = create_rpc_component(remote=remote)
rpc.start()
expected_results = set([GLAMBDA_RESULT_FILE, 'stdout.log', 'stderr.log'])
def test_task(args):
pass
_ = rpc.post_wait({
'type': 'CreateTask',
'task': {
'type': 'GLambda',
'options': {
'method': test_task
}
}
})
results = rpc.poll(timeout=None)['results']
assert set(os.path.basename(r) for r in results) == expected_results
result_directory = os.path.split(results[0])[0]
rpc.post_wait({'type': 'Disconnect'})
def test_empty_mapping():
rpc = create_rpc_component()
rpc.remote = True
rpc.start()
expected_results = set([GLAMBDA_RESULT_FILE, 'stdout.log', 'stderr.log'])
def dummy_task(args):
return True
_ = rpc.post_wait({
'type': 'CreateTask',
'task': {
'type': 'GLambda',
'options': {
'method': dummy_task
}
}
})
results = rpc.poll(timeout=None)['results']
result_directory = os.path.split(results[0])[0]
assert set(os.path.basename(r) for r in results) == expected_results
with open(os.path.join(result_directory, GLAMBDA_RESULT_FILE), 'r') as f:
j = json.loads(f.read())
assert 'data' in j
assert j['data'] is True
def test_dir_nested_mapping():
rpc = create_rpc_component()
rpc.remote = True
rpc.start()
expected_results = set([GLAMBDA_RESULT_FILE, 'stdout.log', 'stderr.log'])
def dummy_task(args):
import os
resources = os.listdir('/golem/resources/foo/baz')
if 'tmpfile' not in resources:
return False
if 'bar' not in resources:
return False
resources = os.listdir('/golem/resources/foo/baz/bar')
if 'tmpfile' not in resources:
return False
return True
os.mkdir('foo')
os.mkdir('foo/bar')
with open('foo/tmpfile', 'wb') as f:
f.write(b'\xDA')
with open('foo/bar/tmpfile', 'wb') as f:
f.write(b'\xDA')
_ = rpc.post_wait({
'type': 'CreateTask',
'task': {
'type': 'GLambda',
'options': {
'method': dummy_task
},
'resources_mapped': {
'foo': 'foo/baz',
}
}
})
shutil.rmtree('foo')
results = rpc.poll(timeout=None)['results']
result_directory = os.path.split(results[0])[0]
assert set(os.path.basename(r) for r in results) == expected_results
with open(os.path.join(result_directory, GLAMBDA_RESULT_FILE), 'r') as f:
j = json.loads(f.read())
assert 'data' in j
assert j['data'] is True
def test_transfer_manager():
src = '/usr/bin/snap'
result = 'tmp2'
rpc_component = create_rpc_component()
rpc_component.start()
transfer_mgr = TransferManager(rpc_component)
transfer_mgr.upload(src, 'tmp')
transfer_mgr.download('tmp', result)
src_size = os.stat(src).st_size
result_size = os.stat(result).st_size
os.remove(result)
assert src_size == result_size
def test_successful_verification():
rpc = create_rpc_component()
rpc.start()
TEST_STRING = 'test'
expected_results = set([GLAMBDA_RESULT_FILE, 'stdout.log', 'stderr.log'])
def test_task(args):
return 1 + 2
response = rpc.post_wait({
'type': 'CreateTask',
'task': {
'type': 'GLambda',
'options': {
'method': test_task,
'verification': {
'type': VerificationMethod.EXTERNALLY_VERIFIED
}
}
}
})
assert response['type'] == 'TaskCreatedEvent'
task_id = response['task_id']
response = rpc.poll(timeout=None)
assert response['type'] == 'VerificationRequired'
assert response['task_id'] == task_id
result = load_result(response['results'])
assert result['data'] == 3
assert 'error' not in result
response = rpc.post_wait({
'type': 'VerifyResults',
'task_id': response['task_id'],
'subtask_id': response['subtask_id'],
'verdict': SubtaskVerificationState.VERIFIED
})
assert response['type'] == 'TaskResults'
assert response['task_id'] == task_id
result = load_result(response['results'])
assert result['data'] == 3
assert 'error' not in result
rpc.post_wait({'type': 'Disconnect'})
def test_file_chunk_resource(remote):
rpc = create_rpc_component(remote=remote)
rpc.start()
testfile = 'testfile'
expected_results = set([GLAMBDA_RESULT_FILE, 'stdout.log', 'stderr.log'])
TESTSTRING = b'\xDA'
# Magic value taken from golemfactory/golem autobahn
chunk_size = 131072 * 4
with open(testfile, 'wb') as f:
for _ in range(chunk_size):
f.write(TESTSTRING)
def test_task(args):
with open(os.path.join('/golem/resources/', testfile), 'rb') as f:
content = f.read()
if not len(content) == chunk_size:
raise ValueError('Uploaded file size mismatch')
return True
_ = rpc.post_wait({
'type': 'CreateTask',
'task': {
'type': 'GLambda',
'options': {
'method': test_task
},
'resources': [os.path.abspath(testfile)]
}
})
results = rpc.poll(timeout=None)['results']
result_directory = os.path.split(results[0])[0]
assert set(os.path.basename(r) for r in results) == expected_results
with open(os.path.join(result_directory, GLAMBDA_RESULT_FILE), 'r') as f:
j = json.loads(f.read())
assert 'data' in j
assert j['data'] is True
rpc.post_wait({
'type': 'Disconnect'
})
def test_blender(remote):
c = create_rpc_component(remote=remote)
# Example assumes 'bmw27_cpu.blend' has been placed in your home directory
cube_blend_path = Path.home() / PurePath('bmw27_cpu.blend')
blender_dict = {
"type": "Blender",
# Give some unique name to the task
"name": "{}".format(str(uuid.uuid1())[:8]),
"timeout": "0:10:00",
"subtask_timeout": "0:09:50",
"subtasks_count": 1,
"bid": 1.0,
"resources": [
# Specify where the input can be found
cube_blend_path.as_posix()
],
"options": {
"output_path": Path.home().as_posix(),
"format": "PNG",
"resolution": [
400,
300
]
}
}
c.start()
_ = c.post_wait({
'type': 'CreateTask',
'task': blender_dict
})
results = c.poll(timeout=None)['results']
print(results)
# Tell RPCComponent to disconnect with remote Golem
c.post_wait({
'type': 'Disconnect'
})
def test_file_resource(remote):
rpc = create_rpc_component(remote=remote)
rpc.start()
testfile = 'testfile'
expected_results = set([GLAMBDA_RESULT_FILE, 'stdout.log', 'stderr.log'])
TESTSTRING = b'\xDA'
with open(testfile, 'wb') as f:
f.write(TESTSTRING)
def test_task(args):
with open(os.path.join('/golem/resources/', testfile), 'rb') as f:
if TESTSTRING != f.read():
raise ValueError('TESTSTRING does not match')
return True
_ = rpc.post_wait({
'type': 'CreateTask',
'task': {
'type': 'GLambda',
'options': {
'method': test_task
},
'resources': [os.path.abspath(testfile)]
}
})
results = rpc.poll(timeout=None)['results']
result_directory = os.path.split(results[0])[0]
assert set(os.path.basename(r) for r in results) == expected_results
with open(os.path.join(result_directory, GLAMBDA_RESULT_FILE), 'r') as f:
j = json.loads(f.read())
assert 'data' in j
assert j['data'] is True
rpc.post_wait({
'type': 'Disconnect'
})
def test_absolute_mapping_exception():
rpc = create_rpc_component()
rpc.remote = True
rpc.start()
def dummy_task(args):
return True
with pytest.raises(AssertionError):
_ = rpc.post_wait({
'type': 'CreateTask',
'task': {
'type': 'GLambda',
'options': {
'method': dummy_task
},
'resources_mapped': {
'tmpfile': '/usr/bin/echo'
}
}
})
def test_invalid_resource(remote):
rpc = create_rpc_component(remote=remote)
rpc.start()
# FIXME remove this ugly exception assertion when moving to test framework
def test_task(args):
pass
try:
rpc.post_wait({
'type': 'CreateTask',
'task': {
'options': {
'method': test_task
}
}
})['results']
except:
pass
else:
assert False
def test_big_file_upload():
src = 'test_big'
result = 'result_big'
with open(src, 'wb') as f:
# 4 GB
f.seek(4 * 1024 * 1024 * 1024 - 1)
f.write(b"\0")
rpc_component = create_rpc_component()
rpc_component.start()
transfer_mgr = TransferManager(rpc_component)
transfer_mgr.upload(src, 'tmp_big_file')
transfer_mgr.download('tmp_big_file', result)
src_size = os.stat(src).st_size
result_size = os.stat(result).st_size
os.remove(src)
os.remove(result)
assert src_size == result_size
def test_big_file_output(remote):
rpc = create_rpc_component(remote=remote)
rpc.start()
expected_results = set(
[GLAMBDA_RESULT_FILE, 'stdout.log', 'stderr.log', 'result.bin'])
FILE_SIZE = 50 * 1024 * 1024
TESTBYTE = b'\xDA'
def test_task(args):
with open('/golem/output/result.bin', 'wb') as f:
# Could use os.truncate but result is platform dependent
# if the resulting file is bigger than before truncation
# which is the case here
for _ in range(FILE_SIZE):
f.write(TESTBYTE)
_ = rpc.post_wait({
'type': 'CreateTask',
'task': {
'type': 'GLambda',
'options': {
'method': test_task,
'outputs': list(expected_results)
},
},
})
results = rpc.poll(timeout=None)['results']
result_directory = os.path.split(results[0])[0]
assert set(os.path.basename(r) for r in results) == expected_results
assert os.stat(
os.path.join(result_directory, 'result.bin')
).st_size == FILE_SIZE
rpc.post_wait({
'type': 'Disconnect'
})
def test_directory_resource_task(remote):
rpc = create_rpc_component(remote=remote)
rpc.start()
TESTSTRING = b'\xDA'
expected_results = set([GLAMBDA_RESULT_FILE, 'stdout.log', 'stderr.log'])
tmpd = PurePath(tempfile.mkdtemp())
tmpd_basename = tmpd.name
os.mkdir(os.path.join(tmpd, 'subdir'))
with open(os.path.join(tmpd, 'tmpfile'), 'wb') as f:
f.write(TESTSTRING)
with open(os.path.join(tmpd, 'subdir', 'subdir_tempfile'), 'wb') as f:
f.write(TESTSTRING)
def test_task(args):
golem_res = Path('/golem/resources')
golem_tmpd = golem_res / Path(tmpd_basename)
tmpd_file = golem_tmpd / 'tmpfile'
tmpd_subdir = golem_tmpd / 'subdir'
tmpd_subdir_file = tmpd_subdir / 'subdir_tempfile'
if not tmpd_file.is_file():
raise AssertionError(str(tmpd_file) + ' is not a file')
with open(tmpd_file, 'rb') as f:
if not TESTSTRING == f.read():
raise AssertionError(
TESTSTRING + ' does not match ' + str(tmpd_file))
if not tmpd_subdir.is_dir():
raise AssertionError(tmpd_subdir + ' is not a directory')
if not tmpd_subdir_file.is_file():
raise AssertionError(tmpd_subdir_file + ' is not a file')
with open(tmpd_subdir_file, 'rb') as f:
if not TESTSTRING == f.read():
raise AssertionError(
TESTSTRING + ' doest not match ' + str(tmpd_subdir_file))
return True
_ = rpc.post_wait({
'type': 'CreateTask',
'task': {
'type': 'GLambda',
'options': {
'method': test_task
},
'resources': [os.path.abspath(tmpd)]
}
})
results = rpc.poll(timeout=None)['results']
result_directory = os.path.split(results[0])[0]
assert set(os.path.basename(r) for r in results) == expected_results
with open(os.path.join(result_directory, GLAMBDA_RESULT_FILE), 'r') as f:
j = json.loads(f.read())
assert 'data' in j
assert j['data'] is True
shutil.rmtree(tmpd)
rpc.post_wait({
'type': 'Disconnect'
})
