def test_task_result_output(remote): rpc = create_rpc_component(remote=remote) rpc.start() TESTSTRING = 'test' expected_results = set([GLAMBDA_RESULT_FILE, 'stdout.log', 'stderr.log']) def test_task(args): return TESTSTRING _ = rpc.post_wait({ 'type': 'CreateTask', 'task': { 'type': 'GLambda', 'options': { 'method': test_task } } }) results = rpc.poll(timeout=None)['results'] result_directory = os.path.split(results[0])[0] assert set(os.path.basename(r) for r in results) == expected_results with open(os.path.join(result_directory, GLAMBDA_RESULT_FILE), 'r') as f: assert f.read() == json.dumps({'data': TESTSTRING}) rpc.post_wait({ 'type': 'Disconnect' })
def test_task_timeout(remote): rpc = create_rpc_component(remote=remote) rpc.start() # FIXME remove exception assertion when moving to test framework def test_task(args): import time time.sleep(5.0) try: _ = rpc.post_wait({ 'type': 'CreateTask', 'task': { 'type': 'GLambda', 'options': { 'method': test_task }, 'timeout': '00:00:00' } }) rpc.poll(timeout=None) except RuntimeError as e: pass else: assert False
def test_file_output(remote): rpc = create_rpc_component(remote=remote) rpc.start() expected_results = set( [GLAMBDA_RESULT_FILE, 'stdout.log', 'stderr.log', 'testfile']) TESTSTRING = b'\xDA' def test_task(args): import os with open('/golem/output/testfile', 'wb') as f: f.write(TESTSTRING) _ = rpc.post_wait({ 'type': 'CreateTask', 'task': { 'type': 'GLambda', 'options': { 'method': test_task, 'outputs': list(expected_results) } } }) results = rpc.poll(timeout=None)['results'] result_directory = os.path.split(results[0])[0] assert set(os.path.basename(r) for r in results) == expected_results with open(os.path.join(result_directory, 'testfile'), 'rb') as f: assert f.read() == TESTSTRING rpc.post_wait({ 'type': 'Disconnect' })
def test_implicit_no_verification(): rpc = create_rpc_component() rpc.start() TEST_STRING = 'test' expected_results = set([GLAMBDA_RESULT_FILE, 'stdout.log', 'stderr.log']) def test_task(args): return 1 + 2 response = rpc.post_wait({ 'type': 'CreateTask', 'task': { 'type': 'GLambda', 'options': { 'method': test_task } } }) assert response['type'] == 'TaskCreatedEvent' task_id = response['task_id'] response = rpc.poll(timeout=None) assert response['type'] == 'TaskResults' assert response['task_id'] == task_id result = load_result(response['results']) # Intentionally do not assert 'data' field assert 'error' not in result assert result['data'] == 3 rpc.post_wait({'type': 'Disconnect'})
def test_raise_exception(remote): rpc = create_rpc_component(remote=remote) rpc.start() TEST_STRING = 'test' expected_results = set([GLAMBDA_RESULT_FILE, 'stdout.log', 'stderr.log']) def test_task(args): raise RuntimeError(TEST_STRING) _ = rpc.post_wait({ 'type': 'CreateTask', 'task': { 'type': 'GLambda', 'options': { 'method': test_task } } }) results = rpc.poll(timeout=None)['results'] result_directory = os.path.split(results[0])[0] assert set(os.path.basename(r) for r in results) == expected_results with open(os.path.join(result_directory, GLAMBDA_RESULT_FILE), 'r') as f: result_json = json.loads(f.read()) assert 'error' in result_json assert result_json['error'] == TEST_STRING assert 'data' not in result_json rpc.post_wait({'type': 'Disconnect'})
def test_task_too_big(remote): rpc = create_rpc_component(remote=remote) rpc.start() # FIXME remove exception assertion when moving to test framework def test_task(args): import time time.sleep(5.0) try: _ = rpc.post_wait({ 'type': 'CreateTask', 'task': { 'type': 'GLambda', 'options': { 'method': test_task }, 'timeout': '00:00:00', # This must be adjusted to Golem:golem/tools/uploadcontroller.py changes 'dummy_data': [0 for _ in range(20 * 1024 * 1024)] } }) rpc.poll(timeout=None) except ValueError as e: pass else: assert False
def test_non_serializable_task_result(remote): rpc = create_rpc_component(remote=remote) rpc.start() expected_results = set([GLAMBDA_RESULT_FILE, 'stdout.log', 'stderr.log']) def test_task(args): return lambda x: print(x) rpc.post({ 'type': 'CreateTask', 'task': { 'type': 'GLambda', 'options': { 'method': test_task }, 'timeout': '00:10:00', } }) _ = rpc.poll(timeout=None) results = rpc.poll(timeout=None)['results'] result_directory = os.path.split(results[0])[0] assert set(os.path.basename(r) for r in results) == expected_results with open(os.path.join(result_directory, GLAMBDA_RESULT_FILE), 'r') as f: result_json = json.loads(f.read()) assert 'error' in result_json assert 'data' not in result_json rpc.post_wait({'type': 'Disconnect'})
def test_empty_resource(remote): rpc = create_rpc_component(remote=remote) rpc.start() expected_results = set([GLAMBDA_RESULT_FILE, 'stdout.log', 'stderr.log']) def test_task(args): pass _ = rpc.post_wait({ 'type': 'CreateTask', 'task': { 'type': 'GLambda', 'options': { 'method': test_task } } }) results = rpc.poll(timeout=None)['results'] assert set(os.path.basename(r) for r in results) == expected_results result_directory = os.path.split(results[0])[0] rpc.post_wait({'type': 'Disconnect'})
def test_empty_mapping(): rpc = create_rpc_component() rpc.remote = True rpc.start() expected_results = set([GLAMBDA_RESULT_FILE, 'stdout.log', 'stderr.log']) def dummy_task(args): return True _ = rpc.post_wait({ 'type': 'CreateTask', 'task': { 'type': 'GLambda', 'options': { 'method': dummy_task } } }) results = rpc.poll(timeout=None)['results'] result_directory = os.path.split(results[0])[0] assert set(os.path.basename(r) for r in results) == expected_results with open(os.path.join(result_directory, GLAMBDA_RESULT_FILE), 'r') as f: j = json.loads(f.read()) assert 'data' in j assert j['data'] is True
def test_dir_nested_mapping(): rpc = create_rpc_component() rpc.remote = True rpc.start() expected_results = set([GLAMBDA_RESULT_FILE, 'stdout.log', 'stderr.log']) def dummy_task(args): import os resources = os.listdir('/golem/resources/foo/baz') if 'tmpfile' not in resources: return False if 'bar' not in resources: return False resources = os.listdir('/golem/resources/foo/baz/bar') if 'tmpfile' not in resources: return False return True os.mkdir('foo') os.mkdir('foo/bar') with open('foo/tmpfile', 'wb') as f: f.write(b'\xDA') with open('foo/bar/tmpfile', 'wb') as f: f.write(b'\xDA') _ = rpc.post_wait({ 'type': 'CreateTask', 'task': { 'type': 'GLambda', 'options': { 'method': dummy_task }, 'resources_mapped': { 'foo': 'foo/baz', } } }) shutil.rmtree('foo') results = rpc.poll(timeout=None)['results'] result_directory = os.path.split(results[0])[0] assert set(os.path.basename(r) for r in results) == expected_results with open(os.path.join(result_directory, GLAMBDA_RESULT_FILE), 'r') as f: j = json.loads(f.read()) assert 'data' in j assert j['data'] is True
def test_transfer_manager(): src = '/usr/bin/snap' result = 'tmp2' rpc_component = create_rpc_component() rpc_component.start() transfer_mgr = TransferManager(rpc_component) transfer_mgr.upload(src, 'tmp') transfer_mgr.download('tmp', result) src_size = os.stat(src).st_size result_size = os.stat(result).st_size os.remove(result) assert src_size == result_size
def test_successful_verification(): rpc = create_rpc_component() rpc.start() TEST_STRING = 'test' expected_results = set([GLAMBDA_RESULT_FILE, 'stdout.log', 'stderr.log']) def test_task(args): return 1 + 2 response = rpc.post_wait({ 'type': 'CreateTask', 'task': { 'type': 'GLambda', 'options': { 'method': test_task, 'verification': { 'type': VerificationMethod.EXTERNALLY_VERIFIED } } } }) assert response['type'] == 'TaskCreatedEvent' task_id = response['task_id'] response = rpc.poll(timeout=None) assert response['type'] == 'VerificationRequired' assert response['task_id'] == task_id result = load_result(response['results']) assert result['data'] == 3 assert 'error' not in result response = rpc.post_wait({ 'type': 'VerifyResults', 'task_id': response['task_id'], 'subtask_id': response['subtask_id'], 'verdict': SubtaskVerificationState.VERIFIED }) assert response['type'] == 'TaskResults' assert response['task_id'] == task_id result = load_result(response['results']) assert result['data'] == 3 assert 'error' not in result rpc.post_wait({'type': 'Disconnect'})
def test_file_chunk_resource(remote): rpc = create_rpc_component(remote=remote) rpc.start() testfile = 'testfile' expected_results = set([GLAMBDA_RESULT_FILE, 'stdout.log', 'stderr.log']) TESTSTRING = b'\xDA' # Magic value taken from golemfactory/golem autobahn chunk_size = 131072 * 4 with open(testfile, 'wb') as f: for _ in range(chunk_size): f.write(TESTSTRING) def test_task(args): with open(os.path.join('/golem/resources/', testfile), 'rb') as f: content = f.read() if not len(content) == chunk_size: raise ValueError('Uploaded file size mismatch') return True _ = rpc.post_wait({ 'type': 'CreateTask', 'task': { 'type': 'GLambda', 'options': { 'method': test_task }, 'resources': [os.path.abspath(testfile)] } }) results = rpc.poll(timeout=None)['results'] result_directory = os.path.split(results[0])[0] assert set(os.path.basename(r) for r in results) == expected_results with open(os.path.join(result_directory, GLAMBDA_RESULT_FILE), 'r') as f: j = json.loads(f.read()) assert 'data' in j assert j['data'] is True rpc.post_wait({ 'type': 'Disconnect' })
def test_blender(remote): c = create_rpc_component(remote=remote) # Example assumes 'bmw27_cpu.blend' has been placed in your home directory cube_blend_path = Path.home() / PurePath('bmw27_cpu.blend') blender_dict = { "type": "Blender", # Give some unique name to the task "name": "{}".format(str(uuid.uuid1())[:8]), "timeout": "0:10:00", "subtask_timeout": "0:09:50", "subtasks_count": 1, "bid": 1.0, "resources": [ # Specify where the input can be found cube_blend_path.as_posix() ], "options": { "output_path": Path.home().as_posix(), "format": "PNG", "resolution": [ 400, 300 ] } } c.start() _ = c.post_wait({ 'type': 'CreateTask', 'task': blender_dict }) results = c.poll(timeout=None)['results'] print(results) # Tell RPCComponent to disconnect with remote Golem c.post_wait({ 'type': 'Disconnect' })
def test_file_resource(remote): rpc = create_rpc_component(remote=remote) rpc.start() testfile = 'testfile' expected_results = set([GLAMBDA_RESULT_FILE, 'stdout.log', 'stderr.log']) TESTSTRING = b'\xDA' with open(testfile, 'wb') as f: f.write(TESTSTRING) def test_task(args): with open(os.path.join('/golem/resources/', testfile), 'rb') as f: if TESTSTRING != f.read(): raise ValueError('TESTSTRING does not match') return True _ = rpc.post_wait({ 'type': 'CreateTask', 'task': { 'type': 'GLambda', 'options': { 'method': test_task }, 'resources': [os.path.abspath(testfile)] } }) results = rpc.poll(timeout=None)['results'] result_directory = os.path.split(results[0])[0] assert set(os.path.basename(r) for r in results) == expected_results with open(os.path.join(result_directory, GLAMBDA_RESULT_FILE), 'r') as f: j = json.loads(f.read()) assert 'data' in j assert j['data'] is True rpc.post_wait({ 'type': 'Disconnect' })
def test_absolute_mapping_exception(): rpc = create_rpc_component() rpc.remote = True rpc.start() def dummy_task(args): return True with pytest.raises(AssertionError): _ = rpc.post_wait({ 'type': 'CreateTask', 'task': { 'type': 'GLambda', 'options': { 'method': dummy_task }, 'resources_mapped': { 'tmpfile': '/usr/bin/echo' } } })
def test_invalid_resource(remote): rpc = create_rpc_component(remote=remote) rpc.start() # FIXME remove this ugly exception assertion when moving to test framework def test_task(args): pass try: rpc.post_wait({ 'type': 'CreateTask', 'task': { 'options': { 'method': test_task } } })['results'] except: pass else: assert False
def test_big_file_upload(): src = 'test_big' result = 'result_big' with open(src, 'wb') as f: # 4 GB f.seek(4 * 1024 * 1024 * 1024 - 1) f.write(b"\0") rpc_component = create_rpc_component() rpc_component.start() transfer_mgr = TransferManager(rpc_component) transfer_mgr.upload(src, 'tmp_big_file') transfer_mgr.download('tmp_big_file', result) src_size = os.stat(src).st_size result_size = os.stat(result).st_size os.remove(src) os.remove(result) assert src_size == result_size
def test_big_file_output(remote): rpc = create_rpc_component(remote=remote) rpc.start() expected_results = set( [GLAMBDA_RESULT_FILE, 'stdout.log', 'stderr.log', 'result.bin']) FILE_SIZE = 50 * 1024 * 1024 TESTBYTE = b'\xDA' def test_task(args): with open('/golem/output/result.bin', 'wb') as f: # Could use os.truncate but result is platform dependent # if the resulting file is bigger than before truncation # which is the case here for _ in range(FILE_SIZE): f.write(TESTBYTE) _ = rpc.post_wait({ 'type': 'CreateTask', 'task': { 'type': 'GLambda', 'options': { 'method': test_task, 'outputs': list(expected_results) }, }, }) results = rpc.poll(timeout=None)['results'] result_directory = os.path.split(results[0])[0] assert set(os.path.basename(r) for r in results) == expected_results assert os.stat( os.path.join(result_directory, 'result.bin') ).st_size == FILE_SIZE rpc.post_wait({ 'type': 'Disconnect' })
def test_directory_resource_task(remote): rpc = create_rpc_component(remote=remote) rpc.start() TESTSTRING = b'\xDA' expected_results = set([GLAMBDA_RESULT_FILE, 'stdout.log', 'stderr.log']) tmpd = PurePath(tempfile.mkdtemp()) tmpd_basename = tmpd.name os.mkdir(os.path.join(tmpd, 'subdir')) with open(os.path.join(tmpd, 'tmpfile'), 'wb') as f: f.write(TESTSTRING) with open(os.path.join(tmpd, 'subdir', 'subdir_tempfile'), 'wb') as f: f.write(TESTSTRING) def test_task(args): golem_res = Path('/golem/resources') golem_tmpd = golem_res / Path(tmpd_basename) tmpd_file = golem_tmpd / 'tmpfile' tmpd_subdir = golem_tmpd / 'subdir' tmpd_subdir_file = tmpd_subdir / 'subdir_tempfile' if not tmpd_file.is_file(): raise AssertionError(str(tmpd_file) + ' is not a file') with open(tmpd_file, 'rb') as f: if not TESTSTRING == f.read(): raise AssertionError( TESTSTRING + ' does not match ' + str(tmpd_file)) if not tmpd_subdir.is_dir(): raise AssertionError(tmpd_subdir + ' is not a directory') if not tmpd_subdir_file.is_file(): raise AssertionError(tmpd_subdir_file + ' is not a file') with open(tmpd_subdir_file, 'rb') as f: if not TESTSTRING == f.read(): raise AssertionError( TESTSTRING + ' doest not match ' + str(tmpd_subdir_file)) return True _ = rpc.post_wait({ 'type': 'CreateTask', 'task': { 'type': 'GLambda', 'options': { 'method': test_task }, 'resources': [os.path.abspath(tmpd)] } }) results = rpc.poll(timeout=None)['results'] result_directory = os.path.split(results[0])[0] assert set(os.path.basename(r) for r in results) == expected_results with open(os.path.join(result_directory, GLAMBDA_RESULT_FILE), 'r') as f: j = json.loads(f.read()) assert 'data' in j assert j['data'] is True shutil.rmtree(tmpd) rpc.post_wait({ 'type': 'Disconnect' })