def _create_tagged_worksheet(self, name, title):
uuid = run_command([self._cl, 'new', name])
run_command([self._cl, 'work', name])
run_command([
self._cl, 'wedit',
'--tag=%s' % SampleWorksheet.TAG,
'--title=%s' % title
])
return uuid
def _test_many_worksheet_copies(self):
# Initialize a worksheet with 10 bundles to be replicated
worksheet_uuid = self._set_worksheet('many_worksheet_copies')
file = TestFile('copy_file', 1)
for _ in range(10):
self._run_bundle([self._cl, 'upload', file.name()])
file.delete()
# Create many worksheets with current worksheet's content copied over
for _ in range(self._args.create_worksheet_count):
other_worksheet_uuid = self._set_worksheet('other_worksheet_copy')
run_command([self._cl, 'wadd', worksheet_uuid, other_worksheet_uuid])
def _create_sample_worksheet(self):
# Write out the contents to a temporary file
with open(SampleWorksheet._WORKSHEET_FILE_PATH, 'w') as file:
file.write('\n'.join(self._content))
# Create the main worksheet used for stress testing the frontend
title = '{} Worksheet'.format(self._description[0].upper() + self._description[1:])
self._create_tagged_worksheet(self._worksheet_name, title)
# Replace the content of the current worksheet with the temporary file's content. Delete the temp file after.
run_command([self._cl, 'wedit', '--file=' + SampleWorksheet._WORKSHEET_FILE_PATH])
os.remove(SampleWorksheet._WORKSHEET_FILE_PATH)
print('Deleted worksheet file at {}.'.format(SampleWorksheet._WORKSHEET_FILE_PATH))
def _create_dependencies(self):
if self._preview_mode:
# When in preview mode, search for existing bundles and worksheets instead of creating new ones
random_worksheets = run_command([
self._cl, 'wsearch',
'.limit=%d' % self._entities_count, '--uuid-only'
]).split('\n')
self._valid_worksheets = random_worksheets
self._private_worksheets = random_worksheets
random_bundles = run_command([
self._cl,
'search',
'state=ready',
'.limit=%d' % self._entities_count,
'--uuid-only',
]).split('\n')
self._valid_bundles = random_bundles
self._private_bundles = random_bundles
return
self._valid_worksheets = []
self._private_worksheets = []
self._valid_bundles = []
self._private_bundles = []
for _ in range(self._entities_count):
# Create valid worksheets with a bundle each for the sample worksheet to reference
id = self._random_id()
name = 'valid_worksheet_%s' % id
title = 'Other Worksheet %s' % id
self._valid_worksheets.append(
self._create_tagged_worksheet(name, title))
uuid = run_command([
self._cl,
'run',
'--request-memory',
'10m',
'--request-docker-image',
'python:3.6.10-slim-buster',
'echo codalab rules!',
'--tags=%s' % SampleWorksheet.TAG,
])
self._valid_bundles.append(uuid)
# Create a valid private worksheet and a bundle each
name = 'valid_private_worksheet_%s' % id
title = 'Other Private Worksheet %s' % id
uuid = self._create_tagged_worksheet(name, title)
run_command([self._cl, 'wperm', uuid, 'public', 'none'])
self._private_worksheets.append(uuid)
uuid = run_command([
self._cl,
'run',
'--request-memory',
'10m',
'--request-docker-image',
'python:3.6.10-slim-buster',
'echo private run',
'--tags=%s' % SampleWorksheet.TAG,
])
run_command([self._cl, 'perm', uuid, 'public', 'none'])
self._private_bundles.append(uuid)
def _test_large_bundle_upload(self) -> None:
self._set_worksheet('large_bundle_upload')
large_file: TestFile = TestFile('large_file',
self._args.large_file_size_gb * 1000)
dependency_uuid: str = self._run_bundle(
[self._cl, 'upload', large_file.name()])
large_file.delete()
uuid: str = self._run_bundle([
self._cl,
'run',
'large_dependency:{}'.format(dependency_uuid),
'wc -c large_dependency',
])
# Wait for the run to finish before cleaning up the dependency
run_command([cl, 'wait', uuid])
def create(self):
# Skip creating a sample worksheet if one already exists
worksheet_uuids = run_command(
[self._cl, 'wsearch', self._worksheet_name, '--uuid-only']
).split('\n')
if re.match(SampleWorksheet._FULL_UUID_REGEX, worksheet_uuids[0]):
print(
'There is already an existing {} with UUID {}. Skipping creating a sample worksheet...'.format(
self._worksheet_name, worksheet_uuids[0]
)
)
return
print('Creating a {} worksheet...'.format(self._description))
self._create_dependencies()
self._add_introduction()
self._add_worksheet_references()
self._add_bundle_references()
self._add_schemas()
self._add_display_modes()
self._add_search()
self._add_invalid_directives()
self._add_rendering_logic()
self._create_sample_worksheet()
print('Done.')
def _test_large_bundle(self):
self._set_worksheet('large_bundles')
# Set this to larger than the max memory on the system to test that data is being
# streamed when the large bundle is being used as a dependency.
large_file = TestFile('large_file',
self._args.large_file_size_gb * 1000)
dependency_uuid = self._run_bundle(
[self._cl, 'upload', large_file.name()])
large_file.delete()
uuid = self._run_bundle([
self._cl,
'run',
'large_dependency:{}'.format(dependency_uuid),
'wc -c large_dependency',
])
# Wait for the run to finish before cleaning up the dependency
run_command([cl, 'wait', uuid])
def _test_large_bundle_result(self) -> None:
def create_large_file_in_bundle(large_file_size_gb: int) -> TestFile:
code: str = 'with open("largefile", "wb") as out:\n\tout.truncate({} * 1024 * 1024 * 1024)'.format(
large_file_size_gb)
return TestFile('large_dependency.py', content=code)
self._set_worksheet('large_bundle_result')
file: TestFile = create_large_file_in_bundle(
self._args.large_dependency_size_gb)
self._run_bundle([self._cl, 'upload', file.name()])
file.delete()
dependency_uuid: str = self._run_bundle(
[self._cl, 'run', ':' + file.name(), 'python ' + file.name()])
uuid: str = self._run_bundle([
self._cl,
'run',
'large_bundle:{}'.format(dependency_uuid),
'wc -c large_bundle/largefile',
])
# Wait for the run to finish before cleaning up the dependency
run_command([cl, 'wait', uuid])
def test_print(self):
output_lines = run_command([self._cl, 'print',
self._worksheet_name]).split('\n')
has_error = False
for i in range(len(self._expected_lines)):
if not re.match(self._expected_lines[i], output_lines[i]):
has_error = True
# Output mismatch message in red
print(
'\033[91mMISMATCH! line: {} pattern: {} output: {}\033[0m'.
format(i + 1, self._expected_lines[i], output_lines[i]))
assert not has_error
print('Finished validating content of the sample worksheet...')
print('Success.')
def _search_bundles(self, query):
if self._preview_mode:
# When in preview mode, just return the cached UUIDs of valid bundles instead of performing a new search
return self._valid_bundles
bundles = run_command([
self._cl,
'search',
query,
'.limit=%d' % self._entities_count,
'created=.sort-',
'--uuid-only',
])
if not bundles:
return []
return bundles.split('\n')
def test_print(self):
self._wait_for_bundles_to_finish()
print('\n\nValidating output of cl print {}...'.format(
self._worksheet_name))
output_lines = run_command([self._cl, 'print',
self._worksheet_name]).split('\n')
has_error = False
for i in range(len(self._expected_lines)):
line = str(i + 1).zfill(5)
if re.match(self._expected_lines[i], output_lines[i]):
print('\x1b[1;34m{}| {}\x1b[0m'.format(line, output_lines[i]))
else:
has_error = True
# Output mismatch message in red
print('\033[91m{}| {} EXPECTED: {} \033[0m'.format(
line, output_lines[i], self._expected_lines[i]))
assert not has_error
print('Finished validating content of the sample worksheet...')
print('Success.')
def _run_bundle(self, args):
args.append('--tags=%s' % StressTestRunner._TAG)
return run_command(args)
def _set_worksheet(self, run_name):
worksheet_name = self._create_worksheet_name(run_name)
uuid = run_command([self._cl, 'new', worksheet_name])
run_command([self._cl, 'work', worksheet_name])
run_command([self._cl, 'wedit', '--tag=%s' % StressTestRunner._TAG])
return uuid
def _heartbeat_cl_commands(cl):
run_command([cl, 'search', 'state=failed', 'created=.sort-'])
run_command([cl, 'workers'])
def _run_bundle(self, args, expected_exit_code=0):
args.append('--tags=%s' % StressTestRunner._TAG)
return run_command(args, expected_exit_code)
def _simple_run(cl):
run_command([cl, 'run', 'echo stress testing...', '--tags=%s' % StressTestRunner._TAG])
def _search_failed_runs(cl):
run_command([cl, 'search', 'state=failed', 'created=.sort-'])
def _wait_for_bundles_to_finish(self):
if self._valid_bundles:
for bundle in self._valid_bundles:
run_command([self._cl, 'wait', bundle])
print('Bundle {} is finished.'.format(bundle))