def test_ligo_user_with_non_gwosc(self):
# Test that LIGO users can make jobs with proprietary channels
# Now if the channels are proprietary, the ligo user should be able to create jobs
self.client.authenticate(self.user, is_ligo=True)
token = get_upload_token(self.client).data['generateBilbyJobUploadToken']['token']
self.params['input']['uploadToken'] = token
for channel_dict in [
{'H1': 'GDS-CALIB_STRAIN', 'L1': 'GWOSC', 'V1': 'GWOSC'},
{'H1': 'GWOSC', 'L1': 'GDS-CALIB_STRAIN', 'V1': 'GWOSC'},
{'H1': 'GWOSC', 'L1': 'GWOSC', 'V1': 'Hrec_hoft_16384Hz'},
]:
ini_string = create_test_ini_string({
'label': 'testjob',
'n-simulation': 0,
'channel-dict': channel_dict
}, complete=True)
self.params['input']['jobFile'] = SimpleUploadedFile(
name='test.tar.gz',
content=create_test_upload_data(ini_string, 'testjob'),
content_type='application/gzip'
)
response = self.client.execute(self.mutation, self.params)
self.assertTrue('jobId' in response.data['uploadBilbyJob']['result'])
# Check that the job is marked as proprietary
job = BilbyJob.objects.all().last()
self.assertTrue(job.is_ligo_job)
job.delete()
def test_ligo_user_with_gwosc(self):
# This test checks that a non LIGO user can still create non LIGO jobs
self.client.authenticate(self.user, is_ligo=True)
token = get_upload_token(self.client).data['generateBilbyJobUploadToken']['token']
self.params['input']['uploadToken'] = token
ini_string = create_test_ini_string(config_dict={
'label': 'testjob',
"n-simulation": 0,
"channel-dict": {'H1': 'GWOSC', 'L1': 'GWOSC'}
}, complete=True)
self.params['input']['jobFile'] = SimpleUploadedFile(
name='test.tar.gz',
content=create_test_upload_data(ini_string, 'testjob'),
content_type='application/gzip'
)
response = self.client.execute(self.mutation, self.params)
self.assertDictEqual(
self.expected_one, response.data, "create bilbyJob mutation returned unexpected data."
)
# Check that the job is marked as not proprietary
self.assertFalse(BilbyJob.objects.all().last().is_ligo_job)
def test_ini_job_submission_supporting_file_calibration3(self):
test_ini_string = create_test_ini_string({
'label':
"Test Name",
'detectors':
"['H1']",
'spline-calibration-envelope-dict':
'{L1:./supporting_files/calib/L1-calib.dat, '
'V1:./supporting_files/calib/V1-calib.dat, '
'H1:./supporting_files/calib/H1-calib.dat}'
})
self.mock_ini_job_submission_with_supporting_files(
test_ini_string,
[[
'L1', './supporting_files/calib/L1-calib.dat',
SupportingFile.CALIBRATION
],
[
'V1', './supporting_files/calib/V1-calib.dat',
SupportingFile.CALIBRATION
],
[
'H1', './supporting_files/calib/H1-calib.dat',
SupportingFile.CALIBRATION
]], 'spline_calibration_envelope_dict')
def test_download_supporting_files_valid_token(self):
test_ini_string = create_test_ini_string(
{
'label': "Test Name",
'detectors': "['H1']",
'psd-dict': '{V1:./supporting_files/psd/V1-psd.dat}'
}, )
self.mock_ini_job_submission_with_supporting_files(
test_ini_string,
[['V1', './supporting_files/psd/V1-psd.dat', SupportingFile.PSD]],
'psd_dict')
supporting_file = SupportingFile.objects.last()
response = self.http_client.get(
f'{reverse(viewname="file_download")}?fileId={supporting_file.download_token}'
)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.headers['Content-Type'],
'application/octet-stream')
self.assertEqual(response.headers['Content-Disposition'],
'inline; filename="V1-psd.dat"')
content = b''.join(list(response))
# Get the supporting file path
file_path = Path(settings.SUPPORTING_FILE_UPLOAD_DIR) / str(
supporting_file.job.id) / str(supporting_file.id)
with open(str(file_path), 'rb') as f:
self.assertEqual(content, f.read())
def test_ini_job_submission_supporting_file_psd1(self):
test_ini_string = create_test_ini_string(
{
'label': "Test Name",
'detectors': "['H1']",
'psd-dict': '{V1:./supporting_files/psd/V1-psd.dat}'
}, )
self.mock_ini_job_submission_with_supporting_files(
test_ini_string,
[['V1', './supporting_files/psd/V1-psd.dat', SupportingFile.PSD]],
'psd_dict')
def test_ini_job_submission_supporting_file_gps(self):
test_ini_string = create_test_ini_string({
'label':
"Test Name",
'detectors':
"['H1']",
'gps-file':
'./supporting_files/gps/gps.dat'
})
self.mock_ini_job_submission_with_supporting_files(
test_ini_string,
[[None, './supporting_files/gps/gps.dat', SupportingFile.GPS]],
'gps_file')
def test_job_upload_unauthorised_user(self):
# Test user not logged in
self.client.authenticate(None)
response = get_upload_token(self.client)
self.assertEqual(response.errors[0].message, 'You do not have permission to perform this action')
self.client.authenticate(self.user, is_ligo=True)
test_name = "Test Name"
test_description = "Test Description"
test_private = False
test_ini_string = create_test_ini_string(
{
'label': test_name,
}
)
test_file = SimpleUploadedFile(
name='test.tar.gz',
content=create_test_upload_data(test_ini_string, test_name),
content_type='application/gzip'
)
test_input = {
"input": {
"uploadToken": uuid.uuid4(),
"details": {
"name": test_name,
"description": test_description,
"private": test_private
},
"jobFile": test_file
}
}
response = self.client.execute(
self.mutation,
test_input
)
self.assertEqual(response.errors[0].message, 'Job upload token is invalid or expired.')
self.assertDictEqual(
{'uploadBilbyJob': None},
response.data
)
self.assertFalse(BilbyJob.objects.all().exists())
def setUp(self):
self.user = User.objects.create(username="******",
first_name="buffy",
last_name="summers")
self.client.authenticate(self.user)
token = get_upload_token(
self.client).data['generateBilbyJobUploadToken']['token']
# Create a new uploaded bilby job
test_name = "myjob"
test_description = "Test Description"
test_private = False
test_ini_string = create_test_ini_string(
{
'label': test_name,
'detectors': "['H1']",
'outdir': './'
}, True)
test_file = SimpleUploadedFile(name='test.tar.gz',
content=create_test_upload_data(
test_ini_string, test_name),
content_type='application/gzip')
test_input = {
"input": {
"uploadToken": token,
"details": {
"description": test_description,
"private": test_private
},
"jobFile": test_file
}
}
self.client.execute(
"""
mutation JobUploadMutation($input: UploadBilbyJobMutationInput!) {
uploadBilbyJob(input: $input) {
result {
jobId
}
}
}
""", test_input)
self.job = BilbyJob.objects.all().last()
def test_ini_job_submission_supporting_file_timeslide(self):
test_ini_string = create_test_ini_string({
'label':
"Test Name",
'detectors':
"['H1']",
'timeslide-file':
'./supporting_files/timeslide/timeslide.dat'
})
self.mock_ini_job_submission_with_supporting_files(
test_ini_string, [[
None, './supporting_files/timeslide/timeslide.dat',
SupportingFile.TIME_SLIDE
]], 'timeslide_file')
def test_ini_job_submission_supporting_file_injection(self):
test_ini_string = create_test_ini_string({
'label':
"Test Name",
'detectors':
"['H1']",
'injection-file':
'./supporting_files/injection/injection.dat'
})
self.mock_ini_job_submission_with_supporting_files(
test_ini_string, [[
None, './supporting_files/injection/injection.dat',
SupportingFile.INJECTION
]], 'injection_file')
def test_ini_job_submission_supporting_file_prior(self):
test_ini_string = create_test_ini_string({
'label':
"Test Name",
'detectors':
"['H1']",
'prior-file':
'./supporting_files/prior/myprior.prior'
})
self.mock_ini_job_submission_with_supporting_files(
test_ini_string, [[
None, './supporting_files/prior/myprior.prior',
SupportingFile.PRIOR
]], 'prior_file')
def test_ini_job_submission_supporting_file_numerical_relativity_file(
self):
test_ini_string = create_test_ini_string({
'label':
"Test Name",
'detectors':
"['H1']",
'numerical-relativity-file':
'./supporting_files/nrf/nrf.dat'
})
self.mock_ini_job_submission_with_supporting_files(
test_ini_string, [[
None, './supporting_files/nrf/nrf.dat',
SupportingFile.NUMERICAL_RELATIVITY
]], 'numerical_relativity_file')
def test_job_upload_missing_data(self):
for missing_dir in ['data', 'result', 'results_page']:
token = get_upload_token(self.client).data['generateBilbyJobUploadToken']['token']
test_name = missing_dir
test_description = f"{missing_dir} Description"
test_private = False
test_ini_string = create_test_ini_string(
{
'label': test_name,
},
True
)
test_file = SimpleUploadedFile(
name='test.tar.gz',
content=create_test_upload_data(test_ini_string, test_name, **{f'include_{missing_dir}': False}),
content_type='application/gzip'
)
test_input = {
"input": {
"uploadToken": token,
"details": {
"description": test_description,
"private": test_private
},
"jobFile": test_file
}
}
response = self.client.execute(
self.mutation,
test_input
)
self.assertEqual(
f"Invalid directory structure, expected directory ./{missing_dir} to exist.",
response.errors[0].message
)
self.assertFalse(BilbyJob.objects.all().exists())
def test_job_upload_many_ini(self):
token = get_upload_token(self.client).data['generateBilbyJobUploadToken']['token']
test_name = "missing_ini"
test_description = "Missing Ini Description"
test_private = False
test_ini_string = create_test_ini_string(
{
'label': test_name,
},
True
)
test_file = SimpleUploadedFile(
name='test.tar.gz',
content=create_test_upload_data(test_ini_string, test_name, multiple_ini_files=True),
content_type='application/gzip'
)
test_input = {
"input": {
"uploadToken": token,
"details": {
"description": test_description,
"private": test_private
},
"jobFile": test_file
}
}
response = self.client.execute(
self.mutation,
test_input
)
self.assertEqual(
"Invalid number of ini files ending in `_config_complete.ini`. There should be exactly one.",
response.errors[0].message
)
self.assertFalse(BilbyJob.objects.all().exists())
def test_job_invalid_tar_gz_name(self):
token = get_upload_token(self.client).data['generateBilbyJobUploadToken']['token']
test_name = "invalid_tar_gz_name"
test_description = "Invalid .tar.gz Name Description"
test_private = False
test_ini_string = create_test_ini_string(
{
'label': test_name,
},
True
)
test_file = SimpleUploadedFile(
name='test.tar.g',
content=create_test_upload_data(test_ini_string, test_name, no_ini_file=True),
content_type='application/gzip'
)
test_input = {
"input": {
"uploadToken": token,
"details": {
"description": test_description,
"private": test_private
},
"jobFile": test_file
}
}
response = self.client.execute(
self.mutation,
test_input
)
self.assertEqual(
"Job upload should be a tar.gz file",
response.errors[0].message
)
self.assertFalse(BilbyJob.objects.all().exists())
def test_non_ligo_user_with_non_gwosc(self):
# Test checks that non-LIGO user cannot create real jobs with non-GWOSC channels, nor with invalid channels
# Now if the channels are proprietary, the non-ligo user should not be able to create jobs
self.client.authenticate(self.user, is_ligo=False)
token = get_upload_token(self.client).data['generateBilbyJobUploadToken']['token']
self.params['input']['uploadToken'] = token
for channel_dict in [
{'H1': 'GDS-CALIB_STRAIN', 'L1': 'GWOSC', 'V1': 'GWOSC'},
{'H1': 'GWOSC', 'L1': 'GDS-CALIB_STRAIN', 'V1': 'GWOSC'},
{'H1': 'GWOSC', 'L1': 'GWOSC', 'V1': 'Hrec_hoft_16384Hz'},
# Also check invalid channels
{'H1': 'testchannel1', 'L1': 'GWOSC', 'V1': 'GWOSC'},
{'H1': 'GWOSC', 'L1': 'imnotarealchannel', 'V1': 'GWOSC'},
{'H1': 'GWOSC', 'L1': 'GWOSC', 'V1': 'GWOSc'},
]:
ini_string = create_test_ini_string({
'label': 'testjob',
'n-simulation': 0,
'channel-dict': channel_dict
}, complete=True)
self.params['input']['jobFile'] = SimpleUploadedFile(
name='test.tar.gz',
content=create_test_upload_data(ini_string, 'testjob'),
content_type='application/gzip'
)
response = self.client.execute(self.mutation, self.params)
self.assertDictEqual(
self.expected_none, response.data, "create bilbyJob mutation returned unexpected data."
)
self.assertEqual(
response.errors[0].message,
'Non-LIGO members may only upload real jobs on GWOSC channels'
)
self.assertFalse(BilbyJob.objects.all().exists())
def test_ini_job_submission_supporting_file_all(self):
test_ini_string = create_test_ini_string({
'label':
"Test Name",
'detectors':
"['H1']",
'psd-dict':
'{L1:./supporting_files/psd/L1-psd.dat, V1:./supporting_files/psd/V1-psd.dat, '
'H1:./supporting_files/psd/H1-psd.dat}',
'spline-calibration-envelope-dict':
'{L1:./supporting_files/calib/L1-calib.dat, '
'V1:./supporting_files/calib/V1-calib.dat, '
'H1:./supporting_files/calib/H1-calib.dat}',
'prior-file':
'./supporting_files/prior/myprior.prior',
'gps-file':
'./supporting_files/gps/gps.dat',
'timeslide-file':
'./supporting_files/timeslide/timeslide.dat',
'injection-file':
'./supporting_files/injection/injection.dat',
'numerical-relativity-file':
'./supporting_files/nrf/nrf.dat'
})
self.mock_ini_job_submission_with_supporting_files(
test_ini_string,
[['L1', './supporting_files/psd/L1-psd.dat', SupportingFile.PSD],
['V1', './supporting_files/psd/V1-psd.dat', SupportingFile.PSD],
['H1', './supporting_files/psd/H1-psd.dat', SupportingFile.PSD],
[
'L1', './supporting_files/calib/L1-calib.dat',
SupportingFile.CALIBRATION
],
[
'V1', './supporting_files/calib/V1-calib.dat',
SupportingFile.CALIBRATION
],
[
'H1', './supporting_files/calib/H1-calib.dat',
SupportingFile.CALIBRATION
],
[
None, './supporting_files/prior/myprior.prior',
SupportingFile.PRIOR
], [None, './supporting_files/gps/gps.dat', SupportingFile.GPS],
[
None, './supporting_files/timeslide/timeslide.dat',
SupportingFile.TIME_SLIDE
],
[
None, './supporting_files/injection/injection.dat',
SupportingFile.INJECTION
],
[
None, './supporting_files/nrf/nrf.dat',
SupportingFile.NUMERICAL_RELATIVITY
]], [
'psd_dict', 'spline_calibration_envelope_dict', 'prior_file',
'gps_file', 'timeslide_file', 'injection_file',
'numerical_relativity_file'
])
def test_ini_job_submission(self, mock_api_call):
self.client.authenticate(self.user, is_ligo=True)
mock_api_call.return_value = {'jobId': 4321}
test_name = "Test Name"
test_description = "Test Description"
test_private = False
test_ini_string = create_test_ini_string(
{
'label': test_name,
'detectors': "['H1']"
}
)
test_input = {
"input": {
"params": {
"details": {
"name": test_name,
"description": test_description,
"private": test_private
},
"iniString": {
"iniString": test_ini_string
}
}
}
}
response = self.client.execute(self.mutation, test_input)
expected = {
'newBilbyJobFromIniString': {
'result': {
'jobId': 'QmlsYnlKb2JOb2RlOjE='
}
}
}
self.assertDictEqual(
expected, response.data, "create bilbyJob mutation returned unexpected data."
)
# And should create all k/v's with default values
job = BilbyJob.objects.all().last()
compare_ini_kvs(self, job, test_ini_string)
self.assertEqual(job.name, test_name)
self.assertEqual(job.description, test_description)
self.assertEqual(job.private, test_private)
# Check that ini labels are correctly set to the job name passed to the job details
test_name = "Test Name1"
test_ini_string = create_test_ini_string(
{
'label': "Not the real job name",
'detectors': "['H1']"
}
)
test_input = {
"input": {
"params": {
"details": {
"name": test_name,
"description": test_description,
"private": test_private
},
"iniString": {
"iniString": test_ini_string
}
}
}
}
response = self.client.execute(self.mutation, test_input)
expected = {
'newBilbyJobFromIniString': {
'result': {
'jobId': 'QmlsYnlKb2JOb2RlOjI='
}
}
}
self.assertDictEqual(
expected, response.data, "create bilbyJob mutation returned unexpected data."
)
# And should create all k/v's with default values
job = BilbyJob.objects.all().last()
compare_ini_kvs(self, job, job.ini_string)
self.assertEqual(job.name, test_name)
self.assertEqual(job.description, test_description)
self.assertEqual(job.private, test_private)
def setUp(self):
self.user = User.objects.create(username="******",
first_name="buffy",
last_name="summers")
self.client.authenticate(self.user)
token = get_upload_token(
self.client).data['generateBilbyJobUploadToken']['token']
# Create a new uploaded bilby job
test_name = "myjob"
test_description = "Test Description"
test_private = False
test_ini_string = create_test_ini_string(
{
'label': test_name,
'outdir': './'
}, True)
test_file = SimpleUploadedFile(name='test.tar.gz',
content=create_test_upload_data(
test_ini_string, test_name),
content_type='application/gzip')
test_input = {
"input": {
"uploadToken": token,
"details": {
"description": test_description,
"private": test_private
},
"jobFile": test_file
}
}
response = self.client.execute(
"""
mutation JobUploadMutation($input: UploadBilbyJobMutationInput!) {
uploadBilbyJob(input: $input) {
result {
jobId
}
}
}
""", test_input)
self.global_id = response.data['uploadBilbyJob']['result']['jobId']
self.job = BilbyJob.objects.all().last()
self.query = f"""
query {{
bilbyResultFiles (jobId: "{self.global_id}") {{
files {{
path
isDir
fileSize
downloadToken
}}
isUploadedJob
}}
}}
"""
self.mutation = """
mutation ResultFileMutation($input: GenerateFileDownloadIdsInput!) {
generateFileDownloadIds(input: $input) {
result
}
}
"""
self.mut_input = {
'input': {
'jobId': self.global_id,
'downloadTokens': None
}
}
self.http_client = Client()
def test_job_upload_success(self):
token = get_upload_token(self.client).data['generateBilbyJobUploadToken']['token']
test_name = "myjob"
test_description = "Test Description"
test_private = False
test_ini_string = create_test_ini_string(
{
'label': test_name,
'outdir': './'
},
True
)
test_file = SimpleUploadedFile(
name='test.tar.gz',
content=create_test_upload_data(test_ini_string, test_name),
content_type='application/gzip'
)
test_input = {
"input": {
"uploadToken": token,
"details": {
"description": test_description,
"private": test_private
},
"jobFile": test_file
}
}
response = self.client.execute(
self.mutation,
test_input
)
expected = {
'uploadBilbyJob': {
'result': {
'jobId': 'QmlsYnlKb2JOb2RlOjE='
}
}
}
self.assertDictEqual(
expected,
response.data
)
# And should create all k/v's with default values
job = BilbyJob.objects.all().last()
compare_ini_kvs(self, job, test_ini_string)
self.assertEqual(job.name, test_name)
self.assertEqual(job.description, test_description)
self.assertEqual(job.private, test_private)
# Check that the output directories and ini file were correctly created
job_dir = job.get_upload_directory()
self.assertTrue(os.path.isdir(job_dir))
self.assertTrue(os.path.isdir(os.path.join(job_dir, 'data')))
self.assertTrue(os.path.isdir(os.path.join(job_dir, 'result')))
self.assertTrue(os.path.isdir(os.path.join(job_dir, 'results_page')))
self.assertTrue(os.path.isfile(os.path.join(job_dir, 'myjob_config_complete.ini')))
self.assertTrue(os.path.isfile(os.path.join(job_dir, 'archive.tar.gz')))
def test_job_upload_success_outdir_replace(self):
token = get_upload_token(self.client).data['generateBilbyJobUploadToken']['token']
test_name = "another_job"
test_description = "Another Description"
test_private = True
test_ini_string = create_test_ini_string(
{
'label': test_name,
'outdir': '/some/path/not/good/'
},
True
)
test_file = SimpleUploadedFile(
name='anothertest.tar.gz',
content=create_test_upload_data(test_ini_string, test_name),
content_type='application/gzip'
)
test_input = {
"input": {
"uploadToken": token,
"details": {
"description": test_description,
"private": test_private
},
"jobFile": test_file
}
}
response = self.client.execute(
self.mutation,
test_input
)
expected = {
'uploadBilbyJob': {
'result': {
'jobId': 'QmlsYnlKb2JOb2RlOjE='
}
}
}
self.assertDictEqual(
expected,
response.data
)
job = BilbyJob.objects.all().last()
self.assertTrue(
IniKeyValue.objects.filter(
job=job,
key='outdir',
value=json.dumps('./')
).exists()
)
self.assertEqual(job.name, test_name)
self.assertEqual(job.description, test_description)
self.assertEqual(job.private, test_private)
# Check that the output directories and ini file were correctly created
job_dir = job.get_upload_directory()
self.assertTrue(os.path.isdir(job_dir))
self.assertTrue(os.path.isdir(os.path.join(job_dir, 'data')))
self.assertTrue(os.path.isdir(os.path.join(job_dir, 'result')))
self.assertTrue(os.path.isdir(os.path.join(job_dir, 'results_page')))
self.assertTrue(os.path.isfile(os.path.join(job_dir, 'another_job_config_complete.ini')))
self.assertTrue(os.path.isfile(os.path.join(job_dir, 'archive.tar.gz')))
def test_cluster_submission(self):
self.client.authenticate(self.user, is_ligo=True)
return_result = {'jobId': 1122}
self.responses.add(
responses.POST,
f"{settings.GWCLOUD_JOB_CONTROLLER_API_URL}/job/",
body=json.dumps(return_result),
status=200
)
test_name = "Test Name"
test_description = "Test Description"
test_private = False
test_ini_string = create_test_ini_string(
{
'label': test_name,
'detectors': "['H1']"
}
)
test_input = {
"input": {
"params": {
"details": {
"name": test_name,
"description": test_description,
"private": test_private
},
"iniString": {
"iniString": test_ini_string
}
}
}
}
# First test no cluster - this should default to 'default'
response = self.client.execute(self.mutation, test_input)
expected = {
'newBilbyJobFromIniString': {
'result': {
'jobId': 'QmlsYnlKb2JOb2RlOjE='
}
}
}
self.assertDictEqual(
expected, response.data, "create bilbyJob mutation returned unexpected data."
)
# Check the job controller id was set as expected
job = BilbyJob.objects.all().last()
self.assertEqual(job.job_controller_id, 1122)
# Check that the correct cluster was used in the request
r = json.loads(self.responses.calls[0].request.body)
self.assertEqual(r['cluster'], 'default')
job.delete()
# Next test default cluster uses the default cluster
test_input['input']['params']['details']['cluster'] = 'default'
response = self.client.execute(self.mutation, test_input)
expected = {
'newBilbyJobFromIniString': {
'result': {
'jobId': 'QmlsYnlKb2JOb2RlOjI='
}
}
}
self.assertDictEqual(
expected, response.data, "create bilbyJob mutation returned unexpected data."
)
# Check that the correct cluster was used in the request
r = json.loads(self.responses.calls[1].request.body)
self.assertEqual(r['cluster'], 'default')
BilbyJob.objects.all().last().delete()
# Next test "another" cluster
test_input['input']['params']['details']['cluster'] = 'another'
response = self.client.execute(self.mutation, test_input)
expected = {
'newBilbyJobFromIniString': {
'result': {
'jobId': 'QmlsYnlKb2JOb2RlOjM='
}
}
}
self.assertDictEqual(
expected, response.data, "create bilbyJob mutation returned unexpected data."
)
# Check that the correct cluster was used in the request
r = json.loads(self.responses.calls[2].request.body)
self.assertEqual(r['cluster'], 'another')
BilbyJob.objects.all().last().delete()
# Finally test invalid clusters are rejected cluster
test_input['input']['params']['details']['cluster'] = 'not_real'
response = self.client.execute(self.mutation, test_input)
self.assertEqual(
response.errors[0].message, "Error submitting job, cluster 'not_real' is not one of [default another]"
)
