def data_download_test_xml(self, xml_file_name, uuids,
client_options='', server_options='', check_sha1=True):
server = None
# delete the bams so we can download them
for uuid in uuids:
if os.path.isfile(self.client_bam(uuid)):
os.remove(self.client_bam(uuid))
if TestConfig.MOCKHUB:
# prepare server
server = GeneTorrentInstance(
'-s server%sroot -q server%sworkdir -c %s ' \
'--security-api %s %s' \
% (
os.path.sep,
os.path.sep,
self.cred_filename,
TestConfig.SECURITY_API,
server_options,
), instance_type=InstanceType.GT_SERVER)
# add new download GTO to server work queue
# add gt_download_mode flag to server gto
for uuid in uuids:
gtodict = read_gto(self.client_gto(uuid))
gtodict['gt_download_mode'] = 'true'
expiry = time.time() + (1 * 3600 * 24) # expire in 1 day
set_key(gtodict, 'expires on', int(expiry))
emit_gto(gtodict, self.server_gto(uuid), True)
else:
self.upload_sleep(3)
# prepare download client
client = GeneTorrentInstance('-d %s ' \
'-p client2 -c %s %s' \
% (
xml_file_name,
self.cred_filename,
client_options,
), instance_type=InstanceType.GT_DOWNLOAD)
# wait for download client to exit
client_sout, client_serr = client.communicate()
if server:
self.terminate_server(server)
# check download client return code
self.assertEqual(client.returncode, 0)
self.assertTrue('Downloaded' in client_serr)
self.assertTrue('state changed to: finished' in client_sout)
# check file hashes on both sides of transfer
for uuid in uuids:
if check_sha1 and TestConfig.MOCKHUB:
self.assertTrue(self.compare_hashes(
self.client_bam2(uuid), self.server_bam(uuid)))
return client.returncode
def test_usage_and_invalid_options(self):
"""
Test usage and invalid options for GeneTorrent
"""
gt = GeneTorrentInstance(
self.resourcedir + "--help", instance_type=InstanceType.GT_DOWNLOAD, add_defaults=False
)
(sout, serr) = gt.communicate()
self.assertIn("Usage", sout)
self.assertEqual(gt.returncode, 0)
gt = GeneTorrentInstance(
self.resourcedir + "--this-option-does-not-exist",
instance_type=InstanceType.GT_DOWNLOAD,
add_defaults=False,
)
(sout, serr) = gt.communicate()
self.assertIn("this-option-does-not-exist", serr)
self.assertEqual(gt.returncode, 9)
gt = GeneTorrentInstance(self.resourcedir + "-z", instance_type=InstanceType.GT_DOWNLOAD, add_defaults=False)
(sout, serr) = gt.communicate()
self.assertIn("option", serr)
self.assertIn("-z", serr)
self.assertEqual(gt.returncode, 9)
gt = GeneTorrentInstance(
self.resourcedir + "-c foo --credential-file=bar",
instance_type=InstanceType.GT_DOWNLOAD,
add_defaults=False,
)
(sout, serr) = gt.communicate()
result = "multiple occurrences" in serr or "specified more than once" in serr
self.assertTrue(result)
self.assertEqual(gt.returncode, 9)
def test_long_upload_options(self):
"""
Test upload mode options to Gene Torrent (-u)
"""
gt = GeneTorrentInstance(
self.resourcedir + "--upload", instance_type=InstanceType.GT_UPLOAD, add_defaults=False
)
(sout, serr) = gt.communicate()
self.assertIn("required", serr)
self.assertIn("missing", serr)
self.assertIn("upload", serr)
self.assertEqual(gt.returncode, 9)
gt = GeneTorrentInstance(
self.resourcedir + "--upload xxx", instance_type=InstanceType.GT_UPLOAD, add_defaults=False
)
(sout, serr) = gt.communicate()
self.assertIn("manifest file not found", serr)
self.assertEqual(gt.returncode, 9)
gt = GeneTorrentInstance(
self.resourcedir + "--upload %s" % (self.cred_filename),
instance_type=InstanceType.GT_UPLOAD,
add_defaults=False,
)
(sout, serr) = gt.communicate()
self.assertIn("include a credential file", serr)
self.assertEqual(gt.returncode, 9)
gt = GeneTorrentInstance(
self.resourcedir + "--upload %s --credential-file %s" % (self.cred_filename, self.cred_filename),
instance_type=InstanceType.GT_UPLOAD,
add_defaults=False,
)
(sout, serr) = gt.communicate()
self.assertIn("error attempting to process the file", serr)
self.assertEqual(gt.returncode, 97)
gt = GeneTorrentInstance(
self.resourcedir
+ "--upload %s --credential-file %s --path /does/not/exit" % (self.cred_filename, self.cred_filename),
instance_type=InstanceType.GT_UPLOAD,
add_defaults=False,
)
(sout, serr) = gt.communicate()
self.assertIn("unable to opening directory", serr)
self.assertEqual(gt.returncode, 9)
def test_upload_download_client_peer_verify(self):
'''Test upload/download client curl SSL peer verification behavior.'''
uuid = uuid4()
self.generate_bam_data(uuid, 1024)
# no server in this test
# prepare upload client
client_manifest = os.path.join(
'client',
str(uuid),
'manifest-generated.xml',
)
client = GeneTorrentInstance('-u %s -p client -c %s ' \
% (client_manifest, self.cred_filename), ssl_no_verify_ca=False,
instance_type = InstanceType.GT_UPLOAD)
# wait for upload client to exit
client_sout, client_serr = client.communicate()
if TestConfig.VERBOSE:
print 'client stdout: ' + client_sout
print 'client stderr: ' + client_serr
self.assertEqual(client.returncode, 203)
self.assertTrue('Peer certificate cannot be authenticated' \
in client_sout)
# upload client properly exited
# now, test download client
client = GeneTorrentInstance( \
'-d %s/cghub/data/analysis/%s -p client -c %s ' \
% (TestConfig.HUB_SERVER, str(uuid), self.cred_filename),
ssl_no_verify_ca=False, instance_type=InstanceType.GT_DOWNLOAD)
# wait for upload client to exit
client_sout, client_serr = client.communicate()
if TestConfig.VERBOSE:
print 'client stdout: ' + client_sout
print 'client stderr: ' + client_serr
self.assertEqual(client.returncode, 87)
self.assertTrue('Peer certificate cannot be authenticated' in \
client_sout)
def test_short_download_options(self):
"""
Test download mode options to Gene Torrent (-d)
"""
gt = GeneTorrentInstance(self.resourcedir + "-d", instance_type=InstanceType.GT_DOWNLOAD, add_defaults=False)
(sout, serr) = gt.communicate()
self.assertIn("required", serr)
self.assertIn("missing", serr)
self.assertIn("download", serr)
self.assertEqual(gt.returncode, 9)
gt = GeneTorrentInstance(
self.resourcedir + "-d xxx", instance_type=InstanceType.GT_DOWNLOAD, add_defaults=False
)
(sout, serr) = gt.communicate()
self.assertIn("include a credential file", serr)
self.assertEqual(gt.returncode, 9)
gt = GeneTorrentInstance(
self.resourcedir + "-d xxx -c /file/does/not/exist",
instance_type=InstanceType.GT_DOWNLOAD,
add_defaults=False,
)
(sout, serr) = gt.communicate()
self.assertIn("credentials file not found", serr)
self.assertEqual(gt.returncode, 55)
gt = GeneTorrentInstance(
self.resourcedir + "-d xxx -c %s" % (self.cred_filename),
instance_type=InstanceType.GT_DOWNLOAD,
add_defaults=False,
)
(sout, serr) = gt.communicate()
self.assertIn("'xxx' is too short", serr)
self.assertEqual(gt.returncode, 201)
gt = GeneTorrentInstance(
self.resourcedir + "-c %s -p /path/does/not/exist -d xxx" % (self.cred_filename),
instance_type=InstanceType.GT_DOWNLOAD,
add_defaults=False,
)
(sout, serr) = gt.communicate()
self.assertIn("unable to opening directory", serr)
self.assertEqual(gt.returncode, 9)
def create_gto_only(self, uuid):
# prepare upload client
client_manifest = os.path.join(
'client',
str(uuid),
'manifest-generated.xml',
)
client = GeneTorrentInstance('-u %s -p client -c %s ' \
'--gto-only' \
% (
client_manifest,
self.cred_filename,
), instance_type=InstanceType.GT_UPLOAD)
# wait for upload client to exit
client_sout, client_serr = client.communicate()
self.assertEqual(client.returncode, 0)
self.assertTrue('upload will be skipped' in str.lower(client_sout))
def test_server_peer_verify(self):
'''Test server client curl SSL peer verification behavior.'''
uuid = uuid4()
self.generate_bam_data(uuid, 1024)
self.create_gto_only(uuid)
# now, test server without --ssl-no-verify-ca
server = GeneTorrentInstance( \
'-s %s -q %s -c %s --security-api ' \
'%s ' \
'--foreground -l stdout:full -R .' \
% ('server' + os.path.sep + 'root',
'server' + os.path.sep + 'workdir',
self.cred_filename,
TestConfig.SECURITY_API
), ssl_no_verify_ca=False, instance_type=InstanceType.GT_SERVER,
add_defaults=False)
# copy gto to server
client_gto = self.client_gto(uuid)
server_gto = self.server_gto(uuid)
gtodict = read_gto(client_gto)
gtodict['gto_download_mode'] = 'true'
emit_gto(gtodict, server_gto, True)
time.sleep(10)
# kill server
server.kill()
# check for SSL signing error
server.stdout_buffer.seek(0)
server_sout = server.stdout_buffer.read()
print server_sout
self.assertTrue('while attempting a CSR signing transaction' in \
server_sout)
def test_long_multiple_modes(self):
"""
Test multiple mode options to Gene Torrent (-d, -s, -u)
"""
gt = GeneTorrentInstance(
self.resourcedir + "--download xxx --server %s --upload xxx -c %s" % (os.getcwd(), self.cred_filename),
instance_type=InstanceType.GT_DOWNLOAD,
add_defaults=False,
)
(sout, serr) = gt.communicate()
self.assertTrue("unknown option" in serr or "unrecognised option" in serr)
self.assertEqual(gt.returncode, 9)
gt = GeneTorrentInstance(
self.resourcedir + "--download xxx --server %s -c %s" % (os.getcwd(), self.cred_filename),
instance_type=InstanceType.GT_DOWNLOAD,
add_defaults=False,
)
(sout, serr) = gt.communicate()
self.assertIn("too short", serr)
self.assertEqual(gt.returncode, 201)
gt = GeneTorrentInstance(
self.resourcedir
+ "--server %s -q %s --download xxx -c %s" % (os.getcwd(), os.getcwd(), self.cred_filename),
instance_type=InstanceType.GT_SERVER,
add_defaults=False,
)
(sout, serr) = gt.communicate()
self.assertTrue("unknown option" in serr or "unrecognised option" in serr)
self.assertEqual(gt.returncode, 9)
gt = GeneTorrentInstance(
self.resourcedir + "--download xxx --upload xxx", instance_type=InstanceType.GT_UPLOAD, add_defaults=False
)
(sout, serr) = gt.communicate()
self.assertTrue("unknown option" in serr or "unrecognised option" in serr)
self.assertEqual(gt.returncode, 9)
def test_short_multiple_modes(self):
"""
Test multiple mode options to Gene Torrent (-d, -s, -u)
"""
gt = GeneTorrentInstance(
self.resourcedir + "-d xxx -s %s -u xxx" % (os.getcwd()),
instance_type=InstanceType.GT_DOWNLOAD,
add_defaults=False,
)
(sout, serr) = gt.communicate()
self.assertTrue("unknown option" in serr or "unrecognised option" in serr)
self.assertEqual(gt.returncode, 9)
gt = GeneTorrentInstance(
self.resourcedir + "-d xxx -s %s" % (os.getcwd()),
instance_type=InstanceType.GT_DOWNLOAD,
add_defaults=False,
)
(sout, serr) = gt.communicate()
self.assertTrue("unknown option" in serr or "unrecognised option" in serr)
self.assertEqual(gt.returncode, 9)
gt = GeneTorrentInstance(
self.resourcedir + "-s '%s' -u xxx" % (os.getcwd()),
instance_type=InstanceType.GT_SERVER,
add_defaults=False,
)
(sout, serr) = gt.communicate()
self.assertTrue("unknown option" in serr or "unrecognised option" in serr)
self.assertEqual(gt.returncode, 9)
gt = GeneTorrentInstance(
self.resourcedir + "-d xxx -u xxx", instance_type=InstanceType.GT_DOWNLOAD, add_defaults=False
)
(sout, serr) = gt.communicate()
self.assertTrue("unknown option" in serr or "unrecognised option" in serr)
self.assertEqual(gt.returncode, 9)
def data_download_test_uuid(self, uuid, client_options='', server_options='',
check_sha1=True, assert_rc=0, assert_serr='', server_ct=1):
# delete the bam so we can download it
if os.path.isfile(self.client_bam(uuid)):
os.remove(self.client_bam(uuid))
servers = []
if TestConfig.MOCKHUB:
# prepare server(s)
for i in range(0, server_ct):
server = GeneTorrentInstance(
'-s server%sroot -q server%sworkdir -c %s ' \
'--security-api %s %s' \
% (
os.path.sep,
os.path.sep,
self.cred_filename,
TestConfig.SECURITY_API,
server_options,
), instance_type=InstanceType.GT_SERVER,
client_num=i)
servers.append(server)
# add new download GTO to server work queue
# add gt_download_mode flag to server gto
gtodict = read_gto(self.client_gto(uuid))
gtodict['gt_download_mode'] = 'true'
expiry = time.time() + (1 * 3600 * 24) # expire in 1 day
set_key(gtodict, 'expires on', int(expiry))
emit_gto(gtodict, self.server_gto(uuid), True)
else:
self.upload_sleep(3)
# prepare download client
client = GeneTorrentInstance('-d %s/cghub/data/analysis/download/%s ' \
'-p client2 %s %s' \
% (
TestConfig.HUB_SERVER,
str(uuid),
'-c ' + self.cred_filename if not '-c ' in client_options else '',
client_options,
), instance_type=InstanceType.GT_DOWNLOAD)
# wait for download client to exit
client_sout, client_serr = client.communicate()
if servers:
for server in servers:
self.terminate_server(server)
# check download client return code
self.assertEqual(client.returncode, assert_rc)
# caller-passed client_serr contains assert
if assert_serr:
self.assertTrue(assert_serr in client_serr)
# only continue checking if assert_rc == 0
if assert_rc == 0:
# check download client
self.assertTrue('Downloaded' in client_serr)
self.assertTrue('state changed to: finished' in client_sout)
# check file hashes on both sides of transfer
if servers and check_sha1:
self.assertTrue(self.compare_hashes(
self.client_bam2(uuid), self.server_bam(uuid)))
return client.returncode
def data_upload_test(self, size, data_generator=DataGenRandom,
ssl=True, client_options='', server_options='', check_sha1=True):
server = None
uuid = uuid4()
self.generate_bam_data(uuid, size, data_generator)
if TestConfig.MOCKHUB:
# prepare server
server = GeneTorrentInstance(
'-s server%sroot -q server%sworkdir -c %s ' \
'--security-api %s %s' \
% (
os.path.sep,
os.path.sep,
self.cred_filename,
TestConfig.SECURITY_API,
server_options
), instance_type=InstanceType.GT_SERVER)
else:
# submit via cgsubmit
cgsubmit_process = Popen(['python', '../cgsubmit', '-s',
TestConfig.HUB_SERVER, '-c', TestConfig.CREDENTIAL, '-u',
str(uuid)],
stdout=PIPE, stderr=PIPE, cwd='client')
cgout, cgerr = cgsubmit_process.communicate()
if TestConfig.VERBOSE:
logger.info('cgsubmit stdout: ' + cgout)
logger.info('cgsubmit stderr: ' + cgerr)
self.assertEqual(cgsubmit_process.returncode, 0)
# prepare upload client
client_manifest = os.path.join(
'client',
str(uuid),
'manifest-generated.xml',
)
# modify manifest for nossl, if necessary
if not ssl:
manifest_file = open(client_manifest, 'r+')
try:
manifest_text = manifest_file.read()
manifest_text = manifest_text.replace('/analysis/',
'/analysis-nossl/')
manifest_file.seek(0)
manifest_file.truncate()
manifest_file.write(manifest_text)
finally:
manifest_file.close()
client = GeneTorrentInstance('-u %s -p client -c %s %s' \
% (
client_manifest,
self.cred_filename,
client_options,
), instance_type=InstanceType.GT_UPLOAD)
# add new upload GTO to server work queue
gto_file = False
ssl_cert = False
while not gto_file or (not ssl_cert and ssl):
# check for premature exits of GT programs
# subprocess.poll() returns return code, nonzero is True
if client.poll():
self.fail('Client exited prematurely')
if server and server.poll():
self.fail('Server exited prematurely')
try:
os.stat(self.client_gto(uuid))
gto_file = True
gtodata = read_gto(self.client_gto(uuid))
ssl_cert = True if 'ssl-cert' in gtodata['info'] else False
except:
time.sleep(1)
copy2(self.client_gto(uuid), os.path.join(
'server',
'workdir',
))
# wait for upload client to exit
client_sout, client_serr = client.communicate()
if server:
self.terminate_server(server)
# check upload client return code
self.assertEqual(client.returncode, 0)
# check upload client stderr complete message (100.000%)
self.assertTrue('100.000% complete' in client_serr)
# check file hashes on both sides of transfer
if server and check_sha1:
self.assertTrue(self.compare_hashes(
self.client_bam(uuid), self.server_bam(uuid)))
return uuid
def test_long_download_options(self):
"""
Test download mode options to Gene Torrent (-d)
"""
gt = GeneTorrentInstance(
self.resourcedir + "--download", instance_type=InstanceType.GT_DOWNLOAD, add_defaults=False
)
(sout, serr) = gt.communicate()
self.assertIn("required", serr)
self.assertIn("missing", serr)
self.assertIn("download", serr)
self.assertEqual(gt.returncode, 9)
gt = GeneTorrentInstance(
self.resourcedir + "--download xxx", instance_type=InstanceType.GT_DOWNLOAD, add_defaults=False
)
(sout, serr) = gt.communicate()
self.assertIn("include a credential file", serr)
self.assertEqual(gt.returncode, 9)
gt = GeneTorrentInstance(
self.resourcedir + "--download xxx -c /file/does/not/exist",
instance_type=InstanceType.GT_DOWNLOAD,
add_defaults=False,
)
(sout, serr) = gt.communicate()
self.assertIn("credentials file not found", serr)
self.assertEqual(gt.returncode, 55)
gt = GeneTorrentInstance(
self.resourcedir + "--download xxx --credential-file %s" % (self.cred_filename),
instance_type=InstanceType.GT_DOWNLOAD,
add_defaults=False,
)
(sout, serr) = gt.communicate()
self.assertIn("'xxx' is too short", serr)
self.assertEqual(gt.returncode, 201)
gt = GeneTorrentInstance(
self.resourcedir + "--credential-file %s --path /path/does/not/exist --download xxx" % (self.cred_filename),
instance_type=InstanceType.GT_DOWNLOAD,
add_defaults=False,
)
(sout, serr) = gt.communicate()
self.assertIn("unable to opening directory", serr)
self.assertEqual(gt.returncode, 9)
gt = GeneTorrentInstance(
self.resourcedir + "--credential-file %s --download xxx --security-api=aaa" % (self.cred_filename),
instance_type=InstanceType.GT_DOWNLOAD,
add_defaults=False,
)
(sout, serr) = gt.communicate()
self.assertIn("Invalid URI for '--security-api", serr)
self.assertEqual(gt.returncode, 9)
gt = GeneTorrentInstance(
self.resourcedir + "--download xxx --max-children=0",
instance_type=InstanceType.GT_DOWNLOAD,
add_defaults=False,
)
(sout, serr) = gt.communicate()
self.assertIn("must be greater than 0", serr)
self.assertEqual(gt.returncode, 9)
def test_upload_download_client_timeout(self):
'''Test upload/download client inactivity timeout.'''
uuid = uuid4()
self.generate_bam_data(uuid, 1024 * 1024 * 1)
# no server in this test
start_time = time.clock()
# prepare upload client
client_manifest = os.path.join(
'client',
str(uuid),
'manifest-generated.xml',
)
client = GeneTorrentInstance('-u %s -p client -c %s ' \
'-k 1 -vv -l stdout:full -R . --ssl-no-verify-ca' \
% (client_manifest, self.cred_filename), instance_type=InstanceType.GT_UPLOAD,
add_defaults=False)
time.sleep(5)
# wait for upload client to exit
client_sout, client_serr = client.communicate()
if TestConfig.VERBOSE:
print 'client stdout: ' + client_sout
print 'client stderr: ' + client_serr
self.assertEqual(client.returncode, 206)
self.assertTrue('Inactivity timeout triggered' in client_sout)
# upload client properly timed out
# now, test download client
# delete bam first, else client will exit since it has the file
client_bam = self.client_bam(uuid)
os.remove(client_bam)
client = GeneTorrentInstance( \
'-d %s/cghub/data/analysis/%s -p client2 -c %s ' \
'-k 1 -vv -l stdout:full -R . --ssl-no-verify-ca' \
% (TestConfig.HUB_SERVER, str(uuid), self.cred_filename),
instance_type=InstanceType.GT_DOWNLOAD, add_defaults=False)
time.sleep(5)
# wait for download client to exit
client_sout, client_serr = client.communicate()
if TestConfig.VERBOSE:
print 'client stdout: ' + client_sout
print 'client stderr: ' + client_serr
self.assertEqual(client.returncode, 206)
self.assertTrue('Inactivity timeout triggered' in client_sout)
end_time = time.clock()
# that should take a little more than 2 minutes,
# allowing for two 60-second timeouts
self.assertTrue(end_time - start_time < 200,
'Two inactivity timeouts took longer than two minutes')
