def test_usage_and_invalid_options(self): """ Test usage and invalid options for GeneTorrent """ gt = GeneTorrentInstance( self.resourcedir + "--help", instance_type=InstanceType.GT_DOWNLOAD, add_defaults=False ) (sout, serr) = gt.communicate() self.assertIn("Usage", sout) self.assertEqual(gt.returncode, 0) gt = GeneTorrentInstance( self.resourcedir + "--this-option-does-not-exist", instance_type=InstanceType.GT_DOWNLOAD, add_defaults=False, ) (sout, serr) = gt.communicate() self.assertIn("this-option-does-not-exist", serr) self.assertEqual(gt.returncode, 9) gt = GeneTorrentInstance(self.resourcedir + "-z", instance_type=InstanceType.GT_DOWNLOAD, add_defaults=False) (sout, serr) = gt.communicate() self.assertIn("option", serr) self.assertIn("-z", serr) self.assertEqual(gt.returncode, 9) gt = GeneTorrentInstance( self.resourcedir + "-c foo --credential-file=bar", instance_type=InstanceType.GT_DOWNLOAD, add_defaults=False, ) (sout, serr) = gt.communicate() result = "multiple occurrences" in serr or "specified more than once" in serr self.assertTrue(result) self.assertEqual(gt.returncode, 9)
def data_download_test_xml(self, xml_file_name, uuids, client_options='', server_options='', check_sha1=True): server = None # delete the bams so we can download them for uuid in uuids: if os.path.isfile(self.client_bam(uuid)): os.remove(self.client_bam(uuid)) if TestConfig.MOCKHUB: # prepare server server = GeneTorrentInstance( '-s server%sroot -q server%sworkdir -c %s ' \ '--security-api %s %s' \ % ( os.path.sep, os.path.sep, self.cred_filename, TestConfig.SECURITY_API, server_options, ), instance_type=InstanceType.GT_SERVER) # add new download GTO to server work queue # add gt_download_mode flag to server gto for uuid in uuids: gtodict = read_gto(self.client_gto(uuid)) gtodict['gt_download_mode'] = 'true' expiry = time.time() + (1 * 3600 * 24) # expire in 1 day set_key(gtodict, 'expires on', int(expiry)) emit_gto(gtodict, self.server_gto(uuid), True) else: self.upload_sleep(3) # prepare download client client = GeneTorrentInstance('-d %s ' \ '-p client2 -c %s %s' \ % ( xml_file_name, self.cred_filename, client_options, ), instance_type=InstanceType.GT_DOWNLOAD) # wait for download client to exit client_sout, client_serr = client.communicate() if server: self.terminate_server(server) # check download client return code self.assertEqual(client.returncode, 0) self.assertTrue('Downloaded' in client_serr) self.assertTrue('state changed to: finished' in client_sout) # check file hashes on both sides of transfer for uuid in uuids: if check_sha1 and TestConfig.MOCKHUB: self.assertTrue(self.compare_hashes( self.client_bam2(uuid), self.server_bam(uuid))) return client.returncode
def test_long_upload_options(self): """ Test upload mode options to Gene Torrent (-u) """ gt = GeneTorrentInstance( self.resourcedir + "--upload", instance_type=InstanceType.GT_UPLOAD, add_defaults=False ) (sout, serr) = gt.communicate() self.assertIn("required", serr) self.assertIn("missing", serr) self.assertIn("upload", serr) self.assertEqual(gt.returncode, 9) gt = GeneTorrentInstance( self.resourcedir + "--upload xxx", instance_type=InstanceType.GT_UPLOAD, add_defaults=False ) (sout, serr) = gt.communicate() self.assertIn("manifest file not found", serr) self.assertEqual(gt.returncode, 9) gt = GeneTorrentInstance( self.resourcedir + "--upload %s" % (self.cred_filename), instance_type=InstanceType.GT_UPLOAD, add_defaults=False, ) (sout, serr) = gt.communicate() self.assertIn("include a credential file", serr) self.assertEqual(gt.returncode, 9) gt = GeneTorrentInstance( self.resourcedir + "--upload %s --credential-file %s" % (self.cred_filename, self.cred_filename), instance_type=InstanceType.GT_UPLOAD, add_defaults=False, ) (sout, serr) = gt.communicate() self.assertIn("error attempting to process the file", serr) self.assertEqual(gt.returncode, 97) gt = GeneTorrentInstance( self.resourcedir + "--upload %s --credential-file %s --path /does/not/exit" % (self.cred_filename, self.cred_filename), instance_type=InstanceType.GT_UPLOAD, add_defaults=False, ) (sout, serr) = gt.communicate() self.assertIn("unable to opening directory", serr) self.assertEqual(gt.returncode, 9)
def test_upload_download_client_peer_verify(self): '''Test upload/download client curl SSL peer verification behavior.''' uuid = uuid4() self.generate_bam_data(uuid, 1024) # no server in this test # prepare upload client client_manifest = os.path.join( 'client', str(uuid), 'manifest-generated.xml', ) client = GeneTorrentInstance('-u %s -p client -c %s ' \ % (client_manifest, self.cred_filename), ssl_no_verify_ca=False, instance_type = InstanceType.GT_UPLOAD) # wait for upload client to exit client_sout, client_serr = client.communicate() if TestConfig.VERBOSE: print 'client stdout: ' + client_sout print 'client stderr: ' + client_serr self.assertEqual(client.returncode, 203) self.assertTrue('Peer certificate cannot be authenticated' \ in client_sout) # upload client properly exited # now, test download client client = GeneTorrentInstance( \ '-d %s/cghub/data/analysis/%s -p client -c %s ' \ % (TestConfig.HUB_SERVER, str(uuid), self.cred_filename), ssl_no_verify_ca=False, instance_type=InstanceType.GT_DOWNLOAD) # wait for upload client to exit client_sout, client_serr = client.communicate() if TestConfig.VERBOSE: print 'client stdout: ' + client_sout print 'client stderr: ' + client_serr self.assertEqual(client.returncode, 87) self.assertTrue('Peer certificate cannot be authenticated' in \ client_sout)
def test_short_download_options(self): """ Test download mode options to Gene Torrent (-d) """ gt = GeneTorrentInstance(self.resourcedir + "-d", instance_type=InstanceType.GT_DOWNLOAD, add_defaults=False) (sout, serr) = gt.communicate() self.assertIn("required", serr) self.assertIn("missing", serr) self.assertIn("download", serr) self.assertEqual(gt.returncode, 9) gt = GeneTorrentInstance( self.resourcedir + "-d xxx", instance_type=InstanceType.GT_DOWNLOAD, add_defaults=False ) (sout, serr) = gt.communicate() self.assertIn("include a credential file", serr) self.assertEqual(gt.returncode, 9) gt = GeneTorrentInstance( self.resourcedir + "-d xxx -c /file/does/not/exist", instance_type=InstanceType.GT_DOWNLOAD, add_defaults=False, ) (sout, serr) = gt.communicate() self.assertIn("credentials file not found", serr) self.assertEqual(gt.returncode, 55) gt = GeneTorrentInstance( self.resourcedir + "-d xxx -c %s" % (self.cred_filename), instance_type=InstanceType.GT_DOWNLOAD, add_defaults=False, ) (sout, serr) = gt.communicate() self.assertIn("'xxx' is too short", serr) self.assertEqual(gt.returncode, 201) gt = GeneTorrentInstance( self.resourcedir + "-c %s -p /path/does/not/exist -d xxx" % (self.cred_filename), instance_type=InstanceType.GT_DOWNLOAD, add_defaults=False, ) (sout, serr) = gt.communicate() self.assertIn("unable to opening directory", serr) self.assertEqual(gt.returncode, 9)
def test_long_multiple_modes(self): """ Test multiple mode options to Gene Torrent (-d, -s, -u) """ gt = GeneTorrentInstance( self.resourcedir + "--download xxx --server %s --upload xxx -c %s" % (os.getcwd(), self.cred_filename), instance_type=InstanceType.GT_DOWNLOAD, add_defaults=False, ) (sout, serr) = gt.communicate() self.assertTrue("unknown option" in serr or "unrecognised option" in serr) self.assertEqual(gt.returncode, 9) gt = GeneTorrentInstance( self.resourcedir + "--download xxx --server %s -c %s" % (os.getcwd(), self.cred_filename), instance_type=InstanceType.GT_DOWNLOAD, add_defaults=False, ) (sout, serr) = gt.communicate() self.assertIn("too short", serr) self.assertEqual(gt.returncode, 201) gt = GeneTorrentInstance( self.resourcedir + "--server %s -q %s --download xxx -c %s" % (os.getcwd(), os.getcwd(), self.cred_filename), instance_type=InstanceType.GT_SERVER, add_defaults=False, ) (sout, serr) = gt.communicate() self.assertTrue("unknown option" in serr or "unrecognised option" in serr) self.assertEqual(gt.returncode, 9) gt = GeneTorrentInstance( self.resourcedir + "--download xxx --upload xxx", instance_type=InstanceType.GT_UPLOAD, add_defaults=False ) (sout, serr) = gt.communicate() self.assertTrue("unknown option" in serr or "unrecognised option" in serr) self.assertEqual(gt.returncode, 9)
def test_short_multiple_modes(self): """ Test multiple mode options to Gene Torrent (-d, -s, -u) """ gt = GeneTorrentInstance( self.resourcedir + "-d xxx -s %s -u xxx" % (os.getcwd()), instance_type=InstanceType.GT_DOWNLOAD, add_defaults=False, ) (sout, serr) = gt.communicate() self.assertTrue("unknown option" in serr or "unrecognised option" in serr) self.assertEqual(gt.returncode, 9) gt = GeneTorrentInstance( self.resourcedir + "-d xxx -s %s" % (os.getcwd()), instance_type=InstanceType.GT_DOWNLOAD, add_defaults=False, ) (sout, serr) = gt.communicate() self.assertTrue("unknown option" in serr or "unrecognised option" in serr) self.assertEqual(gt.returncode, 9) gt = GeneTorrentInstance( self.resourcedir + "-s '%s' -u xxx" % (os.getcwd()), instance_type=InstanceType.GT_SERVER, add_defaults=False, ) (sout, serr) = gt.communicate() self.assertTrue("unknown option" in serr or "unrecognised option" in serr) self.assertEqual(gt.returncode, 9) gt = GeneTorrentInstance( self.resourcedir + "-d xxx -u xxx", instance_type=InstanceType.GT_DOWNLOAD, add_defaults=False ) (sout, serr) = gt.communicate() self.assertTrue("unknown option" in serr or "unrecognised option" in serr) self.assertEqual(gt.returncode, 9)
def create_gto_only(self, uuid): # prepare upload client client_manifest = os.path.join( 'client', str(uuid), 'manifest-generated.xml', ) client = GeneTorrentInstance('-u %s -p client -c %s ' \ '--gto-only' \ % ( client_manifest, self.cred_filename, ), instance_type=InstanceType.GT_UPLOAD) # wait for upload client to exit client_sout, client_serr = client.communicate() self.assertEqual(client.returncode, 0) self.assertTrue('upload will be skipped' in str.lower(client_sout))
def data_download_test_uuid(self, uuid, client_options='', server_options='', check_sha1=True, assert_rc=0, assert_serr='', server_ct=1): # delete the bam so we can download it if os.path.isfile(self.client_bam(uuid)): os.remove(self.client_bam(uuid)) servers = [] if TestConfig.MOCKHUB: # prepare server(s) for i in range(0, server_ct): server = GeneTorrentInstance( '-s server%sroot -q server%sworkdir -c %s ' \ '--security-api %s %s' \ % ( os.path.sep, os.path.sep, self.cred_filename, TestConfig.SECURITY_API, server_options, ), instance_type=InstanceType.GT_SERVER, client_num=i) servers.append(server) # add new download GTO to server work queue # add gt_download_mode flag to server gto gtodict = read_gto(self.client_gto(uuid)) gtodict['gt_download_mode'] = 'true' expiry = time.time() + (1 * 3600 * 24) # expire in 1 day set_key(gtodict, 'expires on', int(expiry)) emit_gto(gtodict, self.server_gto(uuid), True) else: self.upload_sleep(3) # prepare download client client = GeneTorrentInstance('-d %s/cghub/data/analysis/download/%s ' \ '-p client2 %s %s' \ % ( TestConfig.HUB_SERVER, str(uuid), '-c ' + self.cred_filename if not '-c ' in client_options else '', client_options, ), instance_type=InstanceType.GT_DOWNLOAD) # wait for download client to exit client_sout, client_serr = client.communicate() if servers: for server in servers: self.terminate_server(server) # check download client return code self.assertEqual(client.returncode, assert_rc) # caller-passed client_serr contains assert if assert_serr: self.assertTrue(assert_serr in client_serr) # only continue checking if assert_rc == 0 if assert_rc == 0: # check download client self.assertTrue('Downloaded' in client_serr) self.assertTrue('state changed to: finished' in client_sout) # check file hashes on both sides of transfer if servers and check_sha1: self.assertTrue(self.compare_hashes( self.client_bam2(uuid), self.server_bam(uuid))) return client.returncode
def data_upload_test(self, size, data_generator=DataGenRandom, ssl=True, client_options='', server_options='', check_sha1=True): server = None uuid = uuid4() self.generate_bam_data(uuid, size, data_generator) if TestConfig.MOCKHUB: # prepare server server = GeneTorrentInstance( '-s server%sroot -q server%sworkdir -c %s ' \ '--security-api %s %s' \ % ( os.path.sep, os.path.sep, self.cred_filename, TestConfig.SECURITY_API, server_options ), instance_type=InstanceType.GT_SERVER) else: # submit via cgsubmit cgsubmit_process = Popen(['python', '../cgsubmit', '-s', TestConfig.HUB_SERVER, '-c', TestConfig.CREDENTIAL, '-u', str(uuid)], stdout=PIPE, stderr=PIPE, cwd='client') cgout, cgerr = cgsubmit_process.communicate() if TestConfig.VERBOSE: logger.info('cgsubmit stdout: ' + cgout) logger.info('cgsubmit stderr: ' + cgerr) self.assertEqual(cgsubmit_process.returncode, 0) # prepare upload client client_manifest = os.path.join( 'client', str(uuid), 'manifest-generated.xml', ) # modify manifest for nossl, if necessary if not ssl: manifest_file = open(client_manifest, 'r+') try: manifest_text = manifest_file.read() manifest_text = manifest_text.replace('/analysis/', '/analysis-nossl/') manifest_file.seek(0) manifest_file.truncate() manifest_file.write(manifest_text) finally: manifest_file.close() client = GeneTorrentInstance('-u %s -p client -c %s %s' \ % ( client_manifest, self.cred_filename, client_options, ), instance_type=InstanceType.GT_UPLOAD) # add new upload GTO to server work queue gto_file = False ssl_cert = False while not gto_file or (not ssl_cert and ssl): # check for premature exits of GT programs # subprocess.poll() returns return code, nonzero is True if client.poll(): self.fail('Client exited prematurely') if server and server.poll(): self.fail('Server exited prematurely') try: os.stat(self.client_gto(uuid)) gto_file = True gtodata = read_gto(self.client_gto(uuid)) ssl_cert = True if 'ssl-cert' in gtodata['info'] else False except: time.sleep(1) copy2(self.client_gto(uuid), os.path.join( 'server', 'workdir', )) # wait for upload client to exit client_sout, client_serr = client.communicate() if server: self.terminate_server(server) # check upload client return code self.assertEqual(client.returncode, 0) # check upload client stderr complete message (100.000%) self.assertTrue('100.000% complete' in client_serr) # check file hashes on both sides of transfer if server and check_sha1: self.assertTrue(self.compare_hashes( self.client_bam(uuid), self.server_bam(uuid))) return uuid
def test_long_download_options(self): """ Test download mode options to Gene Torrent (-d) """ gt = GeneTorrentInstance( self.resourcedir + "--download", instance_type=InstanceType.GT_DOWNLOAD, add_defaults=False ) (sout, serr) = gt.communicate() self.assertIn("required", serr) self.assertIn("missing", serr) self.assertIn("download", serr) self.assertEqual(gt.returncode, 9) gt = GeneTorrentInstance( self.resourcedir + "--download xxx", instance_type=InstanceType.GT_DOWNLOAD, add_defaults=False ) (sout, serr) = gt.communicate() self.assertIn("include a credential file", serr) self.assertEqual(gt.returncode, 9) gt = GeneTorrentInstance( self.resourcedir + "--download xxx -c /file/does/not/exist", instance_type=InstanceType.GT_DOWNLOAD, add_defaults=False, ) (sout, serr) = gt.communicate() self.assertIn("credentials file not found", serr) self.assertEqual(gt.returncode, 55) gt = GeneTorrentInstance( self.resourcedir + "--download xxx --credential-file %s" % (self.cred_filename), instance_type=InstanceType.GT_DOWNLOAD, add_defaults=False, ) (sout, serr) = gt.communicate() self.assertIn("'xxx' is too short", serr) self.assertEqual(gt.returncode, 201) gt = GeneTorrentInstance( self.resourcedir + "--credential-file %s --path /path/does/not/exist --download xxx" % (self.cred_filename), instance_type=InstanceType.GT_DOWNLOAD, add_defaults=False, ) (sout, serr) = gt.communicate() self.assertIn("unable to opening directory", serr) self.assertEqual(gt.returncode, 9) gt = GeneTorrentInstance( self.resourcedir + "--credential-file %s --download xxx --security-api=aaa" % (self.cred_filename), instance_type=InstanceType.GT_DOWNLOAD, add_defaults=False, ) (sout, serr) = gt.communicate() self.assertIn("Invalid URI for '--security-api", serr) self.assertEqual(gt.returncode, 9) gt = GeneTorrentInstance( self.resourcedir + "--download xxx --max-children=0", instance_type=InstanceType.GT_DOWNLOAD, add_defaults=False, ) (sout, serr) = gt.communicate() self.assertIn("must be greater than 0", serr) self.assertEqual(gt.returncode, 9)
def test_upload_download_client_timeout(self): '''Test upload/download client inactivity timeout.''' uuid = uuid4() self.generate_bam_data(uuid, 1024 * 1024 * 1) # no server in this test start_time = time.clock() # prepare upload client client_manifest = os.path.join( 'client', str(uuid), 'manifest-generated.xml', ) client = GeneTorrentInstance('-u %s -p client -c %s ' \ '-k 1 -vv -l stdout:full -R . --ssl-no-verify-ca' \ % (client_manifest, self.cred_filename), instance_type=InstanceType.GT_UPLOAD, add_defaults=False) time.sleep(5) # wait for upload client to exit client_sout, client_serr = client.communicate() if TestConfig.VERBOSE: print 'client stdout: ' + client_sout print 'client stderr: ' + client_serr self.assertEqual(client.returncode, 206) self.assertTrue('Inactivity timeout triggered' in client_sout) # upload client properly timed out # now, test download client # delete bam first, else client will exit since it has the file client_bam = self.client_bam(uuid) os.remove(client_bam) client = GeneTorrentInstance( \ '-d %s/cghub/data/analysis/%s -p client2 -c %s ' \ '-k 1 -vv -l stdout:full -R . --ssl-no-verify-ca' \ % (TestConfig.HUB_SERVER, str(uuid), self.cred_filename), instance_type=InstanceType.GT_DOWNLOAD, add_defaults=False) time.sleep(5) # wait for download client to exit client_sout, client_serr = client.communicate() if TestConfig.VERBOSE: print 'client stdout: ' + client_sout print 'client stderr: ' + client_serr self.assertEqual(client.returncode, 206) self.assertTrue('Inactivity timeout triggered' in client_sout) end_time = time.clock() # that should take a little more than 2 minutes, # allowing for two 60-second timeouts self.assertTrue(end_time - start_time < 200, 'Two inactivity timeouts took longer than two minutes')