class TestMultiMongo(TestBase): def setUp(self): super(TestMultiMongo, self).setUp() self.setupDB2() schema = { 'author': { 'type': 'string' }, 'title': { 'type': 'string' }, } settings = {'schema': schema, 'mongo_prefix': 'MONGO1'} self.app.register_resource('works', settings) def tearDown(self): super(TestMultiMongo, self).tearDown() self.dropDB2() def setupDB2(self): self.connection = MongoClient() self.connection.drop_database(MONGO1_DBNAME) self.connection[MONGO1_DBNAME].add_user(MONGO1_USERNAME, MONGO1_PASSWORD) self.bulk_insert2() def dropDB2(self): self.connection = MongoClient() self.connection.drop_database(MONGO1_DBNAME) self.connection.close() def bulk_insert2(self): _db = self.connection[MONGO1_DBNAME] works = self.random_works(self.known_resource_count) _db.works.insert(works) self.work = _db.works.find_one() self.connection.close() def random_works(self, num): works = [] for i in range(num): dt = datetime.now() work = { 'author': self.random_string(20), 'title': self.random_string(30), eve.LAST_UPDATED: dt, eve.DATE_CREATED: dt, } works.append(work) return works
class TestMultiMongo(TestBase): def setUp(self): super(TestMultiMongo, self).setUp() self.setupDB2() schema = { 'author': {'type': 'string'}, 'title': {'type': 'string'}, } settings = { 'schema': schema, 'mongo_prefix': 'MONGO1' } self.app.register_resource('works', settings) def tearDown(self): super(TestMultiMongo, self).tearDown() self.dropDB2() def setupDB2(self): self.connection = MongoClient() self.connection.drop_database(MONGO1_DBNAME) self.connection[MONGO1_DBNAME].add_user(MONGO1_USERNAME, MONGO1_PASSWORD) self.bulk_insert2() def dropDB2(self): self.connection = MongoClient() self.connection.drop_database(MONGO1_DBNAME) self.connection.close() def bulk_insert2(self): _db = self.connection[MONGO1_DBNAME] works = self.random_works(self.known_resource_count) _db.works.insert(works) self.work = _db.works.find_one() self.connection.close() def random_works(self, num): works = [] for i in range(num): dt = datetime.now() work = { 'author': self.random_string(20), 'title': self.random_string(30), eve.LAST_UPDATED: dt, eve.DATE_CREATED: dt, } works.append(work) return works
def clear_db_partially(): logging.warn("dropping all tables in database EXCEPT clinical and genomic") # connect to database. #connection = MongoClient(MONGO_HOST, MONGO_PORT) connection = MongoClient(MONGO_URI) # establish connection. if MONGO_USERNAME: connection[MONGO_DBNAME].add_user(MONGO_USERNAME, MONGO_PASSWORD) # setup the db pointer db = connection[MONGO_DBNAME] # clear the database. db.drop_collection("user") db.drop_collection("team") db.drop_collection("filter") db.drop_collection("match") connection.close()
def clear_db_partially(): logging.warn("dropping all tables in database EXCEPT clinical and genomic") # connect to database. #connection = MongoClient(MONGO_HOST, MONGO_PORT) connection = MongoClient(MONGO_URI) # establish connection. if MONGO_USERNAME: connection[MONGO_DBNAME].add_user(MONGO_USERNAME, MONGO_PASSWORD) # setup the db pointer db = connection[MONGO_DBNAME] # clear the database. db.drop_collection("user") db.drop_collection("team") db.drop_collection("filter") db.drop_collection("match") connection.close()
def add_simulated_sv(): # connect to database. #connection = MongoClient(MONGO_HOST, MONGO_PORT) connection = MongoClient(MONGO_URI) # setup the db pointer db = connection[MONGO_DBNAME] # fetch all patients. patients = list(db.clinical.find()) pidx = range(len(patients)) # loop over each synoymm svs = list() from matchminer.constants import synonyms for key in synonyms: for val in synonyms[key]: # get clinical_id. idx = random.choice(pidx) clinical_id = str(patients[idx]['_id']) sample_id = str(patients[idx]['SAMPLE_ID']) # make a SV. sv = { 'CLINICAL_ID': clinical_id, 'VARIANT_CATEGORY': 'SV', 'STRUCTURAL_VARIANT_COMMENT': "tmp6654 " + val, 'WILDTYPE': False, 'SAMPLE_ID': sample_id } svs.append(sv) # return it. connection.close() return svs
def add_simulated_sv(): # connect to database. #connection = MongoClient(MONGO_HOST, MONGO_PORT) connection = MongoClient(MONGO_URI) # setup the db pointer db = connection[MONGO_DBNAME] # fetch all patients. patients = list(db.clinical.find()) pidx = range(len(patients)) # loop over each synoymm svs = list() from matchminer.constants import synonyms for key in synonyms: for val in synonyms[key]: # get clinical_id. idx = random.choice(pidx) clinical_id = str(patients[idx]['_id']) sample_id = str(patients[idx]['SAMPLE_ID']) # make a SV. sv = { 'CLINICAL_ID': clinical_id, 'VARIANT_CATEGORY': 'SV', 'STRUCTURAL_VARIANT_COMMENT': "tmp6654 " + val, 'WILDTYPE': False, 'SAMPLE_ID': sample_id } svs.append(sv) # return it. connection.close() return svs
def bootstrap(ctx, forced=False): # get the current time. cur_dt = formatdate(time.mktime(datetime.now().timetuple())) # connect to database. #connection = MongoClient(MONGO_HOST, MONGO_PORT) connection = MongoClient(MONGO_URI) # establish connection. if MONGO_USERNAME: connection[MONGO_DBNAME].add_user(MONGO_USERNAME, MONGO_PASSWORD) # setup the db pointer db = connection[MONGO_DBNAME] # check if the clinical and genomic empty. if db[COLLECTION_CLINICAL].count() > 0 or db[COLLECTION_GENOMIC].count() > 0: if not forced: logging.warn("attempting to bootstrap the database without forcing") return logging.warn("forcibly resetting the database, may take some time") # clear the database. clear_db_completely() connection.close() # initialize user. init_user_debug(ctx=ctx) # load the clinical_df clinical_df = clinical_load(DATA_CLINICAL_CSV) # store unique keys for every sample. keys = {} # insert each clinical entry new_clinical = 0 for entry in clinical_gen(clinical_df, clinical_schema=clinical['schema']): # insert into db the clinical data. resp, status_code = utility_post(ctx, 'clinical', entry, token='abc123') if status_code != 201: logging.error("unable to insert") assert status_code == 201 new_clinical += 1 # get the unique key. key = resp['_id'] # save it for later linking to genomics keys[entry['SAMPLE_ID']] = key # load the genomic_df. genomic_df = genomic_load(DATA_GENOMIC_CSV) # insert each genomic entry. i = 0 new_genomic = 0 for entry in genomics_gen(genomic_df, genomic_schema=genomic['schema']): # must have a clinical entry. if entry['SAMPLE_ID'] not in keys: continue # hook entry with clinical_id. id = keys[entry['SAMPLE_ID']] entry['CLINICAL_ID'] = id # insert it into db. entry['WILDTYPE'] = bool(entry['WILDTYPE']) resp, status_code = utility_post(ctx, 'genomic', entry, token='abc123') if status_code != 201: logging.error("unable to insert") assert status_code == 201 #if i > 1000: # break i += 1 new_genomic += 1 # post the status update. status = { 'last_update': cur_dt, 'new_clinical': new_clinical, 'updated_clinical': 0, 'new_genomic': new_genomic, 'updated_genomic': 0, 'silent': True } resp, status_code = utility_post(ctx, 'status', status, token='abc123') if status_code != 201: logging.error("unable to insert") assert status_code == 201
class TestMinimal(unittest.TestCase): """ Start the building of the tests for an application based on Eve by subclassing this class and provide proper settings using :func:`setUp()` """ app = ValueStack(close_pymongo_connection) def setUp(self, settings_file=None, url_converters=None): """ Prepare the test fixture :param settings_file: the name of the settings file. Defaults to `eve/tests/test_settings.py`. """ self.this_directory = os.path.dirname(os.path.realpath(__file__)) if settings_file is None: # Load the settings file, using a robust path settings_file = os.path.join(self.this_directory, 'test_settings.py') self.connection = None self.known_resource_count = 101 self.setupDB() self.settings_file = settings_file self.app = eve.Eve(settings=self.settings_file, url_converters=url_converters) self.test_client = self.app.test_client() self.domain = self.app.config['DOMAIN'] def tearDown(self): del self.app self.dropDB() def assert200(self, status): self.assertEqual(status, 200) def assert201(self, status): self.assertEqual(status, 201) def assert301(self, status): self.assertEqual(status, 301) def assert404(self, status): self.assertEqual(status, 404) def assert304(self, status): self.assertEqual(status, 304) def get(self, resource, query='', item=None): if resource in self.domain: resource = self.domain[resource]['url'] if item: request = '/%s/%s%s' % (resource, item, query) else: request = '/%s%s' % (resource, query) r = self.test_client.get(request) return self.parse_response(r) def post(self, url, data, headers=None, content_type='application/json'): if headers is None: headers = [] headers.append(('Content-Type', content_type)) r = self.test_client.post(url, data=json.dumps(data), headers=headers) return self.parse_response(r) def put(self, url, data, headers=None): if headers is None: headers = [] headers.append(('Content-Type', 'application/json')) r = self.test_client.put(url, data=json.dumps(data), headers=headers) return self.parse_response(r) def patch(self, url, data, headers=None): if headers is None: headers = [] headers.append(('Content-Type', 'application/json')) r = self.test_client.patch(url, data=json.dumps(data), headers=headers) return self.parse_response(r) def delete(self, url, headers=None): r = self.test_client.delete(url, headers=headers) return self.parse_response(r) def parse_response(self, r): try: v = json.loads(r.get_data()) except json.JSONDecodeError: v = None return v, r.status_code def assertValidationError(self, response, matches): self.assertTrue(eve.STATUS in response) self.assertTrue(eve.STATUS_ERR in response[eve.STATUS]) self.assertTrue(ISSUES in response) issues = response[ISSUES] self.assertTrue(len(issues)) for k, v in matches.items(): self.assertTrue(k in issues) self.assertTrue(v in issues[k]) def assertExpires(self, resource): # TODO if we ever get access to response.date (it is None), compare # it with Expires r = self.test_client.get(resource) expires = r.headers.get('Expires') self.assertTrue(expires is not None) def assertCacheControl(self, resource): r = self.test_client.get(resource) cache_control = r.headers.get('Cache-Control') self.assertTrue(cache_control is not None) self.assertEqual(cache_control, self.domain[self.known_resource]['cache_control']) def assertIfModifiedSince(self, resource): r = self.test_client.get(resource) last_modified = r.headers.get('Last-Modified') self.assertTrue(last_modified is not None) r = self.test_client.get(resource, headers=[('If-Modified-Since', last_modified)]) self.assert304(r.status_code) self.assertTrue(not r.get_data()) def assertItem(self, item): self.assertEqual(type(item), dict) updated_on = item.get(self.app.config['LAST_UPDATED']) self.assertTrue(updated_on is not None) try: datetime.strptime(updated_on, self.app.config['DATE_FORMAT']) except Exception as e: self.fail('Cannot convert field "%s" to datetime: %s' % (self.app.config['LAST_UPDATED'], e)) created_on = item.get(self.app.config['DATE_CREATED']) self.assertTrue(updated_on is not None) try: datetime.strptime(created_on, self.app.config['DATE_FORMAT']) except Exception as e: self.fail('Cannot convert field "%s" to datetime: %s' % (self.app.config['DATE_CREATED'], e)) link = item.get('_links') _id = item.get(self.app.config['ID_FIELD']) self.assertItemLink(link, _id) def assertPagination(self, response, page, total, max_results): self.assertTrue(self.app.config['META'] in response) meta = response.get(self.app.config['META']) self.assertTrue('page' in meta) self.assertTrue('max_results' in meta) self.assertTrue('total' in meta) self.assertEqual(meta['page'], page) self.assertEqual(meta['max_results'], max_results) self.assertEqual(meta['total'], total) def assertHomeLink(self, links): self.assertTrue('parent' in links) link = links['parent'] self.assertTrue('title' in link) self.assertTrue('href' in link) self.assertEqual('home', link['title']) self.assertEqual("%s" % self._get_server_name(), link['href']) def assertResourceLink(self, links, resource): self.assertTrue('self' in links) link = links['self'] self.assertTrue('title' in link) self.assertTrue('href' in link) url = self.domain[resource]['url'] self.assertEqual(url, link['title']) self.assertEqual("%s/%s" % (self._get_server_name(), url), link['href']) def assertCollectionLink(self, links, resource): self.assertTrue('collection' in links) link = links['collection'] self.assertTrue('title' in link) self.assertTrue('href' in link) url = self.domain[resource]['url'] self.assertEqual(url, link['title']) self.assertEqual("%s/%s" % (self._get_server_name(), url), link['href']) def assertNextLink(self, links, page): self.assertTrue('next' in links) link = links['next'] self.assertTrue('title' in link) self.assertTrue('href' in link) self.assertEqual('next page', link['title']) self.assertTrue("page=%d" % page in link['href']) def assertPrevLink(self, links, page): self.assertTrue('prev' in links) link = links['prev'] self.assertTrue('title' in link) self.assertTrue('href' in link) self.assertEqual('previous page', link['title']) if page > 1: self.assertTrue("page=%d" % page in link['href']) def assertItemLink(self, links, item_id): self.assertTrue('self' in links) link = links['self'] # TODO we are too deep here to get a hold of the due title. Should fix. self.assertTrue('title' in link) self.assertTrue('href' in link) self.assertTrue('/%s' % item_id in link['href']) def assertLastLink(self, links, page): if page: self.assertTrue('last' in links) link = links['last'] self.assertTrue('title' in link) self.assertTrue('href' in link) self.assertEqual('last page', link['title']) self.assertTrue("page=%d" % page in link['href']) else: self.assertTrue('last' not in links) def assert400(self, status): self.assertEqual(status, 400) def assert401(self, status): self.assertEqual(status, 401) def assert401or405(self, status): self.assertTrue(status == 401 or 405) def assert403(self, status): self.assertEqual(status, 403) def assert405(self, status): self.assertEqual(status, 405) def assert412(self, status): self.assertEqual(status, 412) def assert500(self, status): self.assertEqual(status, 500) def setupDB(self): self.connection = MongoClient(MONGO_HOST, MONGO_PORT) self.connection.drop_database(MONGO_DBNAME) if MONGO_USERNAME: self.connection[MONGO_DBNAME].add_user(MONGO_USERNAME, MONGO_PASSWORD) self.bulk_insert() def bulk_insert(self): pass def _get_server_name(self): server_name = self.app.config.get('SERVER_NAME', '') url_protocol = self.app.config.get('URL_PROTOCOL', '') if url_protocol: server_name = '%s://%s' % (url_protocol, server_name) return server_name def dropDB(self): self.connection = MongoClient(MONGO_HOST, MONGO_PORT) self.connection.drop_database(MONGO_DBNAME) self.connection.close()
class TestMinimal(unittest.TestCase): """ Start the building of the tests for an application based on Eve by subclassing this class and provide proper settings using :func:`setUp()` """ def setUp(self, settings_file='eve/tests/test_settings.py'): """ Prepare the test fixture :param settings_file: the name of the settings file. Defaults to `eve/tests/test_settings.py`. """ self.known_resource_count = 101 self.setupDB() self.settings_file = settings_file self.app = eve.Eve(settings=self.settings_file) self.test_client = self.app.test_client() self.domain = self.app.config['DOMAIN'] def tearDown(self): self.dropDB() def assert200(self, status): self.assertEqual(status, 200) def assert301(self, status): self.assertEqual(status, 301) def assert404(self, status): self.assertEqual(status, 404) def assert304(self, status): self.assertEqual(status, 304) def get(self, resource, query='', item=None): if resource in self.domain: resource = self.domain[resource]['url'] if item: request = '/%s/%s/' % (resource, item) else: request = '/%s/%s' % (resource, query) r = self.test_client.get(request) return self.parse_response(r) def parse_response(self, r): v = json.loads(r.get_data()) if r.status_code == 200 else None return v, r.status_code def assertValidationError(self, response, key, matches): self.assertTrue(key in response) k = response[key] self.assertTrue('status' in k) self.assertTrue(eve.STATUS_ERR in k['status']) self.assertTrue('issues' in k) issues = k['issues'] self.assertTrue(len(issues)) for match in matches: self.assertTrue(match in issues[0]) def assertExpires(self, resource): # TODO if we ever get access to response.date (it is None), compare # it with Expires r = self.test_client.get(resource) expires = r.headers.get('Expires') self.assertTrue(expires is not None) def assertCacheControl(self, resource): r = self.test_client.get(resource) cache_control = r.headers.get('Cache-Control') self.assertTrue(cache_control is not None) self.assertEqual(cache_control, self.domain[self.known_resource]['cache_control']) def assertIfModifiedSince(self, resource): r = self.test_client.get(resource) last_modified = r.headers.get('Last-Modified') self.assertTrue(last_modified is not None) r = self.test_client.get(resource, headers=[('If-Modified-Since', last_modified)]) self.assert304(r.status_code) self.assertTrue(not r.get_data()) def assertItem(self, item): self.assertEqual(type(item), dict) _id = item.get(self.app.config['ID_FIELD']) self.assertTrue(_id is not None) match = re.compile(self.app.config['ITEM_URL']).match(_id) self.assertTrue(match is not None) self.assertEqual(match.group(), _id) updated_on = item.get(self.app.config['LAST_UPDATED']) self.assertTrue(updated_on is not None) try: datetime.strptime(updated_on, self.app.config['DATE_FORMAT']) except Exception as e: self.fail('Cannot convert field "%s" to datetime: %s' % (self.app.config['LAST_UPDATED'], e)) link = item.get('_links') self.assertItemLink(link, _id) def assertHomeLink(self, links): self.assertTrue('parent' in links) link = links['parent'] self.assertTrue('title' in link) self.assertTrue('href' in link) self.assertEqual('home', link['title']) self.assertEqual("%s" % self.app.config['SERVER_NAME'], link['href']) def assertResourceLink(self, links, resource): self.assertTrue('self' in links) link = links['self'] self.assertTrue('title' in link) self.assertTrue('href' in link) url = self.domain[resource]['url'] self.assertEqual(url, link['title']) self.assertEqual("%s/%s/" % (self.app.config['SERVER_NAME'], url), link['href']) def assertCollectionLink(self, links, resource): self.assertTrue('collection' in links) link = links['collection'] self.assertTrue('title' in link) self.assertTrue('href' in link) url = self.domain[resource]['url'] self.assertEqual(url, link['title']) self.assertEqual("%s/%s/" % (self.app.config['SERVER_NAME'], url), link['href']) def assertNextLink(self, links, page): self.assertTrue('next' in links) link = links['next'] self.assertTrue('title' in link) self.assertTrue('href' in link) self.assertEqual('next page', link['title']) self.assertTrue("page=%d" % page in link['href']) def assertPrevLink(self, links, page): self.assertTrue('prev' in links) link = links['prev'] self.assertTrue('title' in link) self.assertTrue('href' in link) self.assertEqual('previous page', link['title']) if page > 1: self.assertTrue("page=%d" % page in link['href']) def assertItemLink(self, links, item_id): self.assertTrue('self' in links) link = links['self'] #TODO we are too deep here to get a hold of the due title. Should fix. self.assertTrue('title' in link) self.assertTrue('href' in link) self.assertTrue('/%s/' % item_id in link['href']) def assertLastLink(self, links, page): if page: self.assertTrue('last' in links) link = links['last'] self.assertTrue('title' in link) self.assertTrue('href' in link) self.assertEqual('last page', link['title']) self.assertTrue("page=%d" % page in link['href']) else: self.assertTrue('last' not in links) def assert400(self, status): self.assertEqual(status, 400) def assert401(self, status): self.assertEqual(status, 401) def assert401or405(self, status): self.assertTrue(status == 401 or 405) def assert403(self, status): self.assertEqual(status, 403) def assert405(self, status): self.assertEqual(status, 405) def assert412(self, status): self.assertEqual(status, 412) def assert500(self, status): self.assertEqual(status, 500) def setupDB(self): self.connection = MongoClient(MONGO_HOST, MONGO_PORT) self.connection.drop_database(MONGO_DBNAME) if MONGO_USERNAME: self.connection[MONGO_DBNAME].add_user(MONGO_USERNAME, MONGO_PASSWORD) self.bulk_insert() def bulk_insert(self): pass def dropDB(self): self.connection = MongoClient(MONGO_HOST, MONGO_PORT) self.connection.drop_database(MONGO_DBNAME) self.connection.close()
class TestBase(unittest.TestCase): def setUp(self): """ Prepare the test fixture """ self.setupDB() self.app = run.app self.domain = self.app.config['DOMAIN'] self.test_client = self.app.test_client() self.valid_auth = [('Authorization', 'Basic dG9rZW4xOg==')] self.headers = [('Content-Type', 'application/json')] + self.valid_auth def tearDown(self): self.dropDB() def assert200(self, status): self.assertEqual(status, 200) def assert201(self, status): self.assertEqual(status, 201) def assert204(self, status): self.assertEqual(status, 204) def assert401(self, status): self.assertEqual(status, 401) def setupDB(self): self.connection = MongoClient(MONGO_HOST, MONGO_PORT) self.connection.drop_database(MONGO_DBNAME) if MONGO_USERNAME: self.connection[MONGO_DBNAME].add_user(MONGO_USERNAME, MONGO_PASSWORD) db = self.connection[MONGO_DBNAME] self.accounts_insert(db) self.company_insert(db) db.connection.close() def accounts_insert(self, db): user_accounts = [ {'u': '*****@*****.**', 'p': 'pw1', 't': 'token1', 'r': ['app']}, {'u': '*****@*****.**', 'p': 'pw1', 't': 'token2', 'r': ['user']}, ] db.user_accounts.insert(user_accounts) def company_insert(self, db): company = {'n': 'test_company', 'p': 'pw1'} self.company_id = str(db.companies.insert(company)) def dropDB(self): self.connection = MongoClient(MONGO_HOST, MONGO_PORT) self.connection.drop_database(MONGO_DBNAME) self.connection.close() def post(self, url, payload): r = self.test_client.post(url, data=json.dumps(payload), headers=self.headers) return self.parse_response(r) def put(self, url, payload, etag): headers = self.headers + [('If-Match', etag)] r = self.test_client.put(url, data=json.dumps(payload), headers=headers) return self.parse_response(r) def delete(self, url, etag): headers = self.headers + [('If-Match', etag)] r = self.test_client.delete(url, headers=headers) return self.parse_response(r) def get(self, url): r = self.test_client.get(url, headers=self.headers) return self.parse_response(r) def parse_response(self, r): try: v = json.loads(r.get_data()) except json.JSONDecodeError: v = None return v, r.status_code
class TestBase(unittest.TestCase): def setUp(self): self.setupDB() self.settings_file = 'eve/tests/test_settings.py' self.app = Eve(settings=self.settings_file) self.test_client = self.app.test_client() self.domain = self.app.config['DOMAIN'] self.known_resource = 'contacts' self.known_resource_url = ('/%s/' % self.domain[self.known_resource]['url']) self.empty_resource = 'empty' self.empty_resource_url = '/%s/' % self.empty_resource self.unknown_resource = 'unknown' self.unknown_resource_url = '/%s/' % self.unknown_resource self.unknown_item_id = '4f46445fc88e201858000000' self.unknown_item_name = 'unknown' self.unknown_item_id_url = ( '/%s/%s/' % (self.domain[self.known_resource]['url'], self.unknown_item_id)) self.unknown_item_name_url = ( '/%s/%s/' % (self.domain[self.known_resource]['url'], self.unknown_item_name)) self.readonly_resource = 'payments' self.readonly_resource_url = ( '/%s/' % self.domain[self.readonly_resource]['url']) self.different_resource = 'users' self.different_resource_url = ( '/%s/' % self.domain[self.different_resource]['url']) def tearDown(self): self.dropDB() def assert200(self, status): self.assertEqual(status, 200) def assert301(self, status): self.assertEqual(status, 301) def assert404(self, status): self.assertEqual(status, 404) def assert304(self, status): self.assertEqual(status, 304) def setupDB(self): self.connection = MongoClient() self.connection.drop_database(MONGO_DBNAME) self.connection[MONGO_DBNAME].add_user(MONGO_USERNAME, MONGO_PASSWORD) self.bulk_insert() def bulk_insert(self): _db = self.connection[MONGO_DBNAME] _db.contacts.insert(self.random_contacts(100)) _db.contacts.insert(self.random_users(2)) _db.payments.insert(self.random_payments(10)) _db.invoices.insert(self.random_invoices(1)) self.connection.close() def dropDB(self): self.connection = MongoClient() self.connection.drop_database(MONGO_DBNAME) self.connection.close() def random_contacts(self, num, standard_date_fields=True): schema = DOMAIN['contacts']['schema'] contacts = [] for i in range(num): dt = datetime.now() contact = { 'ref': self.random_string(schema['ref']['maxlength']), 'prog': i, 'role': random.choice(schema['role']['allowed']), 'rows': self.random_rows(random.randint(0, 5)), 'alist': self.random_list(random.randint(0, 5)), 'location': { 'address': 'address ' + self.random_string(5), 'city': 'city ' + self.random_string(3), }, 'born': datetime.today() + timedelta(days=random.randint(-10, 10)), 'tid': ObjectId(), } if standard_date_fields: contact[eve.LAST_UPDATED] = dt contact[eve.DATE_CREATED] = dt contacts.append(contact) return contacts def random_users(self, num): users = self.random_contacts(num) for user in users: user['username'] = self.random_string(10) return users def random_payments(self, num): payments = [] for i in range(num): dt = datetime.now() payment = { 'a_string': self.random_string(10), 'a_number': i, eve.LAST_UPDATED: dt, eve.DATE_CREATED: dt, } payments.append(payment) return payments def random_invoices(self, num): invoices = [] for i in range(num): dt = datetime.now() invoice = { 'inv_number': self.random_string(10), eve.LAST_UPDATED: dt, eve.DATE_CREATED: dt, } invoices.append(invoice) return invoices def random_string(self, num): return (''.join( random.choice(string.ascii_uppercase) for x in range(num))) def random_list(self, num): alist = [] for i in range(num): alist.append(['string' + str(i), random.randint(1000, 9999)]) return alist def random_rows(self, num): schema = DOMAIN['contacts']['schema']['rows']['schema']['schema'] rows = [] for i in range(num): rows.append({ 'sku': self.random_string(schema['sku']['maxlength']), 'price': random.randint(100, 1000), }) return rows
class TestMinimal(unittest.TestCase): """ Start the building of the tests for an application based on Eve by subclassing this class and provide proper settings using :func:`setUp()` """ app = ValueStack(close_pymongo_connection) inserted_clinical = list() inserted_genomic = list() def setUp(self, settings_file=None, url_converters=None): """ Prepare the test fixture :param settings_file: the name of the settings file. Defaults to `eve/tests/test_settings.py`. """ # settings. self.this_directory = os.path.dirname(os.path.realpath(__file__)) if settings_file is None: # Load the settings file, using a robust path settings_file = os.path.join(self.this_directory, 'test_settings.py') # default values. self.connection = None # create the engine. self.cbio = engine.CBioEngine(MONGO_URI, MONGO_DBNAME, data_model.match_schema, muser=MONGO_USERNAME, mpass=MONGO_PASSWORD, collection_clinical=COLLECTION_CLINICAL, collection_genomic=COLLECTION_GENOMIC) # setup the database. self.setupDB() # prepare the app self.settings_file = settings_file self.app = eve.Eve(settings=self.settings_file, url_converters=url_converters, auth=security.TokenAuth, validator=ConsentValidatorEve) # create the test client self.test_client = self.app.test_client() # set domain. self.domain = self.app.config['DOMAIN'] # register hooks self.app = register_hooks(self.app) # register blueprints. self.app.register_blueprint(blueprint) # create the database pointer self.db = self.connection[self.app.config['MONGO_DBNAME']] # initialize the database if necessary. self.initDB() # clear backup. # shutil.rmtree(BACKUP_DIR) # os.makedirs(BACKUP_DIR) def tearDown(self): del self.app self.dropDB() def _insert_match_small(self): # make a complex query. dt = formatdate(time.mktime(datetime.datetime(year=1995, month=1, day=1).timetuple())) c = { "BIRTH_DATE": {"$gte": dt}, } g = { "TRUE_HUGO_SYMBOL": "BRCA2" } rule = { 'USER_ID': self.user_id, 'TEAM_ID': self.team_id, 'clinical_filter': c, 'genomic_filter': g, 'label': 'test', 'status': 1, 'temporary': False } # insert it. r, status_code = self.post('filter', rule) self.assert201(status_code) # return the id. return r['_id'] def _insert_filter(self): # create a filter. g = { 'TRUE_HUGO_SYMBOL': 'PRPF8', 'VARIANT_CATEGORY': 'MUTATION', 'WILDTYPE': False } rule = { 'USER_ID': self.user_id, 'TEAM_ID': self.team_id, 'genomic_filter': g, 'label': 'test', 'temporary': False, 'status': 1 } r, status_code = self.post('filter', rule) self.assert201(status_code) # return the id. return r['_id'] def _get_genomic_records(self, clinical_id): # get all results. return list(self.db['genomic'].find({'CLINICAL_ID': clinical_id})) def _get_match_records(self, clinical_id): # get all results. return list(self.db['match'].find({'CLINICAL_ID': clinical_id})) def assert200(self, status): self.assertEqual(status, 200) def assert201(self, status): self.assertEqual(status, 201) def assert204(self, status): self.assertEqual(status, 204) def assert301(self, status): self.assertEqual(status, 301) def assert304(self, status): self.assertEqual(status, 304) def assert404(self, status): self.assertEqual(status, 404) def assert422(self, status): self.assertEqual(status, 422) def assert405(self, status): self.assertEqual(status, 405) def _make_headers(self, headers=[]): # add the defaults headers.append(('Content-Type', 'application/json')) headers.append(('Authorization', 'Basic ' + str(base64.b64encode(self.user_token + ':')))) # just return list. return headers def get(self, resource, query='', item=None): # make headers. headers = self._make_headers() if resource in self.domain: resource = self.domain[resource]['url'] if item: request = '/%s/%s%s' % (resource, item, query) else: request = '/%s%s' % (resource, query) r = self.test_client.get('api' + request, headers=headers) return self.parse_response(r) def post(self, url, data, headers=None, content_type='application/json'): # clear object ids. for key in data: try: if isinstance(data[key], ObjectId): data[key] = str(data[key]) except TypeError: continue # make headers. if headers is None: headers = self._make_headers() else: headers = self._make_headers(headers=headers) r = self.test_client.post('api/' + url, data=json.dumps(data), headers=headers) return self.parse_response(r) def put(self, url, data, headers=None): # make headers. if headers is None: headers = self._make_headers() else: headers = self._make_headers(headers=headers) r = self.test_client.put('api/' + url, data=json.dumps(data), headers=headers) return self.parse_response(r) def patch(self, url, data, headers=None): # make headers. if headers is None: headers = self._make_headers() else: headers = self._make_headers(headers=headers) r = self.test_client.patch('api/' + url, data=json.dumps(data), headers=headers) return self.parse_response(r) def delete(self, url, headers=None): # make headers. if headers is None: headers = self._make_headers() else: headers = self._make_headers(headers=headers) r = self.test_client.delete('api/' + url, headers=headers) return self.parse_response(r) def parse_response(self, r): try: v = json.loads(r.get_data()) except ValueError: v = None return v, r.status_code def assertValidationErrorStatus(self, status): self.assertEqual(status, self.app.config.get('VALIDATION_ERROR_STATUS')) def assertValidationError(self, response, matches): self.assertTrue(eve.STATUS in response) self.assertTrue(eve.STATUS_ERR in response[eve.STATUS]) self.assertTrue(ISSUES in response) issues = response[ISSUES] self.assertTrue(len(issues)) for k, v in matches.items(): self.assertTrue(k in issues) self.assertTrue(v in issues[k]) def assertExpires(self, resource): # TODO if we ever get access to response.date (it is None), compare # it with Expires r = self.test_client.get(resource) expires = r.headers.get('Expires') self.assertTrue(expires is not None) def assertCacheControl(self, resource): r = self.test_client.get(resource) cache_control = r.headers.get('Cache-Control') self.assertTrue(cache_control is not None) self.assertEqual(cache_control, self.domain[self.known_resource]['cache_control']) def assertIfModifiedSince(self, resource): r = self.test_client.get(resource) last_modified = r.headers.get('Last-Modified') self.assertTrue(last_modified is not None) r = self.test_client.get(resource, headers=[('If-Modified-Since', last_modified)]) self.assert304(r.status_code) self.assertTrue(not r.get_data()) def assertItem(self, item, resource): self.assertEqual(type(item), dict) updated_on = item.get(self.app.config['LAST_UPDATED']) self.assertTrue(updated_on is not None) try: datetime.strptime(updated_on, self.app.config['DATE_FORMAT']) except Exception as e: self.fail('Cannot convert field "%s" to datetime: %s' % (self.app.config['LAST_UPDATED'], e)) created_on = item.get(self.app.config['DATE_CREATED']) self.assertTrue(updated_on is not None) try: datetime.strptime(created_on, self.app.config['DATE_FORMAT']) except Exception as e: self.fail('Cannot convert field "%s" to datetime: %s' % (self.app.config['DATE_CREATED'], e)) link = item.get('_links') _id = item.get(self.domain[resource]['id_field']) self.assertItemLink(link, _id) def assertPagination(self, response, page, total, max_results): p_key, mr_key = self.app.config['QUERY_PAGE'], \ self.app.config['QUERY_MAX_RESULTS'] self.assertTrue(self.app.config['META'] in response) meta = response.get(self.app.config['META']) self.assertTrue(p_key in meta) self.assertTrue(mr_key in meta) self.assertTrue('total' in meta) self.assertEqual(meta[p_key], page) self.assertEqual(meta[mr_key], max_results) self.assertEqual(meta['total'], total) def assertHomeLink(self, links): self.assertTrue('parent' in links) link = links['parent'] self.assertTrue('title' in link) self.assertTrue('href' in link) self.assertEqual('home', link['title']) self.assertEqual("/", link['href']) def assertResourceLink(self, links, resource): self.assertTrue('self' in links) link = links['self'] self.assertTrue('title' in link) self.assertTrue('href' in link) url = self.domain[resource]['url'] self.assertEqual(url, link['title']) self.assertEqual("%s" % url, link['href']) def assertCollectionLink(self, links, resource): self.assertTrue('collection' in links) link = links['collection'] self.assertTrue('title' in link) self.assertTrue('href' in link) url = self.domain[resource]['url'] self.assertEqual(url, link['title']) self.assertEqual("%s" % url, link['href']) def assertNextLink(self, links, page): self.assertTrue('next' in links) link = links['next'] self.assertTrue('title' in link) self.assertTrue('href' in link) self.assertEqual('next page', link['title']) self.assertTrue("%s=%d" % (self.app.config['QUERY_PAGE'], page) in link['href']) def assertPrevLink(self, links, page): self.assertTrue('prev' in links) link = links['prev'] self.assertTrue('title' in link) self.assertTrue('href' in link) self.assertEqual('previous page', link['title']) if page > 1: self.assertTrue("%s=%d" % (self.app.config['QUERY_PAGE'], page) in link['href']) def assertItemLink(self, links, item_id): self.assertTrue('self' in links) link = links['self'] # TODO we are too deep here to get a hold of the due title. Should fix. self.assertTrue('title' in link) self.assertTrue('href' in link) self.assertTrue('/%s' % item_id in link['href']) def assertLastLink(self, links, page): if page: self.assertTrue('last' in links) link = links['last'] self.assertTrue('title' in link) self.assertTrue('href' in link) self.assertEqual('last page', link['title']) self.assertTrue("%s=%d" % (self.app.config['QUERY_PAGE'], page) in link['href']) else: self.assertTrue('last' not in links) def assert400(self, status): self.assertEqual(status, 400) def assert401(self, status): self.assertEqual(status, 401) def assert401or405(self, status): self.assertTrue(status == 401 or 405) def assert403(self, status): self.assertEqual(status, 403) def assert405(self, status): self.assertEqual(status, 405) def assert412(self, status): self.assertEqual(status, 412) def assert500(self, status): self.assertEqual(status, 500) def initDB(self): # check if the clinical and genomic empty. if self.cbio._c.count() == 0 or self.cbio._g.count() == 0: # bootstrap. bootstrap(self) # add testing user. user_db = self.db['user'] team_db = self.db['team'] team = { "_id": ObjectId("66a52871a8c829842fbe618b"), "name": "purple monkey" } team_db.insert_one(team) self.team_id = team['_id'] # add user and team. user = { "_id": ObjectId("5697ecb48a6ba828126f8128"), "first_name": 'John', "last_name": 'Doe', "title": "M.D. PhD", "email": 'jdoe@demo', "token": 'abc123', "user_name": "jd00", 'teams': [self.team_id], 'roles': ['user'] } self.user_id = user['_id'] self.user_token = user['token'] user_db.insert_one(user) # add service. service = { "first_name": 'SERVICE', "last_name": 'SERVICE', "title": "", "email": 'jdoe@demo', "token": 'xyz946', "user_name": "SERVICE", 'teams': [], 'roles': ['service'] } self.service_id = user_db.insert_one(service) self.service_token = service['token'] # add curator curator = { "first_name": 'CURATOR', "last_name": 'CURATOR', "title": "", "email": 'jdoe@demo', "token": 'kjdhfg76', "user_name": "CURATOR", 'teams': [], 'roles': ['curator'] } self.curator_id = user_db.insert_one(curator) self.curator_token = curator['token'] def setupDB(self): # drop the database. self.dropDB() # connect to database. self.connection = MongoClient(MONGO_URI) # establish connection. if MONGO_USERNAME: self.connection[MONGO_DBNAME].add_user(MONGO_USERNAME, MONGO_PASSWORD) def dropDB(self): #return # connect to database. self.connection = MongoClient(MONGO_URI) # drop all tables except the genomic/clinical db = self.connection[MONGO_DBNAME] db.drop_collection("user") db.drop_collection("team") db.drop_collection("filter") db.drop_collection("match") db.drop_collection("hipaa") db.drop_collection("status") db.drop_collection("trial") db.drop_collection("normalize") db.drop_collection("email") # clear extra clinical and genomic. db['clinical'].delete_many({"TOTAL_READS" : 123, "ORD_PHYSICIAN_NPI": 0000}) db['genomic'].delete_many({'COVERAGE': 9123}) db['genomic'].delete_many({"STRUCTURAL_VARIANT_COMMENT": re.compile("tmp6654.*", re.IGNORECASE)}) for id in self.inserted_clinical: db['clinical'].delete_one({'_id': ObjectId(id)}) for id in self.inserted_genomic: db['genomic'].delete_one({'_id': ObjectId(id)}) # last check. db['clinical'].delete_many({"SAMPLE_ID": "TCGA-OR-TEST1"}) # close connection. self.connection.close() def _insert_pair(self): # insert clinical clinical = self._insert_clinical() # insert genomic. genomic = self._insert_genomic(clinical['_id']) # return the pair. return clinical, genomic def _insert_clinical(self, check=True): # make an entry. clinical = self.clinical.copy() # insert it. r, status_code = self.post('clinical', clinical) if check: self.assert201(status_code) # record it. if status_code == 201: self.inserted_clinical.append(r['_id']) # return it return r def _insert_genomic(self, clinical_id, data=False, check=True): if data is False: genomic = self.genomic.copy() else: genomic = data # insert it. r, status_code = self.post('genomic', genomic) if check: self.assert201(status_code) # record it. if status_code == 201: self.inserted_genomic.append(r['_id']) # return it return r # Simulated Data cur_dt = formatdate(time.mktime(datetime.datetime.now().timetuple())) bir_dt = formatdate(time.mktime(datetime.datetime(year=1995, month=1, day=1).timetuple())) clinical = { '_id': ObjectId(), "PCT_TARGET_BASE": 0.6764308658975878, "LAST_NAME": "Mcginnis[Fake]", "TUMOR_PURITY_PERCENT": 0.6273209625954723, "ONCOTREE_PRIMARY_DIAGNOSIS": "PHC", "ONCOTREE_BIOPSY_SITE": "ADRENAL_GLAND", "FIRST_NAME": "Michael", "PATIENT_ID": "b3164f7b-c826-4e08-9ee6-8ff96d29b913", "DATE_RECEIVED_AT_SEQ_CENTER": cur_dt, "SAMPLE_ID": "TCGA-OR-TEST1", "ONCOTREE_BIOPSY_SITE_TYPE": "Metastatic", "TOTAL_READS": 123, "ONCOTREE_PRIMARY_DIAGNOSIS_NAME": "Pheochromocytoma", "PANEL_VERSION": 2, "ONCOTREE_PRIMARY_DIAGNOSIS_META": "Pheochromocytoma", "TOTAL_ALIGNED_READS": 10716821, "POWERPATH_PATIENT_ID": "b3164f7b-c826-4e08-9ee6-8ff96d29b913", "TEST_TYPE": "11-104 Profile", "ORD_PHYSICIAN_NAME": "Maria Bellantoni [fake] M.D.", "ONCOTREE_BIOPSY_SITE_META": "Pheochromocytoma", "ONCOTREE_BIOPSY_SITE_COLOR": "Purple", "MEAN_SAMPLE_COVERAGE": 147, "VITAL_STATUS": "deceased", "MRN": "b3164f7b-c826-4e08-9ee6-8ff96d29b913", "ONCOTREE_BIOPSY_SITE_NAME": "Adrenal Gland", "LAST_VISIT_DATE": cur_dt, "REPORT_COMMENT": "age_at_initial_pathologic_diagnosis 58 ct_scan nan ct_scan_findings nan days_to_initial_pathologic_diagnosis 0 excess_adrenal_hormone_diagnosis_method_type nan excess_adrenal_hormone_history_type mineralocorticoids excess_adrenal_hormone_history_type-2 nan max nan nf1 nan nf1_clinical_diagnosis nan ret nan sdha nan sdhaf2_sdh5 nan sdhb nan sdhc nan sdhd nan tmem127 nan vhl nan molecular_analysis_performed_indicator no histological_type adrenocortical carcinoma- usual type laterality left lymph_node_examined_count nan metastatic_neoplasm_confirmed_diagnosis_method_name nan metastatic_neoplasm_confirmed_diagnosis_method_name-2 nan metastatic_neoplasm_confirmed_diagnosis_method_text nan distant_metastasis_anatomic_site nan metastatic_neoplasm_initial_diagnosis_anatomic_site nan metastatic_neoplasm_initial_diagnosis_anatomic_site-2 nan metastatic_neoplasm_initial_diagnosis_anatomic_site-3 nan mitoses_count 5 number_of_lymphnodes_positive_by_he nan primary_lymph_node_presentation_assessment nan residual_tumor r0 tumor_tissue_site adrenal atypical_mitotic_figures atypical mitotic figures absent cytoplasm_presence_less_than_equal_25_percent cytoplasm presence <= 25% present diffuse_architecture diffuse architecture present invasion_of_tumor_capsule invasion of tumor capsule absent mitotic_rate mitotic rate > 5/50 hpf absent necrosis necrosis present nuclear_grade_iii_iv nuclear grade iii or iv absent sinusoid_invasion sinusoid invasion absent weiss_venous_invasion venous invasion absent weiss_score 3 year_of_initial_pathologic_diagnosis 2000", "ONCOTREE_PRIMARY_DIAGNOSIS_COLOR": "Purple", "ORD_PHYSICIAN_NPI": 0000, "DISEASE_CENTER_DESCR": "Adrenal Gland oncology", "REPORT_DATE": cur_dt, "BIRTH_DATE": bir_dt, "ALT_MRN": "b3164f7b-c826-4e08-9ee6-8ff96d29b913", "REPORT_VERSION": 1, "GENDER": "Male", "PATHOLOGIST_NAME": "Kacie Smith [fake] M.D.", 'QUESTION1_YN': "Y", 'QUESTION3_YN': "Y", 'CRIS_YN': "Y", 'QUESTION2_YN': 'Y', 'BLOCK_NUMBER': '', 'QUESTION5_YN': 'Y', 'ORD_PHYSICIAN_EMAIL': '', 'QUESTION4_YN': '', 'QC_RESULT': '', 'data_push_id': '2017-01-01 05:00:00' } genomic = { '_id': ObjectId(), "TRANSCRIPT_SOURCE": "NM_153618.1", "BESTEFFECT_TRANSCRIPT_EXON": 7, "TRUE_ENTREZ_ID": "80031", "TRUE_VARIANT_CLASSIFICATION": "Missense_Mutation", "CANONICAL_VARIANT_CLASSIFICATION": "Missense_Mutation", "TRUE_STRAND": "+", "TRUE_PROTEIN_CHANGE": "p.I160V", "VARIANT_CATEGORY": "MUTATION", "BESTEFFECT_VARIANT_CLASSIFICATION": "Missense_Mutation", "TIER": 4, "ALLELE_FRACTION": 0.7977839374344301, "CANONICAL_STRAND": "+", "BEST": True, "CANONICAL_CDNA_TRANSCRIPT_ID": "ENST00000316364.5", "ALTERNATE_ALLELE": "A", "CHROMOSOME": "15", "POSITION": 48053888, "WILDTYPE": False, "CLINICAL_ID": ObjectId(clinical['_id']), "CANONICAL_TRANSCRIPT_EXON": 7, "BESTEFFECT_HUGO_SYMBOL": "SEMA6D", "REFERENCE_ALLELE": "A", "COVERAGE": 0, "BESTEFFECT_CDNA_CHANGE": "c.478A>G", "TRUE_CDNA_CHANGE": "c.478A>G", "TRUE_CDNA_TRANSCRIPT_ID": "ENST00000316364.5", "TRUE_HUGO_SYMBOL": "SEMA6D", "CANONICAL_CDNA_CHANGE": "c.478A>G", "SAMPLE_ID": "TCGA-OR-TEST1", "BESTEFFECT_ENTREZ_ID": "80031", "SOMATIC_STATUS": "Somatic", "BESTEFFECT_CDNA_TRANSCRIPT_ID": "ENST00000316364.5", "REFERENCE_GENOME": "37", "BESTEFFECT_PROTEIN_CHANGE": "p.I160V", "TRUE_TRANSCRIPT_EXON": 7, "CANONICAL_PROTEIN_CHANGE": "p.I160V", "CANONICAL_ENTREZ_ID": "80031", "CANONICAL_HUGO_SYMBOL": "SEMA6D", } status = { "_id": ObjectId(), "updated_genomic": 0, "new_genomic": 6063, "silent": False, "new_clinical": 137, "updated_clinical": 17, "last_update": datetime.datetime(year=1995, month=1, day=1) } match = { "_id": ObjectId(), "MATCH_STATUS": 0, "VARIANT_CATEGORY": "MUTATION", "CLINICAL_ID": clinical['_id'], "TRUE_HUGO_SYMBOL": "BRAF", "TEAM_ID": ObjectId(), "FILTER_STATUS": 1, "FILTER_NAME": "MEK Inhibitor", "VARIANTS": [ ObjectId() ], "ONCOTREE_PRIMARY_DIAGNOSIS_NAME": "Diffuse Glioma", "USER_ID": ObjectId(), "MMID": "D47CE1", "FILTER_ID": ObjectId(), "PATIENT_MRN": "XXXXXX", "EMAIL_SUBJECT": "", "EMAIL_ADDRESS": "*****@*****.**", "EMAIL_BODY": "", } trial_status_fields = ['drugs', 'genes', 'tumor_types', 'sponsor', 'coordinating_center', 'phase_summary', 'accrual_goal', 'investigator', 'age_summary', 'protocol_number', 'disease_status', 'nct_number', 'disease_center'] def add_remaining_required_fields(self, table): """ Given a clinical database table as a list of dictionaries, this method will iterate through each entry and add in the remaining missing required clinical fields from the default self.clinical values """ existing_keys = [] for item in table: existing_keys += item.keys() existing_keys = list(set(existing_keys)) required_keys = self.clinical.keys() for key in required_keys: if key == '_id': continue if key not in existing_keys: for item in table: if isinstance(self.clinical[key], ObjectId): item[key] = str(ObjectId()) else: item[key] = self.clinical[key] return table def _check_email(self): email = list(self.db['email'].find()) assert len(email) == 1 self.db['email'].drop() @staticmethod def _rand_alphanum(length): return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(length)) dbnames = ['clinical', 'genomic', 'trial', 'normalize', 'filter', 'match'] def _savedb(self): """ Loads important collections into memory so the database can be wiped for the unit test and then restored afterwards """ self.clinicaldb = list(self.db['clinical'].find()) self.genomicdb = list(self.db['genomic'].find()) self.trialdb = list(self.db['trial'].find()) self.normalizedb = list(self.db['normalize'].find()) self.filterdb = list(self.db['filter'].find()) self.matchdb = list(self.db['match'].find()) for dbname in self.dbnames: self.db[dbname].drop() def _restoredb(self): for db, dbname in zip( [self.clinicaldb, self.genomicdb, self.trialdb, self.normalizedb, self.filterdb, self.matchdb], self.dbnames ): if db: self.db[dbname].insert_many(db)
class TestMinimal(TestBase): def setUp(self): """ Prepare the test fixture """ self.setupDB() super(TestMinimal, self).setUp() self.app = Eve(settings='settings.py', auth=run.Auth) self.test_client = self.app.test_client() self.domain = self.app.config['DOMAIN'] def tearDown(self): self.dropDB() def assert200(self, status): self.assertEqual(status, 200) def assert301(self, status): self.assertEqual(status, 301) def assert404(self, status): self.assertEqual(status, 404) def assert304(self, status): self.assertEqual(status, 304) def assert400(self, status): self.assertEqual(status, 400) def assert401(self, status): self.assertEqual(status, 401) def assert401or405(self, status): self.assertTrue(status == 401 or 405) def assert403(self, status): self.assertEqual(status, 403) def assert405(self, status): self.assertEqual(status, 405) def assert412(self, status): self.assertEqual(status, 412) def assert500(self, status): self.assertEqual(status, 500) def setupDB(self): self.connection = MongoClient(MONGO_HOST, MONGO_PORT) self.connection.drop_database(MONGO_DBNAME) if MONGO_USERNAME: self.connection[MONGO_DBNAME].add_user(MONGO_USERNAME, MONGO_PASSWORD) self.bulk_insert() def bulk_insert(self): accounts = [ { 'u': '*****@*****.**', 'p': 'pw1', 't': 'token1', 'r': ['app'] }, { 'u': '*****@*****.**', 'p': 'pw1', 't': 'token2', 'r': ['user'] }, ] _db = self.connection[MONGO_DBNAME] _db.accounts.insert(accounts) self.connection.close() def dropDB(self): self.connection = MongoClient(MONGO_HOST, MONGO_PORT) self.connection.drop_database(MONGO_DBNAME) self.connection.close()
class TestBase(unittest.TestCase): def setUp(self): self.app = eve_example.app self.client = self.app.test_client() self.domain = self.app.config['DOMAIN'] self.setupDb() def tearDown(self): self.dropDB() del self.app def setupDb(self): self.connection = MongoClient(self.app.config['MONGO_URI']) self.db = self.connection.get_default_database() self.connection.drop_database(self.db) self.bulk_insert() def bulk_insert(self): files = glob(os.path.join(os.path.dirname(__file__), '../data/*.json')) for path in files: collection = self.db[os.path.splitext(os.path.basename(path))[0]] with open(path, 'r') as fh: for line in fh: collection.insert(json.loads(line)) def dropDB(self): self.connection.drop_database(self.app.config['MONGO_DBNAME']) self.connection.close() self.connection = None def epoch(self): return datetime(1970, 1, 1) def date_now(self): return datetime.utcnow().replace(microsecond=0) def basic_auth_headers(self, username, password): auth = base64.b64encode((username + ':' + password).encode('ascii')) return [('Authorization', 'Basic ' + auth.decode('ascii'))] def auth(self, user): return [('Authorization', 'Bearer %s' % user['token'])] def admin_auth(self): return self.basic_auth_headers('admin', self.app.config['ADMIN_PASSWORD']) def resolve_resource(self, resource, item=None): resource_params = [] if isinstance(resource, tuple): resource_params = list(resource[0:-1]) resource = resource[-1] if resource in self.domain: resource = self.domain[resource]['url'] if resource_params: resource = re.sub(r'<.*?>', lambda v: str(resource_params.pop()), resource) if item: if isinstance(item, dict): item = item.get('id') request = '/%s/%s' % (resource, str(item)) else: request = '/%s' % (resource) return request def get(self, resource, item=None, query='', headers=[]): url = self.resolve_resource(resource, item) res = self.client.get(url + query, headers=headers) return self.parse_response(res) def post(self, resource, data, item=None, headers=[], content_type=None): if not content_type: content_type = 'application/json' headers.append(('Content-Type', content_type)) url = self.resolve_resource(resource, item) res = self.client.post(url, data=json.dumps(data), headers=headers) return self.parse_response(res) def put(self, resource, data, item=None, headers=[]): headers.append(('Content-Type', 'application/json')) url = self.resolve_resource(resource, item) res = self.client.put(url, data=json.dumps(data), headers=headers) return self.parse_response(res) def patch(self, resource, data, item=None, headers=[]): headers.append(('Content-Type', 'application/json')) url = self.resolve_resource(resource, item) res = self.client.patch(url, data=json.dumps(data), headers=headers) return self.parse_response(res) def delete(self, resource, item=None, headers=None): url = self.resolve_resource(resource, item) res = self.client.delete(url, headers=headers) return self.parse_response(res) def parse_response(self, res): val = None if res.get_data(): val = res.get_data().decode("utf-8") try: val = json.loads(val) except ValueError: self.fail("'%s' is not valid JSON" % (val)) return val, res.status_code def create(self, resource, data, user=None): if not user: user = self.u.john data, status = self.post(resource, data, headers=self.auth(user)) if status != 201: print(data) self.assertNoError(data) return ObjectId(data['_id']) def fetch(self, resource, item=None, user=None): if not user: user = self.u.john data, status = self.get(resource, item=item, headers=self.auth(user)) self.assertNoError(data) return data def modify(self, resource, item, body, user=None): if not user: user = self.u.john data, status = self.patch(resource, body, item=item, headers=self.auth(user)) self.assertNoError(data) return data def remove(self, resource, item, user=None): if not user: user = self.u.john data, status = self.delete(resource, item, headers=self.auth(user)) self.assertNoError(data) def assertNoError(self, response): if response is None: return self.assertIsInstance(response, dict) if eve.STATUS in response: self.assertTrue(eve.STATUS_OK in response[eve.STATUS], 'status is not OK: %s' % response) def assertValidationError(self, response, matches): self.assertTrue(eve.STATUS in response, 'missing status') self.assertTrue(eve.STATUS_ERR in response[eve.STATUS], 'status is not ERR') self.assertTrue(ISSUES in response, 'response as no issues') issues = response[ISSUES] self.assertTrue(len(issues)) for k, v in matches.items(): self.assertTrue(k in issues, 'no "%s" error' % k) self.assertTrue(v in issues[k], '"%s" error does not contain "%s"' % (k, v)) def assertTimeEqual(self, time1, time2): self.assertAlmostEqual(time1, time2, delta=timedelta(seconds=10))
class TestMinimal(TestBase): def setUp(self): """ Prepare the test fixture """ self.setupDB() super(TestMinimal, self).setUp() self.app = Eve(settings='settings.py', auth=run.Auth) self.test_client = self.app.test_client() self.domain = self.app.config['DOMAIN'] def tearDown(self): self.dropDB() def assert200(self, status): self.assertEqual(status, 200) def assert301(self, status): self.assertEqual(status, 301) def assert404(self, status): self.assertEqual(status, 404) def assert304(self, status): self.assertEqual(status, 304) def assert400(self, status): self.assertEqual(status, 400) def assert401(self, status): self.assertEqual(status, 401) def assert401or405(self, status): self.assertTrue(status == 401 or 405) def assert403(self, status): self.assertEqual(status, 403) def assert405(self, status): self.assertEqual(status, 405) def assert412(self, status): self.assertEqual(status, 412) def assert500(self, status): self.assertEqual(status, 500) def setupDB(self): self.connection = MongoClient(MONGO_HOST, MONGO_PORT) self.connection.drop_database(MONGO_DBNAME) if MONGO_USERNAME: self.connection[MONGO_DBNAME].add_user(MONGO_USERNAME, MONGO_PASSWORD) self.bulk_insert() def bulk_insert(self): accounts = [ {'u': '*****@*****.**', 'p': 'pw1', 't': 'token1', 'r': ['app']}, {'u': '*****@*****.**', 'p': 'pw1', 't': 'token2', 'r': ['user']}, ] _db = self.connection[MONGO_DBNAME] _db.accounts.insert(accounts) self.connection.close() def dropDB(self): self.connection = MongoClient(MONGO_HOST, MONGO_PORT) self.connection.drop_database(MONGO_DBNAME) self.connection.close()
def init_user_debug(db=None, ctx=None): # only activate if necessary. if db is None: # connect to database. #connection = MongoClient(MONGO_HOST, MONGO_PORT) connection = MongoClient(MONGO_URI) # establish connection. if MONGO_USERNAME: connection[MONGO_DBNAME].add_user(MONGO_USERNAME, MONGO_PASSWORD) # setup the db pointer db = connection[MONGO_DBNAME] # add testing user. user_db = db['user'] team_db = db['team'] # add dummy first user. user = { "_id": ObjectId("4444ecb48a6ba828126f4444"), "first_name": 'Jean-luc', "last_name": 'Picard', "title": "Captain", "email": '*****@*****.**', "token": 'abc123', "roles": ["admin"], 'teams': [] } user_id = user['_id'] user_token = user['token'] user_db.insert_one(user) # insert the team first. team = { "_id": ObjectId("66a52871a8c829842fbe618b"), "name": "test team" } # insert it. team_db.insert_one(team) team_id = team['_id'] # add user. user = { "_id": ObjectId("5697ecb48a6ba828126f8128"), "first_name": 'Jean-luc2', "last_name": 'Picard2', "title": "Captain2", "email": '*****@*****.**', "token": 'abc1234', "roles": ["user"], 'teams': [] } user_db.insert_one(user) # insert the team first. team = { "name": "bernd team" } # insert it. r, status = utility_post(ctx, 'team', team, token=user_token) ctx.assert201(status) # add user. user = { "_id": ObjectId("5697ecb48a6ba828126f8128"), "first_name": 'Jean-luc3', "last_name": 'Picard3', "title": "Captain3", "email": '*****@*****.**', "token": 'abc1234', "user_name": "NULL", "roles": ["user"], 'teams': [r['_id']] } r, status = utility_post(ctx, 'user', user, token=user_token) ctx.assert201(status) # close db. if db is None: connection.close() # return results. return user_id, user_token, team_id
def bootstrap(ctx, forced=False): # get the current time. cur_dt = formatdate(time.mktime(datetime.now().timetuple())) # connect to database. #connection = MongoClient(MONGO_HOST, MONGO_PORT) connection = MongoClient(MONGO_URI) # establish connection. if MONGO_USERNAME: connection[MONGO_DBNAME].add_user(MONGO_USERNAME, MONGO_PASSWORD) # setup the db pointer db = connection[MONGO_DBNAME] # check if the clinical and genomic empty. if db[COLLECTION_CLINICAL].count() > 0 or db[COLLECTION_GENOMIC].count( ) > 0: if not forced: logging.warn( "attempting to bootstrap the database without forcing") return logging.warn("forcibly resetting the database, may take some time") # clear the database. clear_db_completely() connection.close() # initialize user. init_user_debug(ctx=ctx) # load the clinical_df clinical_df = clinical_load(DATA_CLINICAL_CSV) # store unique keys for every sample. keys = {} # insert each clinical entry new_clinical = 0 for entry in clinical_gen(clinical_df, clinical_schema=clinical['schema']): # insert into db the clinical data. resp, status_code = utility_post(ctx, 'clinical', entry, token='abc123') if status_code != 201: logging.error("unable to insert") assert status_code == 201 new_clinical += 1 # get the unique key. key = resp['_id'] # save it for later linking to genomics keys[entry['SAMPLE_ID']] = key # load the genomic_df. genomic_df = genomic_load(DATA_GENOMIC_CSV) # insert each genomic entry. i = 0 new_genomic = 0 for entry in genomics_gen(genomic_df, genomic_schema=genomic['schema']): # must have a clinical entry. if entry['SAMPLE_ID'] not in keys: continue # hook entry with clinical_id. id = keys[entry['SAMPLE_ID']] entry['CLINICAL_ID'] = id # insert it into db. entry['WILDTYPE'] = bool(entry['WILDTYPE']) resp, status_code = utility_post(ctx, 'genomic', entry, token='abc123') if status_code != 201: logging.error("unable to insert") assert status_code == 201 #if i > 1000: # break i += 1 new_genomic += 1 # post the status update. status = { 'last_update': cur_dt, 'new_clinical': new_clinical, 'updated_clinical': 0, 'new_genomic': new_genomic, 'updated_genomic': 0, 'silent': True } resp, status_code = utility_post(ctx, 'status', status, token='abc123') if status_code != 201: logging.error("unable to insert") assert status_code == 201
class CBioEngine(object): connection = None clinical_df = None genomic_df = None match_df = None living_genomic = 0 def __init__(self, mongo_uri, mongo_dbname, match_schema, muser=None, mpass=None, collection_clinical="clinical", collection_genomic="genomic"): # save these commmon variables. self.mongo_uri = mongo_uri self.mongo_dbname = mongo_dbname self.muser = muser self.mpass = mpass self.collection_clinical = collection_clinical self.collection_genomic = collection_genomic # establish database connection. self._setup_db() # set variables. self.match_schema = match_schema def match(self, c=None, g=None, c_fields=None, g_fields=None): logging.info("matching: num_clinical=%d, num_genomic=%d" % (self._c.count(), self._g.count())) self.match_clinical(c=c, c_fields=c_fields) self.match_genomic(g=g, g_fields=g_fields) if self.clinical_df.shape[0] != 0 and self.genomic_df.shape[0] != 0: match_df = pd.merge(self.clinical_df, self.genomic_df, how='inner', left_on=['_id', 'SAMPLE_ID'], right_on=['CLINICAL_ID', 'SAMPLE_ID']) match_all_df = match_df.copy() alive_query = {"VITAL_STATUS": "alive"} alive_patients = list(self._c.find(alive_query)) alive_ids = [item['SAMPLE_ID'] for item in alive_patients] match_df.fillna(value='alive', inplace=True) match_df = match_df[match_df['SAMPLE_ID'].isin(alive_ids)] self.genomic_df = self.genomic_df[self.genomic_df['SAMPLE_ID'].isin(alive_ids)] self.living_genomic = self.genomic_df.shape[0] else: match_df = None match_all_df = None alive_ids = self._c.find({}).distinct("SAMPLE_ID") if self.genomic_df.shape[0] == 0: self.living_genomic = 0 else: self.genomic_df = self.genomic_df[self.genomic_df['SAMPLE_ID'].isin(alive_ids)] self.living_genomic = self.genomic_df.shape[0] self.match_df = match_df self.match_all_df = match_all_df return self.match_df def match_clinical(self, c=None, c_fields=None): # fix empty. if c is None: c = {} # query each table. if c_fields is None: results = self._c.find(c, {'_id': 1, 'SAMPLE_ID': 1, 'REPORT_DATE': 1, 'VITAL_STATUS': 1}) else: c_fields['SAMPLE_ID'] = 1 c_fields['VITAL_STATUS'] = 1 c_fields['_id'] = 1 results = self._c.find(c, c_fields) # convert to dataframe. clinical_df = pd.DataFrame(list(results)) # save as global var. self.clinical_df = clinical_df # return it. return self.clinical_df def match_genomic(self, g=None, g_fields=None): # fix empty. if g is None: g = {} # query each table. if g_fields is None: results = self._g.find(g, {'CLINICAL_ID': 1, 'SAMPLE_ID': 1}) else: g_fields['SAMPLE_ID'] = 1 g_fields['CLINICAL_ID'] = 1 results = self._g.find(g, g_fields) # convert to dataframe. genomic_df = pd.DataFrame(list(results)) # save as global var. self.genomic_df = genomic_df # return it. return self.genomic_df def match_iter(self): # short circuit on empty if self.match_df is None: return # create validator. v = Validator(self.match_schema) # names cols. cis = zip(range(len(self.match_df.columns)), list(self.match_df.columns)) # strip out underscore fields. tmp = [] for a, b in cis: if b[0] == "_" and b != "_id_y": continue tmp.append((a,b)) cis = tmp # loop over each entry. for row in self.match_df.itertuples(): # create dictionary. tmp = {} for i, key in cis: # extract value. val = row[i+1] # convert nan if pd.isnull(val): val = None # convert special key. if key == "_id_y": key = "GENOMIC_ID" # save it to dict. tmp[key] = val # yield the match yield tmp def _setup_db(self): # connect to database. self.connection = MongoClient(self.mongo_uri) # make user. if self.muser is not None: self.connection[self.mongo_dbname].add_user(self.muser, self.mpass) # establish the collection interface. self._c = self.connection[self.mongo_dbname][self.collection_clinical] self._g = self.connection[self.mongo_dbname][self.collection_genomic] def _close_db(self): self.connection.close()
def init_user_debug(db=None, ctx=None): # only activate if necessary. if db is None: # connect to database. #connection = MongoClient(MONGO_HOST, MONGO_PORT) connection = MongoClient(MONGO_URI) # establish connection. if MONGO_USERNAME: connection[MONGO_DBNAME].add_user(MONGO_USERNAME, MONGO_PASSWORD) # setup the db pointer db = connection[MONGO_DBNAME] # add testing user. user_db = db['user'] team_db = db['team'] # add dummy first user. user = { "_id": ObjectId("4444ecb48a6ba828126f4444"), "first_name": 'Jean-luc', "last_name": 'Picard', "title": "Captain", "email": '*****@*****.**', "token": 'abc123', "roles": ["admin"], 'teams': [] } user_id = user['_id'] user_token = user['token'] user_db.insert_one(user) # insert the team first. team = {"_id": ObjectId("66a52871a8c829842fbe618b"), "name": "test team"} # insert it. team_db.insert_one(team) team_id = team['_id'] # add user. user = { "_id": ObjectId("5697ecb48a6ba828126f8128"), "first_name": 'Jean-luc2', "last_name": 'Picard2', "title": "Captain2", "email": '*****@*****.**', "token": 'abc1234', "roles": ["user"], 'teams': [] } user_db.insert_one(user) # insert the team first. team = {"name": "bernd team"} # insert it. r, status = utility_post(ctx, 'team', team, token=user_token) ctx.assert201(status) # add user. user = { "_id": ObjectId("5697ecb48a6ba828126f8128"), "first_name": 'Jean-luc3', "last_name": 'Picard3', "title": "Captain3", "email": '*****@*****.**', "token": 'abc1234', "user_name": "NULL", "roles": ["user"], 'teams': [r['_id']] } r, status = utility_post(ctx, 'user', user, token=user_token) ctx.assert201(status) # close db. if db is None: connection.close() # return results. return user_id, user_token, team_id
class TestBase(unittest.TestCase): def setUp(self): self.setupDB() self.settings_file = 'eve/tests/test_settings.py' self.app = Eve(settings=self.settings_file) self.test_client = self.app.test_client() self.domain = self.app.config['DOMAIN'] self.known_resource = 'contacts' self.known_resource_url = ('/%s/' % self.domain[self.known_resource]['url']) self.empty_resource = 'empty' self.empty_resource_url = '/%s/' % self.empty_resource self.unknown_resource = 'unknown' self.unknown_resource_url = '/%s/' % self.unknown_resource self.unknown_item_id = '4f46445fc88e201858000000' self.unknown_item_name = 'unknown' self.unknown_item_id_url = ('/%s/%s/' % (self.domain[self.known_resource]['url'], self.unknown_item_id)) self.unknown_item_name_url = ('/%s/%s/' % (self.domain[self.known_resource]['url'], self.unknown_item_name)) self.readonly_resource = 'payments' self.readonly_resource_url = ( '/%s/' % self.domain[self.readonly_resource]['url']) self.different_resource = 'users' self.different_resource_url = ('/%s/' % self.domain[ self.different_resource]['url']) def tearDown(self): self.dropDB() def assert200(self, status): self.assertEqual(status, 200) def assert301(self, status): self.assertEqual(status, 301) def assert404(self, status): self.assertEqual(status, 404) def assert304(self, status): self.assertEqual(status, 304) def setupDB(self): self.connection = MongoClient() self.connection.drop_database(MONGO_DBNAME) self.connection[MONGO_DBNAME].add_user(MONGO_USERNAME, MONGO_PASSWORD) self.bulk_insert() def bulk_insert(self): _db = self.connection[MONGO_DBNAME] _db.contacts.insert(self.random_contacts(100)) _db.contacts.insert(self.random_users(2)) _db.payments.insert(self.random_payments(10)) _db.invoices.insert(self.random_invoices(1)) self.connection.close() def dropDB(self): self.connection = MongoClient() self.connection.drop_database(MONGO_DBNAME) self.connection.close() def random_contacts(self, num, standard_date_fields=True): schema = DOMAIN['contacts']['schema'] contacts = [] for i in range(num): dt = datetime.now() contact = { 'ref': self.random_string(schema['ref']['maxlength']), 'prog': i, 'role': random.choice(schema['role']['allowed']), 'rows': self.random_rows(random.randint(0, 5)), 'alist': self.random_list(random.randint(0, 5)), 'location': { 'address': 'address ' + self.random_string(5), 'city': 'city ' + self.random_string(3), }, 'born': datetime.today() + timedelta( days=random.randint(-10, 10)), 'tid': ObjectId(), } if standard_date_fields: contact[eve.LAST_UPDATED] = dt contact[eve.DATE_CREATED] = dt contacts.append(contact) return contacts def random_users(self, num): users = self.random_contacts(num) for user in users: user['username'] = self.random_string(10) return users def random_payments(self, num): payments = [] for i in range(num): dt = datetime.now() payment = { 'a_string': self.random_string(10), 'a_number': i, eve.LAST_UPDATED: dt, eve.DATE_CREATED: dt, } payments.append(payment) return payments def random_invoices(self, num): invoices = [] for i in range(num): dt = datetime.now() invoice = { 'inv_number': self.random_string(10), eve.LAST_UPDATED: dt, eve.DATE_CREATED: dt, } invoices.append(invoice) return invoices def random_string(self, num): return (''.join(random.choice(string.ascii_uppercase) for x in range(num))) def random_list(self, num): alist = [] for i in range(num): alist.append(['string' + str(i), random.randint(1000, 9999)]) return alist def random_rows(self, num): schema = DOMAIN['contacts']['schema']['rows']['schema']['schema'] rows = [] for i in range(num): rows.append( { 'sku': self.random_string(schema['sku']['maxlength']), 'price': random.randint(100, 1000), } ) return rows
class TestMinimal(unittest.TestCase): """ Start the building of the tests for an application based on Eve by subclassing this class and provide proper settings using :func:`setUp()` """ def setUp(self, settings_file=None): """ Prepare the test fixture :param settings_file: the name of the settings file. Defaults to `eve/tests/test_settings.py`. """ if settings_file is None: # Load the settings file, using a robust path THIS_DIRECTORY = os.path.dirname(os.path.realpath(__file__)) settings_file = os.path.join(THIS_DIRECTORY, 'test_settings.py') self.known_resource_count = 101 self.setupDB() self.settings_file = settings_file self.app = eve.Eve(settings=self.settings_file) self.test_client = self.app.test_client() self.domain = self.app.config['DOMAIN'] def tearDown(self): self.dropDB() def assert200(self, status): self.assertEqual(status, 200) def assert301(self, status): self.assertEqual(status, 301) def assert404(self, status): self.assertEqual(status, 404) def assert304(self, status): self.assertEqual(status, 304) def get(self, resource, query='', item=None): if resource in self.domain: resource = self.domain[resource]['url'] if item: request = '/%s/%s' % (resource, item) else: request = '/%s%s' % (resource, query) r = self.test_client.get(request) return self.parse_response(r) def parse_response(self, r): v = json.loads(r.get_data()) if r.status_code == 200 else None return v, r.status_code def assertValidationError(self, response, key, matches): self.assertTrue(key in response) k = response[key] self.assertTrue('status' in k) self.assertTrue(eve.STATUS_ERR in k['status']) self.assertTrue('issues' in k) issues = k['issues'] self.assertTrue(len(issues)) for match in matches: self.assertTrue(match in issues[0]) def assertExpires(self, resource): # TODO if we ever get access to response.date (it is None), compare # it with Expires r = self.test_client.get(resource) expires = r.headers.get('Expires') self.assertTrue(expires is not None) def assertCacheControl(self, resource): r = self.test_client.get(resource) cache_control = r.headers.get('Cache-Control') self.assertTrue(cache_control is not None) self.assertEqual(cache_control, self.domain[self.known_resource]['cache_control']) def assertIfModifiedSince(self, resource): r = self.test_client.get(resource) last_modified = r.headers.get('Last-Modified') self.assertTrue(last_modified is not None) r = self.test_client.get(resource, headers=[('If-Modified-Since', last_modified) ]) self.assert304(r.status_code) self.assertTrue(not r.get_data()) def assertItem(self, item): self.assertEqual(type(item), dict) _id = item.get(self.app.config['ID_FIELD']) self.assertTrue(_id is not None) match = re.compile(self.app.config['ITEM_URL']).match(_id) self.assertTrue(match is not None) self.assertEqual(match.group(), _id) updated_on = item.get(self.app.config['LAST_UPDATED']) self.assertTrue(updated_on is not None) try: datetime.strptime(updated_on, self.app.config['DATE_FORMAT']) except Exception as e: self.fail('Cannot convert field "%s" to datetime: %s' % (self.app.config['LAST_UPDATED'], e)) created_on = item.get(self.app.config['DATE_CREATED']) self.assertTrue(updated_on is not None) try: datetime.strptime(created_on, self.app.config['DATE_FORMAT']) except Exception as e: self.fail('Cannot convert field "%s" to datetime: %s' % (self.app.config['DATE_CREATED'], e)) link = item.get('_links') self.assertItemLink(link, _id) def assertHomeLink(self, links): self.assertTrue('parent' in links) link = links['parent'] self.assertTrue('title' in link) self.assertTrue('href' in link) self.assertEqual('home', link['title']) self.assertEqual("%s" % self.app.config['SERVER_NAME'], link['href']) def assertResourceLink(self, links, resource): self.assertTrue('self' in links) link = links['self'] self.assertTrue('title' in link) self.assertTrue('href' in link) url = self.domain[resource]['url'] self.assertEqual(url, link['title']) self.assertEqual("%s/%s" % (self.app.config['SERVER_NAME'], url), link['href']) def assertCollectionLink(self, links, resource): self.assertTrue('collection' in links) link = links['collection'] self.assertTrue('title' in link) self.assertTrue('href' in link) url = self.domain[resource]['url'] self.assertEqual(url, link['title']) self.assertEqual("%s/%s" % (self.app.config['SERVER_NAME'], url), link['href']) def assertNextLink(self, links, page): self.assertTrue('next' in links) link = links['next'] self.assertTrue('title' in link) self.assertTrue('href' in link) self.assertEqual('next page', link['title']) self.assertTrue("page=%d" % page in link['href']) def assertPrevLink(self, links, page): self.assertTrue('prev' in links) link = links['prev'] self.assertTrue('title' in link) self.assertTrue('href' in link) self.assertEqual('previous page', link['title']) if page > 1: self.assertTrue("page=%d" % page in link['href']) def assertItemLink(self, links, item_id): self.assertTrue('self' in links) link = links['self'] #TODO we are too deep here to get a hold of the due title. Should fix. self.assertTrue('title' in link) self.assertTrue('href' in link) self.assertTrue('/%s' % item_id in link['href']) def assertLastLink(self, links, page): if page: self.assertTrue('last' in links) link = links['last'] self.assertTrue('title' in link) self.assertTrue('href' in link) self.assertEqual('last page', link['title']) self.assertTrue("page=%d" % page in link['href']) else: self.assertTrue('last' not in links) def assert400(self, status): self.assertEqual(status, 400) def assert401(self, status): self.assertEqual(status, 401) def assert401or405(self, status): self.assertTrue(status == 401 or 405) def assert403(self, status): self.assertEqual(status, 403) def assert405(self, status): self.assertEqual(status, 405) def assert412(self, status): self.assertEqual(status, 412) def assert500(self, status): self.assertEqual(status, 500) def setupDB(self): self.connection = MongoClient(MONGO_HOST, MONGO_PORT) self.connection.drop_database(MONGO_DBNAME) if MONGO_USERNAME: self.connection[MONGO_DBNAME].add_user(MONGO_USERNAME, MONGO_PASSWORD) self.bulk_insert() def bulk_insert(self): pass def dropDB(self): self.connection = MongoClient(MONGO_HOST, MONGO_PORT) self.connection.drop_database(MONGO_DBNAME) self.connection.close()
class TestBase(TestMinimal): DEVICES = 'devices' DEVICE_EVENT = 'events/devices' EVENTS = 'events' PLACES = 'places' SNAPSHOT = 'snapshot' ACCOUNTS = 'accounts' def setUp(self, settings_file=None, url_converters=None): from ereuse_devicehub import default_settings as settings self.set_settings(settings) self.app = DeviceHub() self.prepare() @staticmethod def set_settings(settings): settings.MONGO_DBNAME = 'devicehubtest' settings.DATABASES = 'dht1', 'dht2' # Some tests use 2 databases settings.DHT1_DBNAME = 'dht1_' settings.DHT2_DBNAME = 'dht2_' settings.GRD_DEBUG = True # We do not want to actually fulfill GRD settings.APP_NAME = 'DeviceHub' settings.DEBUG = True settings.TESTING = True settings.LOG = True settings.GRD = False settings.AGENT_ACCOUNTS = { 'self': ('*****@*****.**', '12345') } def prepare(self): self.MONGO_DBNAME = self.app.config['MONGO_DBNAME'] self.MONGO_HOST = self.app.config['MONGO_HOST'] self.MONGO_PORT = self.app.config['MONGO_PORT'] self.DATABASES = self.app.config['DATABASES'] self.connection = None self.setupDB() self.test_client = self.app.test_client() self.domain = self.app.config['DOMAIN'] self.token = self._login() self.auth_header = ('authorization', 'Basic ' + self.token) def setupDB(self): self.connection = MongoClient(self.MONGO_HOST, self.MONGO_PORT) self.db = self.connection[self.MONGO_DBNAME] self.drop_databases() self.create_dummy_user() self.create_self_machine_account() # We call the method again as we have erased the DB self.app.grd_submitter_caller = SubmitterCaller(self.app, GRDSubmitter) # self.app.grd_submitter_caller.token = self.app.grd_submitter_caller.prepare_user(self.app) # self.app.grd_submitter_caller.process = None def create_dummy_user(self): self.db.accounts.insert( { 'email': "[email protected]", 'password': AccountDomain.encrypt_password('1234'), 'role': 'admin', 'token': 'NOFATDNNUB', 'databases': self.app.config['DATABASES'], 'defaultDatabase': self.app.config['DATABASES'][0], '@type': 'Account' } ) self.account = self.db.accounts.find_one({'email': '[email protected]'}) def create_self_machine_account(self): email, password = self.app.config['AGENT_ACCOUNTS']['self'] self.db.accounts.insert( { 'role': 'superuser', 'token': 'QYADFBPNZZDFJEWAFGGF', 'databases': self.app.config['DATABASES'], '@type': 'Account', 'email': email, 'password': AccountDomain.encrypt_password(password) } ) def tearDown(self): self.dropDB() del self.app def drop_databases(self): self.connection.drop_database(self.MONGO_DBNAME) for database in self.DATABASES: self.connection.drop_database(self.app.config[database.upper().replace('-', '') + '_DBNAME']) def dropDB(self): self.drop_databases() self.connection.close() def full(self, resource_name: str, resource: dict or str or ObjectId) -> dict: return resource if type(resource) is dict else self.get(resource_name, '', str(resource))[0] def select_database(self, url): if 'accounts' in url: return '' else: return self.app.config['DATABASES'][0] def get(self, resource, query='', item=None, authorize=True, database=None): if resource in self.domain: resource = self.domain[resource]['url'] if item: request = '/%s/%s%s' % (resource, item, query) else: request = '/%s%s' % (resource, query) database = database or self.select_database(resource) return self._get(database + request, self.token if authorize else None) def _get(self, url, token=None, **kwargs): environ_base = {'HTTP_AUTHORIZATION': 'Basic ' + token} if token else {} environ_base.update(kwargs.pop('environ_base', {})) r = self.test_client.get(url, environ_base=environ_base, **kwargs) return self.parse_response(r) def post(self, url, data, headers=None, content_type='application/json'): full_url = self.select_database(url) + '/' + url headers = headers or [] return self._post(full_url, data, self.token, headers, content_type) def _post(self, url, data, token, headers=None, content_type='application/json'): headers = headers or [] headers.append(('authorization', 'Basic ' + token)) if type(data) is str: headers.append(('Content-Type', content_type)) r = self.test_client.post(url, data=data, headers=headers) return self.parse_response(r) return super(TestBase, self).post(url, data, headers, content_type) def patch(self, url, data, headers=None): headers = headers or [] return self._patch(self.select_database(url) + '/' + url, data, self.token, headers) def _patch(self, url, data, token, headers=None): headers = headers or [] headers.append(('authorization', 'Basic ' + token)) return super(TestBase, self).patch(url, data, headers) def put(self, url, data, headers=None): headers = headers or [] return super(TestBase, self).put(self.select_database(url) + '/' + url, data, headers + [self.auth_header]) def delete(self, url, headers=None): headers = headers or [] return super(TestBase, self).delete(self.select_database(url) + '/' + url, headers + [self.auth_header]) def _login(self) -> str: return super(TestBase, self).post('/login', {"email": "[email protected]", "password": "******"})[0]['token'] def assert308(self, status): self.assertEqual(status, 308)
class TestMinimal(pea.TestCase): """ Start the building of the tests for the PulsePod API by subclassing this class and provide proper settings using :func:`setUp()` """ def setUp(self, config=None): """ Prepare the test fixture :param config: the name of the config class. Defaults to `testing`. """ super(TestMinimal, self).setUp() self.this_directory = os.path.dirname(os.path.realpath(__file__)) if config is None: config = 'testing' elif config in ('production', 'development', 'heroku'): assert 0, "Unable to run tests with %s config" % config self.connection = None self.app = create_app('testing') self.app_context = self.app.app_context() self.app_context.push() self.setupDB() self.test_client = self.app.test_client() self.domain = self.app.config['DOMAIN'] self.config = self.app.config self.DataMessage = { 'message_content': u'020f5be1018446df5314512d41b6028446df5' + '339ecb6427438df53cee69f4255038446df538dc656427438df530dbe' + '9442642adf53011aa641' } self.DeployMessage = { 'message_content': u'030f5b310026032229c58b436b4003e1b655' } def tearDown(self): super(TestMinimal, self).tearDown() self.dropDB() self.app_context.pop() del self.app def assert200(self, status): self.assertEqual(status, 200) def assert201(self, status): self.assertEqual(status, 201) def assert301(self, status): self.assertEqual(status, 301) def assert404(self, status): self.assertEqual(status, 404) def assert304(self, status): self.assertEqual(status, 304) def get(self, resource, query='', item=None, headers=None): if resource in self.domain: resource = self.domain[resource]['url'] if item: request = '/%s/%s%s' % (resource, item, query) else: request = '/%s%s' % (resource, query) r = self.test_client.get(request) return self.parse_response(r) def post(self, url, data, headers=None, content_type='application/json'): if headers is None: headers = [] headers.append(('Content-Type', content_type)) r = self.test_client.post(url, data=json.dumps(data), headers=headers) return self.parse_response(r) def put(self, url, data, headers=None): if headers is None: headers = [] headers.append(('Content-Type', 'application/json')) r = self.test_client.put(url, data=json.dumps(data), headers=headers) return self.parse_response(r) def patch(self, url, data, headers=None): if headers is None: headers = [] headers.append(('Content-Type', 'application/json')) r = self.test_client.patch(url, data=json.dumps(data), headers=headers) return self.parse_response(r) def delete(self, url, headers=None): r = self.test_client.delete(url, headers=headers) return self.parse_response(r) def bulk_insert(self): pass def parse_response(self, r): try: v = json.loads(r.get_data()) except json.JSONDecodeError: v = None return v, r.status_code def setupDB(self): settings = self.app.config['MONGODB_SETTINGS'] self.connection = MongoClient(settings['HOST'], settings['PORT']) self.connection.drop_database(settings['DB']) if 'USERNAME' in settings: self.connection[settings['DB']].add_user( settings['USERNAME'], settings['PASSWORD']) self.Pod = models.Pod self.User = models.User self.Message = models.Message self.Notebook = models.Notebook self.Sensor = models.Sensor self.Data = models.Data self.bulk_insert() def dropDB(self): settings = self.app.config['MONGODB_SETTINGS'] self.connection = MongoClient( settings['HOST'], settings['PORT']) self.connection.drop_database(settings['DB']) self.connection.close()
class TestMinimal(unittest.TestCase): """ Start the building of the tests for an application based on Eve by subclassing this class and provide proper settings using :func:`setUp()` """ app = ValueStack(close_pymongo_connection) def setUp(self, settings_file=None, url_converters=None): """ Prepare the test fixture :param settings_file: the name of the settings file. Defaults to `eve/tests/test_settings.py`. """ self.this_directory = os.path.dirname(os.path.realpath(__file__)) if settings_file is None: # Load the settings file, using a robust path settings_file = os.path.join(self.this_directory, 'test_settings.py') self.connection = None self.known_resource_count = 101 self.setupDB() self.settings_file = settings_file self.app = eve.Eve(settings=self.settings_file, url_converters=url_converters) self.test_client = self.app.test_client() self.domain = self.app.config['DOMAIN'] def tearDown(self): del self.app self.dropDB() def assert200(self, status): self.assertEqual(status, 200) def assert201(self, status): self.assertEqual(status, 201) def assert204(self, status): self.assertEqual(status, 204) def assert301(self, status): self.assertEqual(status, 301) def assert304(self, status): self.assertEqual(status, 304) def assert404(self, status): self.assertEqual(status, 404) def assert422(self, status): self.assertEqual(status, 422) def get(self, resource, query='', item=None): if resource in self.domain: resource = self.domain[resource]['url'] if item: request = '/%s/%s%s' % (resource, item, query) else: request = '/%s%s' % (resource, query) r = self.test_client.get(request) return self.parse_response(r) def post(self, url, data, headers=None, content_type='application/json'): if headers is None: headers = [] headers.append(('Content-Type', content_type)) r = self.test_client.post(url, data=json.dumps(data), headers=headers) return self.parse_response(r) def put(self, url, data, headers=None): if headers is None: headers = [] headers.append(('Content-Type', 'application/json')) r = self.test_client.put(url, data=json.dumps(data), headers=headers) return self.parse_response(r) def patch(self, url, data, headers=None): if headers is None: headers = [] headers.append(('Content-Type', 'application/json')) r = self.test_client.patch(url, data=json.dumps(data), headers=headers) return self.parse_response(r) def delete(self, url, headers=None): r = self.test_client.delete(url, headers=headers) return self.parse_response(r) def parse_response(self, r): try: v = json.loads(r.get_data()) except json.JSONDecodeError: v = None return v, r.status_code def assertValidationErrorStatus(self, status): self.assertEqual(status, self.app.config.get('VALIDATION_ERROR_STATUS')) def assertValidationError(self, response, matches): self.assertTrue(eve.STATUS in response) self.assertTrue(eve.STATUS_ERR in response[eve.STATUS]) self.assertTrue(ISSUES in response) issues = response[ISSUES] self.assertTrue(len(issues)) for k, v in matches.items(): self.assertTrue(k in issues) self.assertTrue(v in issues[k]) def assertExpires(self, resource): # TODO if we ever get access to response.date (it is None), compare # it with Expires r = self.test_client.get(resource) expires = r.headers.get('Expires') self.assertTrue(expires is not None) def assertCacheControl(self, resource): r = self.test_client.get(resource) cache_control = r.headers.get('Cache-Control') self.assertTrue(cache_control is not None) self.assertEqual(cache_control, self.domain[self.known_resource]['cache_control']) def assertIfModifiedSince(self, resource): r = self.test_client.get(resource) last_modified = r.headers.get('Last-Modified') self.assertTrue(last_modified is not None) r = self.test_client.get(resource, headers=[('If-Modified-Since', last_modified) ]) self.assert304(r.status_code) self.assertTrue(not r.get_data()) def assertItem(self, item, resource): self.assertEqual(type(item), dict) updated_on = item.get(self.app.config['LAST_UPDATED']) self.assertTrue(updated_on is not None) try: datetime.strptime(updated_on, self.app.config['DATE_FORMAT']) except Exception as e: self.fail('Cannot convert field "%s" to datetime: %s' % (self.app.config['LAST_UPDATED'], e)) created_on = item.get(self.app.config['DATE_CREATED']) self.assertTrue(updated_on is not None) try: datetime.strptime(created_on, self.app.config['DATE_FORMAT']) except Exception as e: self.fail('Cannot convert field "%s" to datetime: %s' % (self.app.config['DATE_CREATED'], e)) link = item.get('_links') _id = item.get(self.domain[resource]['id_field']) self.assertItemLink(link, _id) def assertPagination(self, response, page, total, max_results): p_key, mr_key = self.app.config['QUERY_PAGE'], \ self.app.config['QUERY_MAX_RESULTS'] self.assertTrue(self.app.config['META'] in response) meta = response.get(self.app.config['META']) self.assertTrue(p_key in meta) self.assertTrue(mr_key in meta) self.assertTrue('total' in meta) self.assertEqual(meta[p_key], page) self.assertEqual(meta[mr_key], max_results) self.assertEqual(meta['total'], total) def assertHomeLink(self, links): self.assertTrue('parent' in links) link = links['parent'] self.assertTrue('title' in link) self.assertTrue('href' in link) self.assertEqual('home', link['title']) self.assertEqual("/", link['href']) def assertResourceLink(self, links, resource): self.assertTrue('self' in links) link = links['self'] self.assertTrue('title' in link) self.assertTrue('href' in link) url = self.domain[resource]['url'] self.assertEqual(url, link['title']) self.assertEqual("%s" % url, link['href']) def assertCollectionLink(self, links, resource): self.assertTrue('collection' in links) link = links['collection'] self.assertTrue('title' in link) self.assertTrue('href' in link) url = self.domain[resource]['url'] self.assertEqual(url, link['title']) self.assertEqual("%s" % url, link['href']) def assertNextLink(self, links, page): self.assertTrue('next' in links) link = links['next'] self.assertTrue('title' in link) self.assertTrue('href' in link) self.assertEqual('next page', link['title']) self.assertTrue("%s=%d" % (self.app.config['QUERY_PAGE'], page) in link['href']) def assertPrevLink(self, links, page): self.assertTrue('prev' in links) link = links['prev'] self.assertTrue('title' in link) self.assertTrue('href' in link) self.assertEqual('previous page', link['title']) if page > 1: self.assertTrue( "%s=%d" % (self.app.config['QUERY_PAGE'], page) in link['href']) def assertItemLink(self, links, item_id): self.assertTrue('self' in links) link = links['self'] # TODO we are too deep here to get a hold of the due title. Should fix. self.assertTrue('title' in link) self.assertTrue('href' in link) self.assertTrue('/%s' % item_id in link['href']) def assertLastLink(self, links, page): if page: self.assertTrue('last' in links) link = links['last'] self.assertTrue('title' in link) self.assertTrue('href' in link) self.assertEqual('last page', link['title']) self.assertTrue( "%s=%d" % (self.app.config['QUERY_PAGE'], page) in link['href']) else: self.assertTrue('last' not in links) def assert400(self, status): self.assertEqual(status, 400) def assert401(self, status): self.assertEqual(status, 401) def assert401or405(self, status): self.assertTrue(status == 401 or 405) def assert403(self, status): self.assertEqual(status, 403) def assert405(self, status): self.assertEqual(status, 405) def assert412(self, status): self.assertEqual(status, 412) def assert500(self, status): self.assertEqual(status, 500) def setupDB(self): self.connection = MongoClient(MONGO_HOST, MONGO_PORT) self.connection.drop_database(MONGO_DBNAME) if MONGO_USERNAME: self.connection[MONGO_DBNAME].add_user(MONGO_USERNAME, MONGO_PASSWORD) self.bulk_insert() def bulk_insert(self): pass def dropDB(self): self.connection = MongoClient(MONGO_HOST, MONGO_PORT) self.connection.drop_database(MONGO_DBNAME) self.connection.close()
class CBioEngine(object): connection = None clinical_df = None genomic_df = None match_df = None living_genomic = 0 def __init__(self, mongo_uri, mongo_dbname, match_schema, muser=None, mpass=None, collection_clinical="clinical", collection_genomic="genomic"): # save these commmon variables. self.mongo_uri = mongo_uri self.mongo_dbname = mongo_dbname self.muser = muser self.mpass = mpass self.collection_clinical = collection_clinical self.collection_genomic = collection_genomic # establish database connection. self._setup_db() # set variables. self.match_schema = match_schema def match(self, c=None, g=None, c_fields=None, g_fields=None): logging.info("matching: num_clinical=%d, num_genomic=%d" % (self._c.count(), self._g.count())) self.match_clinical(c=c, c_fields=c_fields) self.match_genomic(g=g, g_fields=g_fields) if self.clinical_df.shape[0] != 0 and self.genomic_df.shape[0] != 0: match_df = pd.merge(self.clinical_df, self.genomic_df, how='inner', left_on=['_id', 'SAMPLE_ID'], right_on=['CLINICAL_ID', 'SAMPLE_ID']) match_all_df = match_df.copy() alive_query = {"VITAL_STATUS": "alive"} alive_patients = list(self._c.find(alive_query)) alive_ids = [item['SAMPLE_ID'] for item in alive_patients] match_df.fillna(value='alive', inplace=True) match_df = match_df[match_df['SAMPLE_ID'].isin(alive_ids)] self.genomic_df = self.genomic_df[ self.genomic_df['SAMPLE_ID'].isin(alive_ids)] self.living_genomic = self.genomic_df.shape[0] else: match_df = None match_all_df = None alive_ids = self._c.find({}).distinct("SAMPLE_ID") if self.genomic_df.shape[0] == 0: self.living_genomic = 0 else: self.genomic_df = self.genomic_df[ self.genomic_df['SAMPLE_ID'].isin(alive_ids)] self.living_genomic = self.genomic_df.shape[0] self.match_df = match_df self.match_all_df = match_all_df return self.match_df def match_clinical(self, c=None, c_fields=None): # fix empty. if c is None: c = {} # query each table. if c_fields is None: results = self._c.find(c, { '_id': 1, 'SAMPLE_ID': 1, 'REPORT_DATE': 1, 'VITAL_STATUS': 1 }) else: c_fields['SAMPLE_ID'] = 1 c_fields['VITAL_STATUS'] = 1 c_fields['_id'] = 1 results = self._c.find(c, c_fields) # convert to dataframe. clinical_df = pd.DataFrame(list(results)) # save as global var. self.clinical_df = clinical_df # return it. return self.clinical_df def match_genomic(self, g=None, g_fields=None): # fix empty. if g is None: g = {} # query each table. if g_fields is None: results = self._g.find(g, {'CLINICAL_ID': 1, 'SAMPLE_ID': 1}) else: g_fields['SAMPLE_ID'] = 1 g_fields['CLINICAL_ID'] = 1 results = self._g.find(g, g_fields) # convert to dataframe. genomic_df = pd.DataFrame(list(results)) # save as global var. self.genomic_df = genomic_df # return it. return self.genomic_df def match_iter(self): # short circuit on empty if self.match_df is None: return # create validator. v = Validator(self.match_schema) # names cols. cis = zip(range(len(self.match_df.columns)), list(self.match_df.columns)) # strip out underscore fields. tmp = [] for a, b in cis: if b[0] == "_" and b != "_id_y": continue tmp.append((a, b)) cis = tmp # loop over each entry. for row in self.match_df.itertuples(): # create dictionary. tmp = {} for i, key in cis: # extract value. val = row[i + 1] # convert nan if pd.isnull(val): val = None # convert special key. if key == "_id_y": key = "GENOMIC_ID" # save it to dict. tmp[key] = val # yield the match yield tmp def _setup_db(self): # connect to database. self.connection = MongoClient(self.mongo_uri) # make user. if self.muser is not None: self.connection[self.mongo_dbname].add_user(self.muser, self.mpass) # establish the collection interface. self._c = self.connection[self.mongo_dbname][self.collection_clinical] self._g = self.connection[self.mongo_dbname][self.collection_genomic] def _close_db(self): self.connection.close()