def setupDB(self):
self.connection = MongoClient(MONGO_HOST, MONGO_PORT)
self.connection.drop_database(MONGO_DBNAME)
if MONGO_USERNAME:
self.connection[MONGO_DBNAME].add_user(MONGO_USERNAME,
MONGO_PASSWORD)
self.bulk_insert()
def setupDB(self):
self.connection = MongoClient(MONGO_HOST, MONGO_PORT)
self.connection.drop_database(MONGO_DBNAME)
if MONGO_USERNAME:
self.connection[MONGO_DBNAME].add_user(MONGO_USERNAME,
MONGO_PASSWORD)
db = self.connection[MONGO_DBNAME]
self.accounts_insert(db)
self.company_insert(db)
db.connection.close()
def _setup_db(self):
# connect to database.
self.connection = MongoClient(self.mongo_uri)
# make user.
if self.muser is not None:
self.connection[self.mongo_dbname].add_user(self.muser, self.mpass)
# establish the collection interface.
self._c = self.connection[self.mongo_dbname][self.collection_clinical]
self._g = self.connection[self.mongo_dbname][self.collection_genomic]
def dropDB(self):
settings = self.app.config['MONGODB_SETTINGS']
self.connection = MongoClient(
settings['HOST'],
settings['PORT'])
self.connection.drop_database(settings['DB'])
self.connection.close()
def get_collection(name):
"""
return a connection to a specific collection
:param name: name of the collection
:return: mongoDB collection
"""
try:
collection = app.data.driver.db[name]
except RuntimeError as e:
# connect to database.
#connection = MongoClient(MONGO_HOST, MONGO_PORT)
connection = MongoClient(MONGO_URI)
if MONGO_USERNAME:
connection[MONGO_DBNAME].add_user(MONGO_USERNAME, MONGO_PASSWORD)
db = connection[MONGO_DBNAME]
# get the collection
collection = db[name]
# return it
return collection
def setupDB(self):
self.connection = MongoClient(MONGO_HOST, MONGO_PORT)
self.connection.drop_database(MONGO_DBNAME)
if MONGO_USERNAME:
self.connection[MONGO_DBNAME].add_user(MONGO_USERNAME,
MONGO_PASSWORD)
self.bulk_insert()
def dropDB(self):
#return
# connect to database.
self.connection = MongoClient(MONGO_URI)
# drop all tables except the genomic/clinical
db = self.connection[MONGO_DBNAME]
db.drop_collection("user")
db.drop_collection("team")
db.drop_collection("filter")
db.drop_collection("match")
db.drop_collection("hipaa")
db.drop_collection("status")
db.drop_collection("trial")
db.drop_collection("normalize")
db.drop_collection("email")
# clear extra clinical and genomic.
db['clinical'].delete_many({"TOTAL_READS" : 123, "ORD_PHYSICIAN_NPI": 0000})
db['genomic'].delete_many({'COVERAGE': 9123})
db['genomic'].delete_many({"STRUCTURAL_VARIANT_COMMENT": re.compile("tmp6654.*", re.IGNORECASE)})
for id in self.inserted_clinical:
db['clinical'].delete_one({'_id': ObjectId(id)})
for id in self.inserted_genomic:
db['genomic'].delete_one({'_id': ObjectId(id)})
# last check.
db['clinical'].delete_many({"SAMPLE_ID": "TCGA-OR-TEST1"})
# close connection.
self.connection.close()
def setupDB(self):
self.connection = MongoClient(self.MONGO_HOST, self.MONGO_PORT)
self.db = self.connection[self.MONGO_DBNAME]
self.drop_databases()
self.create_dummy_user()
self.create_self_machine_account()
# We call the method again as we have erased the DB
self.app.grd_submitter_caller = SubmitterCaller(self.app, GRDSubmitter)
def register():
client = MongoClient()
db = client.app_users
email = request.json.get('email', None)
password = request.json.get('password', None)
cred = {"email": email, "password": password}
post = db.users
posted_id = post.insert_one(cred).inserted_id
return jsonify({'success': True, 'email': email, 'password': password})
class TestMultiMongo(TestBase):
def setUp(self):
super(TestMultiMongo, self).setUp()
self.setupDB2()
schema = {
'author': {
'type': 'string'
},
'title': {
'type': 'string'
},
}
settings = {'schema': schema, 'mongo_prefix': 'MONGO1'}
self.app.register_resource('works', settings)
def tearDown(self):
super(TestMultiMongo, self).tearDown()
self.dropDB2()
def setupDB2(self):
self.connection = MongoClient()
self.connection.drop_database(MONGO1_DBNAME)
self.connection[MONGO1_DBNAME].add_user(MONGO1_USERNAME,
MONGO1_PASSWORD)
self.bulk_insert2()
def dropDB2(self):
self.connection = MongoClient()
self.connection.drop_database(MONGO1_DBNAME)
self.connection.close()
def bulk_insert2(self):
_db = self.connection[MONGO1_DBNAME]
works = self.random_works(self.known_resource_count)
_db.works.insert(works)
self.work = _db.works.find_one()
self.connection.close()
def random_works(self, num):
works = []
for i in range(num):
dt = datetime.now()
work = {
'author': self.random_string(20),
'title': self.random_string(30),
eve.LAST_UPDATED: dt,
eve.DATE_CREATED: dt,
}
works.append(work)
return works
class TestMultiMongo(TestBase):
def setUp(self):
super(TestMultiMongo, self).setUp()
self.setupDB2()
schema = {
'author': {'type': 'string'},
'title': {'type': 'string'},
}
settings = {
'schema': schema,
'mongo_prefix': 'MONGO1'
}
self.app.register_resource('works', settings)
def tearDown(self):
super(TestMultiMongo, self).tearDown()
self.dropDB2()
def setupDB2(self):
self.connection = MongoClient()
self.connection.drop_database(MONGO1_DBNAME)
self.connection[MONGO1_DBNAME].add_user(MONGO1_USERNAME,
MONGO1_PASSWORD)
self.bulk_insert2()
def dropDB2(self):
self.connection = MongoClient()
self.connection.drop_database(MONGO1_DBNAME)
self.connection.close()
def bulk_insert2(self):
_db = self.connection[MONGO1_DBNAME]
works = self.random_works(self.known_resource_count)
_db.works.insert(works)
self.work = _db.works.find_one()
self.connection.close()
def random_works(self, num):
works = []
for i in range(num):
dt = datetime.now()
work = {
'author': self.random_string(20),
'title': self.random_string(30),
eve.LAST_UPDATED: dt,
eve.DATE_CREATED: dt,
}
works.append(work)
return works
def clear_db_partially():
logging.warn("dropping all tables in database EXCEPT clinical and genomic")
# connect to database.
#connection = MongoClient(MONGO_HOST, MONGO_PORT)
connection = MongoClient(MONGO_URI)
# establish connection.
if MONGO_USERNAME:
connection[MONGO_DBNAME].add_user(MONGO_USERNAME, MONGO_PASSWORD)
# setup the db pointer
db = connection[MONGO_DBNAME]
# clear the database.
db.drop_collection("user")
db.drop_collection("team")
db.drop_collection("filter")
db.drop_collection("match")
connection.close()
def clear_db_partially():
logging.warn("dropping all tables in database EXCEPT clinical and genomic")
# connect to database.
#connection = MongoClient(MONGO_HOST, MONGO_PORT)
connection = MongoClient(MONGO_URI)
# establish connection.
if MONGO_USERNAME:
connection[MONGO_DBNAME].add_user(MONGO_USERNAME, MONGO_PASSWORD)
# setup the db pointer
db = connection[MONGO_DBNAME]
# clear the database.
db.drop_collection("user")
db.drop_collection("team")
db.drop_collection("filter")
db.drop_collection("match")
connection.close()
def _setup_db(self):
# connect to database.
self.connection = MongoClient(self.mongo_uri)
# make user.
if self.muser is not None:
self.connection[self.mongo_dbname].add_user(self.muser, self.mpass)
# establish the collection interface.
self._c = self.connection[self.mongo_dbname][self.collection_clinical]
self._g = self.connection[self.mongo_dbname][self.collection_genomic]
def setupDB(self):
# drop the database.
self.dropDB()
# connect to database.
self.connection = MongoClient(MONGO_URI)
# establish connection.
if MONGO_USERNAME:
self.connection[MONGO_DBNAME].add_user(MONGO_USERNAME,
MONGO_PASSWORD)
def add_simulated_sv():
# connect to database.
#connection = MongoClient(MONGO_HOST, MONGO_PORT)
connection = MongoClient(MONGO_URI)
# setup the db pointer
db = connection[MONGO_DBNAME]
# fetch all patients.
patients = list(db.clinical.find())
pidx = range(len(patients))
# loop over each synoymm
svs = list()
from matchminer.constants import synonyms
for key in synonyms:
for val in synonyms[key]:
# get clinical_id.
idx = random.choice(pidx)
clinical_id = str(patients[idx]['_id'])
sample_id = str(patients[idx]['SAMPLE_ID'])
# make a SV.
sv = {
'CLINICAL_ID': clinical_id,
'VARIANT_CATEGORY': 'SV',
'STRUCTURAL_VARIANT_COMMENT': "tmp6654 " + val,
'WILDTYPE': False,
'SAMPLE_ID': sample_id
}
svs.append(sv)
# return it.
connection.close()
return svs
def add_simulated_sv():
# connect to database.
#connection = MongoClient(MONGO_HOST, MONGO_PORT)
connection = MongoClient(MONGO_URI)
# setup the db pointer
db = connection[MONGO_DBNAME]
# fetch all patients.
patients = list(db.clinical.find())
pidx = range(len(patients))
# loop over each synoymm
svs = list()
from matchminer.constants import synonyms
for key in synonyms:
for val in synonyms[key]:
# get clinical_id.
idx = random.choice(pidx)
clinical_id = str(patients[idx]['_id'])
sample_id = str(patients[idx]['SAMPLE_ID'])
# make a SV.
sv = {
'CLINICAL_ID': clinical_id,
'VARIANT_CATEGORY': 'SV',
'STRUCTURAL_VARIANT_COMMENT': "tmp6654 " + val,
'WILDTYPE': False,
'SAMPLE_ID': sample_id
}
svs.append(sv)
# return it.
connection.close()
return svs
def setupDB(self):
settings = self.app.config['MONGODB_SETTINGS']
self.connection = MongoClient(settings['HOST'],
settings['PORT'])
self.connection.drop_database(settings['DB'])
if 'USERNAME' in settings:
self.connection[settings['DB']].add_user(
settings['USERNAME'], settings['PASSWORD'])
self.Pod = models.Pod
self.User = models.User
self.Message = models.Message
self.Notebook = models.Notebook
self.Sensor = models.Sensor
self.Data = models.Data
self.bulk_insert()
def uploader():
if request.method == 'POST':
f = request.files['file']
filename = f.filename
f.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
print filename
vul_data = filename.split("-")
vul_name = vul_data[0]
vul_class = vul_data[1]
print "Vulnerability Name from file: "
print vul_name
print "Vulnerability Class from File: "
print vul_class
client = MongoClient("localhost", 27017, maxPoolSize=50)
db = client.report_helper
collection = db['vulnerability']
cursor = collection.find({})
for document in cursor:
print(document['vul_finding'])
fin_temp = document['vul_finding']
print "Checking For"
print fin_temp
if vul_name in fin_temp:
print "Vulnerability Name from DB: "
print document['vul_finding']
print "Vulnerability Criticality "
print document['vul_crit']
print "Vulnerability Description "
print document['vul_desc']
print "Vulnerability Risk/ Implication "
print document['vul__risk']
print "Vulnerability Recommendation "
print document['vul_recom']
diction = {
'vul_finding': document['vul_finding'],
'vul_crit': document['vul_crit'],
'vul_desc': document['vul_desc'],
'vul__risk': document['vul__risk'],
'vul_recom': document['vul_recom']
}
df = pd.DataFrame(data=diction, index=[0])
df = (df.T)
print(df)
df.to_excel('test.xlsx')
print "NOT FOUND"
return 'file uploaded successfully'
def get_db():
"""
return a database connection.
:return: mongoDB pointer
"""
try:
db = app.data.driver.db
except RuntimeError as e:
# connect to database.
#connection = MongoClient(MONGO_HOST, MONGO_PORT)
connection = MongoClient(MONGO_URI)
if MONGO_USERNAME:
connection[MONGO_DBNAME].add_user(MONGO_USERNAME, MONGO_PASSWORD)
db = connection[MONGO_DBNAME]
# return it
return db
def setupDB2(self):
self.connection = MongoClient()
self.connection.drop_database(MONGO1_DBNAME)
self.connection[MONGO1_DBNAME].add_user(MONGO1_USERNAME,
MONGO1_PASSWORD)
self.bulk_insert2()
class TestMinimal(unittest.TestCase):
""" Start the building of the tests for an application
based on Eve by subclassing this class and provide proper settings
using :func:`setUp()`
"""
app = ValueStack(close_pymongo_connection)
def setUp(self, settings_file=None, url_converters=None):
""" Prepare the test fixture
:param settings_file: the name of the settings file. Defaults
to `eve/tests/test_settings.py`.
"""
self.this_directory = os.path.dirname(os.path.realpath(__file__))
if settings_file is None:
# Load the settings file, using a robust path
settings_file = os.path.join(self.this_directory,
'test_settings.py')
self.connection = None
self.known_resource_count = 101
self.setupDB()
self.settings_file = settings_file
self.app = eve.Eve(settings=self.settings_file,
url_converters=url_converters)
self.test_client = self.app.test_client()
self.domain = self.app.config['DOMAIN']
def tearDown(self):
del self.app
self.dropDB()
def assert200(self, status):
self.assertEqual(status, 200)
def assert201(self, status):
self.assertEqual(status, 201)
def assert301(self, status):
self.assertEqual(status, 301)
def assert404(self, status):
self.assertEqual(status, 404)
def assert304(self, status):
self.assertEqual(status, 304)
def get(self, resource, query='', item=None):
if resource in self.domain:
resource = self.domain[resource]['url']
if item:
request = '/%s/%s%s' % (resource, item, query)
else:
request = '/%s%s' % (resource, query)
r = self.test_client.get(request)
return self.parse_response(r)
def post(self, url, data, headers=None, content_type='application/json'):
if headers is None:
headers = []
headers.append(('Content-Type', content_type))
r = self.test_client.post(url, data=json.dumps(data), headers=headers)
return self.parse_response(r)
def put(self, url, data, headers=None):
if headers is None:
headers = []
headers.append(('Content-Type', 'application/json'))
r = self.test_client.put(url, data=json.dumps(data), headers=headers)
return self.parse_response(r)
def patch(self, url, data, headers=None):
if headers is None:
headers = []
headers.append(('Content-Type', 'application/json'))
r = self.test_client.patch(url, data=json.dumps(data), headers=headers)
return self.parse_response(r)
def delete(self, url, headers=None):
r = self.test_client.delete(url, headers=headers)
return self.parse_response(r)
def parse_response(self, r):
try:
v = json.loads(r.get_data())
except json.JSONDecodeError:
v = None
return v, r.status_code
def assertValidationError(self, response, matches):
self.assertTrue(eve.STATUS in response)
self.assertTrue(eve.STATUS_ERR in response[eve.STATUS])
self.assertTrue(ISSUES in response)
issues = response[ISSUES]
self.assertTrue(len(issues))
for k, v in matches.items():
self.assertTrue(k in issues)
self.assertTrue(v in issues[k])
def assertExpires(self, resource):
# TODO if we ever get access to response.date (it is None), compare
# it with Expires
r = self.test_client.get(resource)
expires = r.headers.get('Expires')
self.assertTrue(expires is not None)
def assertCacheControl(self, resource):
r = self.test_client.get(resource)
cache_control = r.headers.get('Cache-Control')
self.assertTrue(cache_control is not None)
self.assertEqual(cache_control,
self.domain[self.known_resource]['cache_control'])
def assertIfModifiedSince(self, resource):
r = self.test_client.get(resource)
last_modified = r.headers.get('Last-Modified')
self.assertTrue(last_modified is not None)
r = self.test_client.get(resource, headers=[('If-Modified-Since',
last_modified)])
self.assert304(r.status_code)
self.assertTrue(not r.get_data())
def assertItem(self, item):
self.assertEqual(type(item), dict)
updated_on = item.get(self.app.config['LAST_UPDATED'])
self.assertTrue(updated_on is not None)
try:
datetime.strptime(updated_on, self.app.config['DATE_FORMAT'])
except Exception as e:
self.fail('Cannot convert field "%s" to datetime: %s' %
(self.app.config['LAST_UPDATED'], e))
created_on = item.get(self.app.config['DATE_CREATED'])
self.assertTrue(updated_on is not None)
try:
datetime.strptime(created_on, self.app.config['DATE_FORMAT'])
except Exception as e:
self.fail('Cannot convert field "%s" to datetime: %s' %
(self.app.config['DATE_CREATED'], e))
link = item.get('_links')
_id = item.get(self.app.config['ID_FIELD'])
self.assertItemLink(link, _id)
def assertPagination(self, response, page, total, max_results):
self.assertTrue(self.app.config['META'] in response)
meta = response.get(self.app.config['META'])
self.assertTrue('page' in meta)
self.assertTrue('max_results' in meta)
self.assertTrue('total' in meta)
self.assertEqual(meta['page'], page)
self.assertEqual(meta['max_results'], max_results)
self.assertEqual(meta['total'], total)
def assertHomeLink(self, links):
self.assertTrue('parent' in links)
link = links['parent']
self.assertTrue('title' in link)
self.assertTrue('href' in link)
self.assertEqual('home', link['title'])
self.assertEqual("%s" % self._get_server_name(),
link['href'])
def assertResourceLink(self, links, resource):
self.assertTrue('self' in links)
link = links['self']
self.assertTrue('title' in link)
self.assertTrue('href' in link)
url = self.domain[resource]['url']
self.assertEqual(url, link['title'])
self.assertEqual("%s/%s" % (self._get_server_name(),
url),
link['href'])
def assertCollectionLink(self, links, resource):
self.assertTrue('collection' in links)
link = links['collection']
self.assertTrue('title' in link)
self.assertTrue('href' in link)
url = self.domain[resource]['url']
self.assertEqual(url, link['title'])
self.assertEqual("%s/%s" % (self._get_server_name(),
url), link['href'])
def assertNextLink(self, links, page):
self.assertTrue('next' in links)
link = links['next']
self.assertTrue('title' in link)
self.assertTrue('href' in link)
self.assertEqual('next page', link['title'])
self.assertTrue("page=%d" % page in link['href'])
def assertPrevLink(self, links, page):
self.assertTrue('prev' in links)
link = links['prev']
self.assertTrue('title' in link)
self.assertTrue('href' in link)
self.assertEqual('previous page', link['title'])
if page > 1:
self.assertTrue("page=%d" % page in link['href'])
def assertItemLink(self, links, item_id):
self.assertTrue('self' in links)
link = links['self']
# TODO we are too deep here to get a hold of the due title. Should fix.
self.assertTrue('title' in link)
self.assertTrue('href' in link)
self.assertTrue('/%s' % item_id in link['href'])
def assertLastLink(self, links, page):
if page:
self.assertTrue('last' in links)
link = links['last']
self.assertTrue('title' in link)
self.assertTrue('href' in link)
self.assertEqual('last page', link['title'])
self.assertTrue("page=%d" % page in link['href'])
else:
self.assertTrue('last' not in links)
def assert400(self, status):
self.assertEqual(status, 400)
def assert401(self, status):
self.assertEqual(status, 401)
def assert401or405(self, status):
self.assertTrue(status == 401 or 405)
def assert403(self, status):
self.assertEqual(status, 403)
def assert405(self, status):
self.assertEqual(status, 405)
def assert412(self, status):
self.assertEqual(status, 412)
def assert500(self, status):
self.assertEqual(status, 500)
def setupDB(self):
self.connection = MongoClient(MONGO_HOST, MONGO_PORT)
self.connection.drop_database(MONGO_DBNAME)
if MONGO_USERNAME:
self.connection[MONGO_DBNAME].add_user(MONGO_USERNAME,
MONGO_PASSWORD)
self.bulk_insert()
def bulk_insert(self):
pass
def _get_server_name(self):
server_name = self.app.config.get('SERVER_NAME', '')
url_protocol = self.app.config.get('URL_PROTOCOL', '')
if url_protocol:
server_name = '%s://%s' % (url_protocol, server_name)
return server_name
def dropDB(self):
self.connection = MongoClient(MONGO_HOST, MONGO_PORT)
self.connection.drop_database(MONGO_DBNAME)
self.connection.close()
def dropDB2(self):
self.connection = MongoClient()
self.connection.drop_database(MONGO1_DBNAME)
self.connection.close()
class TestBase(unittest.TestCase):
def setUp(self):
self.setupDB()
self.settings_file = 'eve/tests/test_settings.py'
self.app = Eve(settings=self.settings_file)
self.test_client = self.app.test_client()
self.domain = self.app.config['DOMAIN']
self.known_resource = 'contacts'
self.known_resource_url = ('/%s/' %
self.domain[self.known_resource]['url'])
self.empty_resource = 'empty'
self.empty_resource_url = '/%s/' % self.empty_resource
self.unknown_resource = 'unknown'
self.unknown_resource_url = '/%s/' % self.unknown_resource
self.unknown_item_id = '4f46445fc88e201858000000'
self.unknown_item_name = 'unknown'
self.unknown_item_id_url = ('/%s/%s/' %
(self.domain[self.known_resource]['url'],
self.unknown_item_id))
self.unknown_item_name_url = ('/%s/%s/' %
(self.domain[self.known_resource]['url'],
self.unknown_item_name))
self.readonly_resource = 'payments'
self.readonly_resource_url = (
'/%s/' % self.domain[self.readonly_resource]['url'])
self.different_resource = 'users'
self.different_resource_url = ('/%s/' %
self.domain[
self.different_resource]['url'])
def tearDown(self):
self.dropDB()
def assert200(self, status):
self.assertEqual(status, 200)
def assert301(self, status):
self.assertEqual(status, 301)
def assert404(self, status):
self.assertEqual(status, 404)
def assert304(self, status):
self.assertEqual(status, 304)
def setupDB(self):
self.connection = MongoClient()
self.connection.drop_database(MONGO_DBNAME)
self.connection[MONGO_DBNAME].add_user(MONGO_USERNAME, MONGO_PASSWORD)
self.bulk_insert()
def bulk_insert(self):
_db = self.connection[MONGO_DBNAME]
_db.contacts.insert(self.random_contacts(100))
_db.contacts.insert(self.random_users(2))
_db.payments.insert(self.random_payments(10))
_db.invoices.insert(self.random_invoices(1))
self.connection.close()
def dropDB(self):
self.connection = MongoClient()
self.connection.drop_database(MONGO_DBNAME)
self.connection.close()
def random_contacts(self, num, standard_date_fields=True):
schema = DOMAIN['contacts']['schema']
contacts = []
for i in range(num):
dt = datetime.now()
contact = {
'ref': self.random_string(schema['ref']['maxlength']),
'prog': i,
'role': random.choice(schema['role']['allowed']),
'rows': self.random_rows(random.randint(0, 5)),
'alist': self.random_list(random.randint(0, 5)),
'location': {
'address': 'address ' + self.random_string(5),
'city': 'city ' + self.random_string(3),
},
'born': datetime.today() + timedelta(
days=random.randint(-10, 10)),
'tid': ObjectId(),
}
if standard_date_fields:
contact[eve.LAST_UPDATED] = dt
contact[eve.DATE_CREATED] = dt
contacts.append(contact)
return contacts
def random_users(self, num):
users = self.random_contacts(num)
for user in users:
user['username'] = self.random_string(10)
return users
def random_payments(self, num):
payments = []
for i in range(num):
dt = datetime.now()
payment = {
'a_string': self.random_string(10),
'a_number': i,
eve.LAST_UPDATED: dt,
eve.DATE_CREATED: dt,
}
payments.append(payment)
return payments
def random_invoices(self, num):
invoices = []
for i in range(num):
dt = datetime.now()
invoice = {
'inv_number': self.random_string(10),
eve.LAST_UPDATED: dt,
eve.DATE_CREATED: dt,
}
invoices.append(invoice)
return invoices
def random_string(self, num):
return (''.join(random.choice(string.ascii_uppercase)
for x in range(num)))
def random_list(self, num):
alist = []
for i in range(num):
alist.append(['string' + str(i), random.randint(1000, 9999)])
return alist
def random_rows(self, num):
schema = DOMAIN['contacts']['schema']['rows']['schema']['schema']
rows = []
for i in range(num):
rows.append(
{
'sku': self.random_string(schema['sku']['maxlength']),
'price': random.randint(100, 1000),
}
)
return rows
import json
import pprint
import os
import datetime
app = Flask(__name__, static_url_path='/', static_folder='')
mongo_uri = os.getenv('MONGODB_URI')
db_name = os.getenv('MONGODB_NAME')
db = None
# if we are running in a heroku environment, or have a shared db, connect to that
if (mongo_uri):
with app.app_context():
assert db_name is not None # I'll eat a sock if this throws an error
db = MongoClient(mongo_uri)[db_name]
# else try to connect to local mongo instance
else:
with app.app_context():
db = PyMongo(app).db
@app.route('/', methods=['GET'])
def index():
sys.stdout.flush() # debugging heroku issue where stdout is buffered
return app.send_static_file('index.html')
@app.route('/js/<path:path>', methods=['GET'])
def send_js(path):
return send_from_directory('js', path)
class TestBase(unittest.TestCase):
def setUp(self):
self.app = eve_example.app
self.client = self.app.test_client()
self.domain = self.app.config['DOMAIN']
self.setupDb()
def tearDown(self):
self.dropDB()
del self.app
def setupDb(self):
self.connection = MongoClient(self.app.config['MONGO_URI'])
self.db = self.connection.get_default_database()
self.connection.drop_database(self.db)
self.bulk_insert()
def bulk_insert(self):
files = glob(os.path.join(os.path.dirname(__file__), '../data/*.json'))
for path in files:
collection = self.db[os.path.splitext(os.path.basename(path))[0]]
with open(path, 'r') as fh:
for line in fh:
collection.insert(json.loads(line))
def dropDB(self):
self.connection.drop_database(self.app.config['MONGO_DBNAME'])
self.connection.close()
self.connection = None
def epoch(self):
return datetime(1970, 1, 1)
def date_now(self):
return datetime.utcnow().replace(microsecond=0)
def basic_auth_headers(self, username, password):
auth = base64.b64encode((username + ':' + password).encode('ascii'))
return [('Authorization', 'Basic ' + auth.decode('ascii'))]
def auth(self, user):
return [('Authorization', 'Bearer %s' % user['token'])]
def admin_auth(self):
return self.basic_auth_headers('admin', self.app.config['ADMIN_PASSWORD'])
def resolve_resource(self, resource, item=None):
resource_params = []
if isinstance(resource, tuple):
resource_params = list(resource[0:-1])
resource = resource[-1]
if resource in self.domain:
resource = self.domain[resource]['url']
if resource_params:
resource = re.sub(r'<.*?>', lambda v: str(resource_params.pop()), resource)
if item:
if isinstance(item, dict):
item = item.get('id')
request = '/%s/%s' % (resource, str(item))
else:
request = '/%s' % (resource)
return request
def get(self, resource, item=None, query='', headers=[]):
url = self.resolve_resource(resource, item)
res = self.client.get(url + query, headers=headers)
return self.parse_response(res)
def post(self, resource, data, item=None, headers=[], content_type=None):
if not content_type:
content_type = 'application/json'
headers.append(('Content-Type', content_type))
url = self.resolve_resource(resource, item)
res = self.client.post(url, data=json.dumps(data), headers=headers)
return self.parse_response(res)
def put(self, resource, data, item=None, headers=[]):
headers.append(('Content-Type', 'application/json'))
url = self.resolve_resource(resource, item)
res = self.client.put(url, data=json.dumps(data), headers=headers)
return self.parse_response(res)
def patch(self, resource, data, item=None, headers=[]):
headers.append(('Content-Type', 'application/json'))
url = self.resolve_resource(resource, item)
res = self.client.patch(url, data=json.dumps(data), headers=headers)
return self.parse_response(res)
def delete(self, resource, item=None, headers=None):
url = self.resolve_resource(resource, item)
res = self.client.delete(url, headers=headers)
return self.parse_response(res)
def parse_response(self, res):
val = None
if res.get_data():
val = res.get_data().decode("utf-8")
try:
val = json.loads(val)
except ValueError:
self.fail("'%s' is not valid JSON" % (val))
return val, res.status_code
def create(self, resource, data, user=None):
if not user:
user = self.u.john
data, status = self.post(resource, data, headers=self.auth(user))
if status != 201:
print(data)
self.assertNoError(data)
return ObjectId(data['_id'])
def fetch(self, resource, item=None, user=None):
if not user:
user = self.u.john
data, status = self.get(resource, item=item, headers=self.auth(user))
self.assertNoError(data)
return data
def modify(self, resource, item, body, user=None):
if not user:
user = self.u.john
data, status = self.patch(resource, body, item=item,
headers=self.auth(user))
self.assertNoError(data)
return data
def remove(self, resource, item, user=None):
if not user:
user = self.u.john
data, status = self.delete(resource, item,
headers=self.auth(user))
self.assertNoError(data)
def assertNoError(self, response):
if response is None:
return
self.assertIsInstance(response, dict)
if eve.STATUS in response:
self.assertTrue(eve.STATUS_OK in response[eve.STATUS],
'status is not OK: %s' % response)
def assertValidationError(self, response, matches):
self.assertTrue(eve.STATUS in response, 'missing status')
self.assertTrue(eve.STATUS_ERR in response[eve.STATUS],
'status is not ERR')
self.assertTrue(ISSUES in response, 'response as no issues')
issues = response[ISSUES]
self.assertTrue(len(issues))
for k, v in matches.items():
self.assertTrue(k in issues, 'no "%s" error' % k)
self.assertTrue(v in issues[k],
'"%s" error does not contain "%s"' % (k, v))
def assertTimeEqual(self, time1, time2):
self.assertAlmostEqual(time1, time2, delta=timedelta(seconds=10))
class TestBase(unittest.TestCase):
def setUp(self):
""" Prepare the test fixture """
self.setupDB()
self.app = run.app
self.domain = self.app.config['DOMAIN']
self.test_client = self.app.test_client()
self.valid_auth = [('Authorization', 'Basic dG9rZW4xOg==')]
self.headers = [('Content-Type', 'application/json')] + self.valid_auth
def tearDown(self):
self.dropDB()
def assert200(self, status):
self.assertEqual(status, 200)
def assert201(self, status):
self.assertEqual(status, 201)
def assert204(self, status):
self.assertEqual(status, 204)
def assert401(self, status):
self.assertEqual(status, 401)
def setupDB(self):
self.connection = MongoClient(MONGO_HOST, MONGO_PORT)
self.connection.drop_database(MONGO_DBNAME)
if MONGO_USERNAME:
self.connection[MONGO_DBNAME].add_user(MONGO_USERNAME,
MONGO_PASSWORD)
db = self.connection[MONGO_DBNAME]
self.accounts_insert(db)
self.company_insert(db)
db.connection.close()
def accounts_insert(self, db):
user_accounts = [
{'u': '*****@*****.**', 'p': 'pw1', 't': 'token1', 'r': ['app']},
{'u': '*****@*****.**', 'p': 'pw1', 't': 'token2', 'r': ['user']},
]
db.user_accounts.insert(user_accounts)
def company_insert(self, db):
company = {'n': 'test_company', 'p': 'pw1'}
self.company_id = str(db.companies.insert(company))
def dropDB(self):
self.connection = MongoClient(MONGO_HOST, MONGO_PORT)
self.connection.drop_database(MONGO_DBNAME)
self.connection.close()
def post(self, url, payload):
r = self.test_client.post(url, data=json.dumps(payload),
headers=self.headers)
return self.parse_response(r)
def put(self, url, payload, etag):
headers = self.headers + [('If-Match', etag)]
r = self.test_client.put(url, data=json.dumps(payload),
headers=headers)
return self.parse_response(r)
def delete(self, url, etag):
headers = self.headers + [('If-Match', etag)]
r = self.test_client.delete(url, headers=headers)
return self.parse_response(r)
def get(self, url):
r = self.test_client.get(url, headers=self.headers)
return self.parse_response(r)
def parse_response(self, r):
try:
v = json.loads(r.get_data())
except json.JSONDecodeError:
v = None
return v, r.status_code
class TestMinimal(unittest.TestCase):
""" Start the building of the tests for an application
based on Eve by subclassing this class and provide proper settings
using :func:`setUp()`
"""
def setUp(self, settings_file=None):
""" Prepare the test fixture
:param settings_file: the name of the settings file. Defaults
to `eve/tests/test_settings.py`.
"""
if settings_file is None:
# Load the settings file, using a robust path
THIS_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
settings_file = os.path.join(THIS_DIRECTORY, 'test_settings.py')
self.known_resource_count = 101
self.setupDB()
self.settings_file = settings_file
self.app = eve.Eve(settings=self.settings_file)
self.test_client = self.app.test_client()
self.domain = self.app.config['DOMAIN']
def tearDown(self):
self.dropDB()
def assert200(self, status):
self.assertEqual(status, 200)
def assert301(self, status):
self.assertEqual(status, 301)
def assert404(self, status):
self.assertEqual(status, 404)
def assert304(self, status):
self.assertEqual(status, 304)
def get(self, resource, query='', item=None):
if resource in self.domain:
resource = self.domain[resource]['url']
if item:
request = '/%s/%s' % (resource, item)
else:
request = '/%s%s' % (resource, query)
r = self.test_client.get(request)
return self.parse_response(r)
def parse_response(self, r):
v = json.loads(r.get_data()) if r.status_code == 200 else None
return v, r.status_code
def assertValidationError(self, response, key, matches):
self.assertTrue(key in response)
k = response[key]
self.assertTrue('status' in k)
self.assertTrue(eve.STATUS_ERR in k['status'])
self.assertTrue('issues' in k)
issues = k['issues']
self.assertTrue(len(issues))
for match in matches:
self.assertTrue(match in issues[0])
def assertExpires(self, resource):
# TODO if we ever get access to response.date (it is None), compare
# it with Expires
r = self.test_client.get(resource)
expires = r.headers.get('Expires')
self.assertTrue(expires is not None)
def assertCacheControl(self, resource):
r = self.test_client.get(resource)
cache_control = r.headers.get('Cache-Control')
self.assertTrue(cache_control is not None)
self.assertEqual(cache_control,
self.domain[self.known_resource]['cache_control'])
def assertIfModifiedSince(self, resource):
r = self.test_client.get(resource)
last_modified = r.headers.get('Last-Modified')
self.assertTrue(last_modified is not None)
r = self.test_client.get(resource,
headers=[('If-Modified-Since', last_modified)
])
self.assert304(r.status_code)
self.assertTrue(not r.get_data())
def assertItem(self, item):
self.assertEqual(type(item), dict)
_id = item.get(self.app.config['ID_FIELD'])
self.assertTrue(_id is not None)
match = re.compile(self.app.config['ITEM_URL']).match(_id)
self.assertTrue(match is not None)
self.assertEqual(match.group(), _id)
updated_on = item.get(self.app.config['LAST_UPDATED'])
self.assertTrue(updated_on is not None)
try:
datetime.strptime(updated_on, self.app.config['DATE_FORMAT'])
except Exception as e:
self.fail('Cannot convert field "%s" to datetime: %s' %
(self.app.config['LAST_UPDATED'], e))
created_on = item.get(self.app.config['DATE_CREATED'])
self.assertTrue(updated_on is not None)
try:
datetime.strptime(created_on, self.app.config['DATE_FORMAT'])
except Exception as e:
self.fail('Cannot convert field "%s" to datetime: %s' %
(self.app.config['DATE_CREATED'], e))
link = item.get('_links')
self.assertItemLink(link, _id)
def assertHomeLink(self, links):
self.assertTrue('parent' in links)
link = links['parent']
self.assertTrue('title' in link)
self.assertTrue('href' in link)
self.assertEqual('home', link['title'])
self.assertEqual("%s" % self.app.config['SERVER_NAME'], link['href'])
def assertResourceLink(self, links, resource):
self.assertTrue('self' in links)
link = links['self']
self.assertTrue('title' in link)
self.assertTrue('href' in link)
url = self.domain[resource]['url']
self.assertEqual(url, link['title'])
self.assertEqual("%s/%s" % (self.app.config['SERVER_NAME'], url),
link['href'])
def assertCollectionLink(self, links, resource):
self.assertTrue('collection' in links)
link = links['collection']
self.assertTrue('title' in link)
self.assertTrue('href' in link)
url = self.domain[resource]['url']
self.assertEqual(url, link['title'])
self.assertEqual("%s/%s" % (self.app.config['SERVER_NAME'], url),
link['href'])
def assertNextLink(self, links, page):
self.assertTrue('next' in links)
link = links['next']
self.assertTrue('title' in link)
self.assertTrue('href' in link)
self.assertEqual('next page', link['title'])
self.assertTrue("page=%d" % page in link['href'])
def assertPrevLink(self, links, page):
self.assertTrue('prev' in links)
link = links['prev']
self.assertTrue('title' in link)
self.assertTrue('href' in link)
self.assertEqual('previous page', link['title'])
if page > 1:
self.assertTrue("page=%d" % page in link['href'])
def assertItemLink(self, links, item_id):
self.assertTrue('self' in links)
link = links['self']
#TODO we are too deep here to get a hold of the due title. Should fix.
self.assertTrue('title' in link)
self.assertTrue('href' in link)
self.assertTrue('/%s' % item_id in link['href'])
def assertLastLink(self, links, page):
if page:
self.assertTrue('last' in links)
link = links['last']
self.assertTrue('title' in link)
self.assertTrue('href' in link)
self.assertEqual('last page', link['title'])
self.assertTrue("page=%d" % page in link['href'])
else:
self.assertTrue('last' not in links)
def assert400(self, status):
self.assertEqual(status, 400)
def assert401(self, status):
self.assertEqual(status, 401)
def assert401or405(self, status):
self.assertTrue(status == 401 or 405)
def assert403(self, status):
self.assertEqual(status, 403)
def assert405(self, status):
self.assertEqual(status, 405)
def assert412(self, status):
self.assertEqual(status, 412)
def assert500(self, status):
self.assertEqual(status, 500)
def setupDB(self):
self.connection = MongoClient(MONGO_HOST, MONGO_PORT)
self.connection.drop_database(MONGO_DBNAME)
if MONGO_USERNAME:
self.connection[MONGO_DBNAME].add_user(MONGO_USERNAME,
MONGO_PASSWORD)
self.bulk_insert()
def bulk_insert(self):
pass
def dropDB(self):
self.connection = MongoClient(MONGO_HOST, MONGO_PORT)
self.connection.drop_database(MONGO_DBNAME)
self.connection.close()
def setupDb(self):
self.connection = MongoClient(self.app.config['MONGO_URI'])
self.db = self.connection.get_default_database()
self.connection.drop_database(self.db)
self.bulk_insert()
def init_user_debug(db=None, ctx=None):
# only activate if necessary.
if db is None:
# connect to database.
#connection = MongoClient(MONGO_HOST, MONGO_PORT)
connection = MongoClient(MONGO_URI)
# establish connection.
if MONGO_USERNAME:
connection[MONGO_DBNAME].add_user(MONGO_USERNAME, MONGO_PASSWORD)
# setup the db pointer
db = connection[MONGO_DBNAME]
# add testing user.
user_db = db['user']
team_db = db['team']
# add dummy first user.
user = {
"_id": ObjectId("4444ecb48a6ba828126f4444"),
"first_name": 'Jean-luc',
"last_name": 'Picard',
"title": "Captain",
"email": '*****@*****.**',
"token": 'abc123',
"roles": ["admin"],
'teams': []
}
user_id = user['_id']
user_token = user['token']
user_db.insert_one(user)
# insert the team first.
team = {"_id": ObjectId("66a52871a8c829842fbe618b"), "name": "test team"}
# insert it.
team_db.insert_one(team)
team_id = team['_id']
# add user.
user = {
"_id": ObjectId("5697ecb48a6ba828126f8128"),
"first_name": 'Jean-luc2',
"last_name": 'Picard2',
"title": "Captain2",
"email": '*****@*****.**',
"token": 'abc1234',
"roles": ["user"],
'teams': []
}
user_db.insert_one(user)
# insert the team first.
team = {"name": "bernd team"}
# insert it.
r, status = utility_post(ctx, 'team', team, token=user_token)
ctx.assert201(status)
# add user.
user = {
"_id": ObjectId("5697ecb48a6ba828126f8128"),
"first_name": 'Jean-luc3',
"last_name": 'Picard3',
"title": "Captain3",
"email": '*****@*****.**',
"token": 'abc1234',
"user_name": "NULL",
"roles": ["user"],
'teams': [r['_id']]
}
r, status = utility_post(ctx, 'user', user, token=user_token)
ctx.assert201(status)
# close db.
if db is None:
connection.close()
# return results.
return user_id, user_token, team_id
class TestMinimal(TestBase):
def setUp(self):
""" Prepare the test fixture
"""
self.setupDB()
super(TestMinimal, self).setUp()
self.app = Eve(settings='settings.py', auth=run.Auth)
self.test_client = self.app.test_client()
self.domain = self.app.config['DOMAIN']
def tearDown(self):
self.dropDB()
def assert200(self, status):
self.assertEqual(status, 200)
def assert301(self, status):
self.assertEqual(status, 301)
def assert404(self, status):
self.assertEqual(status, 404)
def assert304(self, status):
self.assertEqual(status, 304)
def assert400(self, status):
self.assertEqual(status, 400)
def assert401(self, status):
self.assertEqual(status, 401)
def assert401or405(self, status):
self.assertTrue(status == 401 or 405)
def assert403(self, status):
self.assertEqual(status, 403)
def assert405(self, status):
self.assertEqual(status, 405)
def assert412(self, status):
self.assertEqual(status, 412)
def assert500(self, status):
self.assertEqual(status, 500)
def setupDB(self):
self.connection = MongoClient(MONGO_HOST, MONGO_PORT)
self.connection.drop_database(MONGO_DBNAME)
if MONGO_USERNAME:
self.connection[MONGO_DBNAME].add_user(MONGO_USERNAME,
MONGO_PASSWORD)
self.bulk_insert()
def bulk_insert(self):
accounts = [
{
'u': '*****@*****.**',
'p': 'pw1',
't': 'token1',
'r': ['app']
},
{
'u': '*****@*****.**',
'p': 'pw1',
't': 'token2',
'r': ['user']
},
]
_db = self.connection[MONGO_DBNAME]
_db.accounts.insert(accounts)
self.connection.close()
def dropDB(self):
self.connection = MongoClient(MONGO_HOST, MONGO_PORT)
self.connection.drop_database(MONGO_DBNAME)
self.connection.close()
def bootstrap(ctx, forced=False):
# get the current time.
cur_dt = formatdate(time.mktime(datetime.now().timetuple()))
# connect to database.
#connection = MongoClient(MONGO_HOST, MONGO_PORT)
connection = MongoClient(MONGO_URI)
# establish connection.
if MONGO_USERNAME:
connection[MONGO_DBNAME].add_user(MONGO_USERNAME, MONGO_PASSWORD)
# setup the db pointer
db = connection[MONGO_DBNAME]
# check if the clinical and genomic empty.
if db[COLLECTION_CLINICAL].count() > 0 or db[COLLECTION_GENOMIC].count(
) > 0:
if not forced:
logging.warn(
"attempting to bootstrap the database without forcing")
return
logging.warn("forcibly resetting the database, may take some time")
# clear the database.
clear_db_completely()
connection.close()
# initialize user.
init_user_debug(ctx=ctx)
# load the clinical_df
clinical_df = clinical_load(DATA_CLINICAL_CSV)
# store unique keys for every sample.
keys = {}
# insert each clinical entry
new_clinical = 0
for entry in clinical_gen(clinical_df, clinical_schema=clinical['schema']):
# insert into db the clinical data.
resp, status_code = utility_post(ctx,
'clinical',
entry,
token='abc123')
if status_code != 201:
logging.error("unable to insert")
assert status_code == 201
new_clinical += 1
# get the unique key.
key = resp['_id']
# save it for later linking to genomics
keys[entry['SAMPLE_ID']] = key
# load the genomic_df.
genomic_df = genomic_load(DATA_GENOMIC_CSV)
# insert each genomic entry.
i = 0
new_genomic = 0
for entry in genomics_gen(genomic_df, genomic_schema=genomic['schema']):
# must have a clinical entry.
if entry['SAMPLE_ID'] not in keys:
continue
# hook entry with clinical_id.
id = keys[entry['SAMPLE_ID']]
entry['CLINICAL_ID'] = id
# insert it into db.
entry['WILDTYPE'] = bool(entry['WILDTYPE'])
resp, status_code = utility_post(ctx, 'genomic', entry, token='abc123')
if status_code != 201:
logging.error("unable to insert")
assert status_code == 201
#if i > 1000:
# break
i += 1
new_genomic += 1
# post the status update.
status = {
'last_update': cur_dt,
'new_clinical': new_clinical,
'updated_clinical': 0,
'new_genomic': new_genomic,
'updated_genomic': 0,
'silent': True
}
resp, status_code = utility_post(ctx, 'status', status, token='abc123')
if status_code != 201:
logging.error("unable to insert")
assert status_code == 201
class TestMinimal(unittest.TestCase):
""" Start the building of the tests for an application
based on Eve by subclassing this class and provide proper settings
using :func:`setUp()`
"""
app = ValueStack(close_pymongo_connection)
def setUp(self, settings_file=None, url_converters=None):
""" Prepare the test fixture
:param settings_file: the name of the settings file. Defaults
to `eve/tests/test_settings.py`.
"""
self.this_directory = os.path.dirname(os.path.realpath(__file__))
if settings_file is None:
# Load the settings file, using a robust path
settings_file = os.path.join(self.this_directory,
'test_settings.py')
self.connection = None
self.known_resource_count = 101
self.setupDB()
self.settings_file = settings_file
self.app = eve.Eve(settings=self.settings_file,
url_converters=url_converters)
self.test_client = self.app.test_client()
self.domain = self.app.config['DOMAIN']
def tearDown(self):
del self.app
self.dropDB()
def assert200(self, status):
self.assertEqual(status, 200)
def assert201(self, status):
self.assertEqual(status, 201)
def assert204(self, status):
self.assertEqual(status, 204)
def assert301(self, status):
self.assertEqual(status, 301)
def assert304(self, status):
self.assertEqual(status, 304)
def assert404(self, status):
self.assertEqual(status, 404)
def assert422(self, status):
self.assertEqual(status, 422)
def get(self, resource, query='', item=None):
if resource in self.domain:
resource = self.domain[resource]['url']
if item:
request = '/%s/%s%s' % (resource, item, query)
else:
request = '/%s%s' % (resource, query)
r = self.test_client.get(request)
return self.parse_response(r)
def post(self, url, data, headers=None, content_type='application/json'):
if headers is None:
headers = []
headers.append(('Content-Type', content_type))
r = self.test_client.post(url, data=json.dumps(data), headers=headers)
return self.parse_response(r)
def put(self, url, data, headers=None):
if headers is None:
headers = []
headers.append(('Content-Type', 'application/json'))
r = self.test_client.put(url, data=json.dumps(data), headers=headers)
return self.parse_response(r)
def patch(self, url, data, headers=None):
if headers is None:
headers = []
headers.append(('Content-Type', 'application/json'))
r = self.test_client.patch(url, data=json.dumps(data), headers=headers)
return self.parse_response(r)
def delete(self, url, headers=None):
r = self.test_client.delete(url, headers=headers)
return self.parse_response(r)
def parse_response(self, r):
try:
v = json.loads(r.get_data())
except json.JSONDecodeError:
v = None
return v, r.status_code
def assertValidationErrorStatus(self, status):
self.assertEqual(status,
self.app.config.get('VALIDATION_ERROR_STATUS'))
def assertValidationError(self, response, matches):
self.assertTrue(eve.STATUS in response)
self.assertTrue(eve.STATUS_ERR in response[eve.STATUS])
self.assertTrue(ISSUES in response)
issues = response[ISSUES]
self.assertTrue(len(issues))
for k, v in matches.items():
self.assertTrue(k in issues)
self.assertTrue(v in issues[k])
def assertExpires(self, resource):
# TODO if we ever get access to response.date (it is None), compare
# it with Expires
r = self.test_client.get(resource)
expires = r.headers.get('Expires')
self.assertTrue(expires is not None)
def assertCacheControl(self, resource):
r = self.test_client.get(resource)
cache_control = r.headers.get('Cache-Control')
self.assertTrue(cache_control is not None)
self.assertEqual(cache_control,
self.domain[self.known_resource]['cache_control'])
def assertIfModifiedSince(self, resource):
r = self.test_client.get(resource)
last_modified = r.headers.get('Last-Modified')
self.assertTrue(last_modified is not None)
r = self.test_client.get(resource,
headers=[('If-Modified-Since', last_modified)
])
self.assert304(r.status_code)
self.assertTrue(not r.get_data())
def assertItem(self, item, resource):
self.assertEqual(type(item), dict)
updated_on = item.get(self.app.config['LAST_UPDATED'])
self.assertTrue(updated_on is not None)
try:
datetime.strptime(updated_on, self.app.config['DATE_FORMAT'])
except Exception as e:
self.fail('Cannot convert field "%s" to datetime: %s' %
(self.app.config['LAST_UPDATED'], e))
created_on = item.get(self.app.config['DATE_CREATED'])
self.assertTrue(updated_on is not None)
try:
datetime.strptime(created_on, self.app.config['DATE_FORMAT'])
except Exception as e:
self.fail('Cannot convert field "%s" to datetime: %s' %
(self.app.config['DATE_CREATED'], e))
link = item.get('_links')
_id = item.get(self.domain[resource]['id_field'])
self.assertItemLink(link, _id)
def assertPagination(self, response, page, total, max_results):
p_key, mr_key = self.app.config['QUERY_PAGE'], \
self.app.config['QUERY_MAX_RESULTS']
self.assertTrue(self.app.config['META'] in response)
meta = response.get(self.app.config['META'])
self.assertTrue(p_key in meta)
self.assertTrue(mr_key in meta)
self.assertTrue('total' in meta)
self.assertEqual(meta[p_key], page)
self.assertEqual(meta[mr_key], max_results)
self.assertEqual(meta['total'], total)
def assertHomeLink(self, links):
self.assertTrue('parent' in links)
link = links['parent']
self.assertTrue('title' in link)
self.assertTrue('href' in link)
self.assertEqual('home', link['title'])
self.assertEqual("/", link['href'])
def assertResourceLink(self, links, resource):
self.assertTrue('self' in links)
link = links['self']
self.assertTrue('title' in link)
self.assertTrue('href' in link)
url = self.domain[resource]['url']
self.assertEqual(url, link['title'])
self.assertEqual("%s" % url, link['href'])
def assertCollectionLink(self, links, resource):
self.assertTrue('collection' in links)
link = links['collection']
self.assertTrue('title' in link)
self.assertTrue('href' in link)
url = self.domain[resource]['url']
self.assertEqual(url, link['title'])
self.assertEqual("%s" % url, link['href'])
def assertNextLink(self, links, page):
self.assertTrue('next' in links)
link = links['next']
self.assertTrue('title' in link)
self.assertTrue('href' in link)
self.assertEqual('next page', link['title'])
self.assertTrue("%s=%d" %
(self.app.config['QUERY_PAGE'], page) in link['href'])
def assertPrevLink(self, links, page):
self.assertTrue('prev' in links)
link = links['prev']
self.assertTrue('title' in link)
self.assertTrue('href' in link)
self.assertEqual('previous page', link['title'])
if page > 1:
self.assertTrue(
"%s=%d" %
(self.app.config['QUERY_PAGE'], page) in link['href'])
def assertItemLink(self, links, item_id):
self.assertTrue('self' in links)
link = links['self']
# TODO we are too deep here to get a hold of the due title. Should fix.
self.assertTrue('title' in link)
self.assertTrue('href' in link)
self.assertTrue('/%s' % item_id in link['href'])
def assertLastLink(self, links, page):
if page:
self.assertTrue('last' in links)
link = links['last']
self.assertTrue('title' in link)
self.assertTrue('href' in link)
self.assertEqual('last page', link['title'])
self.assertTrue(
"%s=%d" %
(self.app.config['QUERY_PAGE'], page) in link['href'])
else:
self.assertTrue('last' not in links)
def assert400(self, status):
self.assertEqual(status, 400)
def assert401(self, status):
self.assertEqual(status, 401)
def assert401or405(self, status):
self.assertTrue(status == 401 or 405)
def assert403(self, status):
self.assertEqual(status, 403)
def assert405(self, status):
self.assertEqual(status, 405)
def assert412(self, status):
self.assertEqual(status, 412)
def assert500(self, status):
self.assertEqual(status, 500)
def setupDB(self):
self.connection = MongoClient(MONGO_HOST, MONGO_PORT)
self.connection.drop_database(MONGO_DBNAME)
if MONGO_USERNAME:
self.connection[MONGO_DBNAME].add_user(MONGO_USERNAME,
MONGO_PASSWORD)
self.bulk_insert()
def bulk_insert(self):
pass
def dropDB(self):
self.connection = MongoClient(MONGO_HOST, MONGO_PORT)
self.connection.drop_database(MONGO_DBNAME)
self.connection.close()
def mongo_conn_user():
client_user = MongoClient(conn_user + ip_port0_user + auth_user)
db = client_user[db_name_user]
return db
class TestMinimal(unittest.TestCase):
""" Start the building of the tests for an application
based on Eve by subclassing this class and provide proper settings
using :func:`setUp()`
"""
app = ValueStack(close_pymongo_connection)
inserted_clinical = list()
inserted_genomic = list()
def setUp(self, settings_file=None, url_converters=None):
""" Prepare the test fixture
:param settings_file: the name of the settings file. Defaults
to `eve/tests/test_settings.py`.
"""
# settings.
self.this_directory = os.path.dirname(os.path.realpath(__file__))
if settings_file is None:
# Load the settings file, using a robust path
settings_file = os.path.join(self.this_directory,
'test_settings.py')
# default values.
self.connection = None
# create the engine.
self.cbio = engine.CBioEngine(MONGO_URI,
MONGO_DBNAME,
data_model.match_schema,
muser=MONGO_USERNAME,
mpass=MONGO_PASSWORD,
collection_clinical=COLLECTION_CLINICAL,
collection_genomic=COLLECTION_GENOMIC)
# setup the database.
self.setupDB()
# prepare the app
self.settings_file = settings_file
self.app = eve.Eve(settings=self.settings_file,
url_converters=url_converters,
auth=security.TokenAuth,
validator=ConsentValidatorEve)
# create the test client
self.test_client = self.app.test_client()
# set domain.
self.domain = self.app.config['DOMAIN']
# register hooks
self.app = register_hooks(self.app)
# register blueprints.
self.app.register_blueprint(blueprint)
# create the database pointer
self.db = self.connection[self.app.config['MONGO_DBNAME']]
# initialize the database if necessary.
self.initDB()
# clear backup.
# shutil.rmtree(BACKUP_DIR)
# os.makedirs(BACKUP_DIR)
def tearDown(self):
del self.app
self.dropDB()
def _insert_match_small(self):
# make a complex query.
dt = formatdate(time.mktime(datetime.datetime(year=1995, month=1, day=1).timetuple()))
c = {
"BIRTH_DATE": {"$gte": dt},
}
g = {
"TRUE_HUGO_SYMBOL": "BRCA2"
}
rule = {
'USER_ID': self.user_id,
'TEAM_ID': self.team_id,
'clinical_filter': c,
'genomic_filter': g,
'label': 'test',
'status': 1,
'temporary': False
}
# insert it.
r, status_code = self.post('filter', rule)
self.assert201(status_code)
# return the id.
return r['_id']
def _insert_filter(self):
# create a filter.
g = {
'TRUE_HUGO_SYMBOL': 'PRPF8',
'VARIANT_CATEGORY': 'MUTATION',
'WILDTYPE': False
}
rule = {
'USER_ID': self.user_id,
'TEAM_ID': self.team_id,
'genomic_filter': g,
'label': 'test',
'temporary': False,
'status': 1
}
r, status_code = self.post('filter', rule)
self.assert201(status_code)
# return the id.
return r['_id']
def _get_genomic_records(self, clinical_id):
# get all results.
return list(self.db['genomic'].find({'CLINICAL_ID': clinical_id}))
def _get_match_records(self, clinical_id):
# get all results.
return list(self.db['match'].find({'CLINICAL_ID': clinical_id}))
def assert200(self, status):
self.assertEqual(status, 200)
def assert201(self, status):
self.assertEqual(status, 201)
def assert204(self, status):
self.assertEqual(status, 204)
def assert301(self, status):
self.assertEqual(status, 301)
def assert304(self, status):
self.assertEqual(status, 304)
def assert404(self, status):
self.assertEqual(status, 404)
def assert422(self, status):
self.assertEqual(status, 422)
def assert405(self, status):
self.assertEqual(status, 405)
def _make_headers(self, headers=[]):
# add the defaults
headers.append(('Content-Type', 'application/json'))
headers.append(('Authorization', 'Basic ' + str(base64.b64encode(self.user_token + ':'))))
# just return list.
return headers
def get(self, resource, query='', item=None):
# make headers.
headers = self._make_headers()
if resource in self.domain:
resource = self.domain[resource]['url']
if item:
request = '/%s/%s%s' % (resource, item, query)
else:
request = '/%s%s' % (resource, query)
r = self.test_client.get('api' + request, headers=headers)
return self.parse_response(r)
def post(self, url, data, headers=None, content_type='application/json'):
# clear object ids.
for key in data:
try:
if isinstance(data[key], ObjectId):
data[key] = str(data[key])
except TypeError:
continue
# make headers.
if headers is None:
headers = self._make_headers()
else:
headers = self._make_headers(headers=headers)
r = self.test_client.post('api/' + url, data=json.dumps(data), headers=headers)
return self.parse_response(r)
def put(self, url, data, headers=None):
# make headers.
if headers is None:
headers = self._make_headers()
else:
headers = self._make_headers(headers=headers)
r = self.test_client.put('api/' + url, data=json.dumps(data), headers=headers)
return self.parse_response(r)
def patch(self, url, data, headers=None):
# make headers.
if headers is None:
headers = self._make_headers()
else:
headers = self._make_headers(headers=headers)
r = self.test_client.patch('api/' + url, data=json.dumps(data), headers=headers)
return self.parse_response(r)
def delete(self, url, headers=None):
# make headers.
if headers is None:
headers = self._make_headers()
else:
headers = self._make_headers(headers=headers)
r = self.test_client.delete('api/' + url, headers=headers)
return self.parse_response(r)
def parse_response(self, r):
try:
v = json.loads(r.get_data())
except ValueError:
v = None
return v, r.status_code
def assertValidationErrorStatus(self, status):
self.assertEqual(status,
self.app.config.get('VALIDATION_ERROR_STATUS'))
def assertValidationError(self, response, matches):
self.assertTrue(eve.STATUS in response)
self.assertTrue(eve.STATUS_ERR in response[eve.STATUS])
self.assertTrue(ISSUES in response)
issues = response[ISSUES]
self.assertTrue(len(issues))
for k, v in matches.items():
self.assertTrue(k in issues)
self.assertTrue(v in issues[k])
def assertExpires(self, resource):
# TODO if we ever get access to response.date (it is None), compare
# it with Expires
r = self.test_client.get(resource)
expires = r.headers.get('Expires')
self.assertTrue(expires is not None)
def assertCacheControl(self, resource):
r = self.test_client.get(resource)
cache_control = r.headers.get('Cache-Control')
self.assertTrue(cache_control is not None)
self.assertEqual(cache_control,
self.domain[self.known_resource]['cache_control'])
def assertIfModifiedSince(self, resource):
r = self.test_client.get(resource)
last_modified = r.headers.get('Last-Modified')
self.assertTrue(last_modified is not None)
r = self.test_client.get(resource, headers=[('If-Modified-Since',
last_modified)])
self.assert304(r.status_code)
self.assertTrue(not r.get_data())
def assertItem(self, item, resource):
self.assertEqual(type(item), dict)
updated_on = item.get(self.app.config['LAST_UPDATED'])
self.assertTrue(updated_on is not None)
try:
datetime.strptime(updated_on, self.app.config['DATE_FORMAT'])
except Exception as e:
self.fail('Cannot convert field "%s" to datetime: %s' %
(self.app.config['LAST_UPDATED'], e))
created_on = item.get(self.app.config['DATE_CREATED'])
self.assertTrue(updated_on is not None)
try:
datetime.strptime(created_on, self.app.config['DATE_FORMAT'])
except Exception as e:
self.fail('Cannot convert field "%s" to datetime: %s' %
(self.app.config['DATE_CREATED'], e))
link = item.get('_links')
_id = item.get(self.domain[resource]['id_field'])
self.assertItemLink(link, _id)
def assertPagination(self, response, page, total, max_results):
p_key, mr_key = self.app.config['QUERY_PAGE'], \
self.app.config['QUERY_MAX_RESULTS']
self.assertTrue(self.app.config['META'] in response)
meta = response.get(self.app.config['META'])
self.assertTrue(p_key in meta)
self.assertTrue(mr_key in meta)
self.assertTrue('total' in meta)
self.assertEqual(meta[p_key], page)
self.assertEqual(meta[mr_key], max_results)
self.assertEqual(meta['total'], total)
def assertHomeLink(self, links):
self.assertTrue('parent' in links)
link = links['parent']
self.assertTrue('title' in link)
self.assertTrue('href' in link)
self.assertEqual('home', link['title'])
self.assertEqual("/", link['href'])
def assertResourceLink(self, links, resource):
self.assertTrue('self' in links)
link = links['self']
self.assertTrue('title' in link)
self.assertTrue('href' in link)
url = self.domain[resource]['url']
self.assertEqual(url, link['title'])
self.assertEqual("%s" % url, link['href'])
def assertCollectionLink(self, links, resource):
self.assertTrue('collection' in links)
link = links['collection']
self.assertTrue('title' in link)
self.assertTrue('href' in link)
url = self.domain[resource]['url']
self.assertEqual(url, link['title'])
self.assertEqual("%s" % url, link['href'])
def assertNextLink(self, links, page):
self.assertTrue('next' in links)
link = links['next']
self.assertTrue('title' in link)
self.assertTrue('href' in link)
self.assertEqual('next page', link['title'])
self.assertTrue("%s=%d" % (self.app.config['QUERY_PAGE'], page)
in link['href'])
def assertPrevLink(self, links, page):
self.assertTrue('prev' in links)
link = links['prev']
self.assertTrue('title' in link)
self.assertTrue('href' in link)
self.assertEqual('previous page', link['title'])
if page > 1:
self.assertTrue("%s=%d" % (self.app.config['QUERY_PAGE'], page)
in link['href'])
def assertItemLink(self, links, item_id):
self.assertTrue('self' in links)
link = links['self']
# TODO we are too deep here to get a hold of the due title. Should fix.
self.assertTrue('title' in link)
self.assertTrue('href' in link)
self.assertTrue('/%s' % item_id in link['href'])
def assertLastLink(self, links, page):
if page:
self.assertTrue('last' in links)
link = links['last']
self.assertTrue('title' in link)
self.assertTrue('href' in link)
self.assertEqual('last page', link['title'])
self.assertTrue("%s=%d" % (self.app.config['QUERY_PAGE'], page)
in link['href'])
else:
self.assertTrue('last' not in links)
def assert400(self, status):
self.assertEqual(status, 400)
def assert401(self, status):
self.assertEqual(status, 401)
def assert401or405(self, status):
self.assertTrue(status == 401 or 405)
def assert403(self, status):
self.assertEqual(status, 403)
def assert405(self, status):
self.assertEqual(status, 405)
def assert412(self, status):
self.assertEqual(status, 412)
def assert500(self, status):
self.assertEqual(status, 500)
def initDB(self):
# check if the clinical and genomic empty.
if self.cbio._c.count() == 0 or self.cbio._g.count() == 0:
# bootstrap.
bootstrap(self)
# add testing user.
user_db = self.db['user']
team_db = self.db['team']
team = {
"_id": ObjectId("66a52871a8c829842fbe618b"),
"name": "purple monkey"
}
team_db.insert_one(team)
self.team_id = team['_id']
# add user and team.
user = {
"_id": ObjectId("5697ecb48a6ba828126f8128"),
"first_name": 'John',
"last_name": 'Doe',
"title": "M.D. PhD",
"email": 'jdoe@demo',
"token": 'abc123',
"user_name": "jd00",
'teams': [self.team_id],
'roles': ['user']
}
self.user_id = user['_id']
self.user_token = user['token']
user_db.insert_one(user)
# add service.
service = {
"first_name": 'SERVICE',
"last_name": 'SERVICE',
"title": "",
"email": 'jdoe@demo',
"token": 'xyz946',
"user_name": "SERVICE",
'teams': [],
'roles': ['service']
}
self.service_id = user_db.insert_one(service)
self.service_token = service['token']
# add curator
curator = {
"first_name": 'CURATOR',
"last_name": 'CURATOR',
"title": "",
"email": 'jdoe@demo',
"token": 'kjdhfg76',
"user_name": "CURATOR",
'teams': [],
'roles': ['curator']
}
self.curator_id = user_db.insert_one(curator)
self.curator_token = curator['token']
def setupDB(self):
# drop the database.
self.dropDB()
# connect to database.
self.connection = MongoClient(MONGO_URI)
# establish connection.
if MONGO_USERNAME:
self.connection[MONGO_DBNAME].add_user(MONGO_USERNAME,
MONGO_PASSWORD)
def dropDB(self):
#return
# connect to database.
self.connection = MongoClient(MONGO_URI)
# drop all tables except the genomic/clinical
db = self.connection[MONGO_DBNAME]
db.drop_collection("user")
db.drop_collection("team")
db.drop_collection("filter")
db.drop_collection("match")
db.drop_collection("hipaa")
db.drop_collection("status")
db.drop_collection("trial")
db.drop_collection("normalize")
db.drop_collection("email")
# clear extra clinical and genomic.
db['clinical'].delete_many({"TOTAL_READS" : 123, "ORD_PHYSICIAN_NPI": 0000})
db['genomic'].delete_many({'COVERAGE': 9123})
db['genomic'].delete_many({"STRUCTURAL_VARIANT_COMMENT": re.compile("tmp6654.*", re.IGNORECASE)})
for id in self.inserted_clinical:
db['clinical'].delete_one({'_id': ObjectId(id)})
for id in self.inserted_genomic:
db['genomic'].delete_one({'_id': ObjectId(id)})
# last check.
db['clinical'].delete_many({"SAMPLE_ID": "TCGA-OR-TEST1"})
# close connection.
self.connection.close()
def _insert_pair(self):
# insert clinical
clinical = self._insert_clinical()
# insert genomic.
genomic = self._insert_genomic(clinical['_id'])
# return the pair.
return clinical, genomic
def _insert_clinical(self, check=True):
# make an entry.
clinical = self.clinical.copy()
# insert it.
r, status_code = self.post('clinical', clinical)
if check:
self.assert201(status_code)
# record it.
if status_code == 201:
self.inserted_clinical.append(r['_id'])
# return it
return r
def _insert_genomic(self, clinical_id, data=False, check=True):
if data is False:
genomic = self.genomic.copy()
else:
genomic = data
# insert it.
r, status_code = self.post('genomic', genomic)
if check:
self.assert201(status_code)
# record it.
if status_code == 201:
self.inserted_genomic.append(r['_id'])
# return it
return r
# Simulated Data
cur_dt = formatdate(time.mktime(datetime.datetime.now().timetuple()))
bir_dt = formatdate(time.mktime(datetime.datetime(year=1995, month=1, day=1).timetuple()))
clinical = {
'_id': ObjectId(),
"PCT_TARGET_BASE": 0.6764308658975878,
"LAST_NAME": "Mcginnis[Fake]",
"TUMOR_PURITY_PERCENT": 0.6273209625954723,
"ONCOTREE_PRIMARY_DIAGNOSIS": "PHC",
"ONCOTREE_BIOPSY_SITE": "ADRENAL_GLAND",
"FIRST_NAME": "Michael",
"PATIENT_ID": "b3164f7b-c826-4e08-9ee6-8ff96d29b913",
"DATE_RECEIVED_AT_SEQ_CENTER": cur_dt,
"SAMPLE_ID": "TCGA-OR-TEST1",
"ONCOTREE_BIOPSY_SITE_TYPE": "Metastatic",
"TOTAL_READS": 123,
"ONCOTREE_PRIMARY_DIAGNOSIS_NAME": "Pheochromocytoma",
"PANEL_VERSION": 2,
"ONCOTREE_PRIMARY_DIAGNOSIS_META": "Pheochromocytoma",
"TOTAL_ALIGNED_READS": 10716821,
"POWERPATH_PATIENT_ID": "b3164f7b-c826-4e08-9ee6-8ff96d29b913",
"TEST_TYPE": "11-104 Profile",
"ORD_PHYSICIAN_NAME": "Maria Bellantoni [fake] M.D.",
"ONCOTREE_BIOPSY_SITE_META": "Pheochromocytoma",
"ONCOTREE_BIOPSY_SITE_COLOR": "Purple",
"MEAN_SAMPLE_COVERAGE": 147,
"VITAL_STATUS": "deceased",
"MRN": "b3164f7b-c826-4e08-9ee6-8ff96d29b913",
"ONCOTREE_BIOPSY_SITE_NAME": "Adrenal Gland",
"LAST_VISIT_DATE": cur_dt,
"REPORT_COMMENT": "age_at_initial_pathologic_diagnosis 58 ct_scan nan ct_scan_findings nan days_to_initial_pathologic_diagnosis 0 excess_adrenal_hormone_diagnosis_method_type nan excess_adrenal_hormone_history_type mineralocorticoids excess_adrenal_hormone_history_type-2 nan max nan nf1 nan nf1_clinical_diagnosis nan ret nan sdha nan sdhaf2_sdh5 nan sdhb nan sdhc nan sdhd nan tmem127 nan vhl nan molecular_analysis_performed_indicator no histological_type adrenocortical carcinoma- usual type laterality left lymph_node_examined_count nan metastatic_neoplasm_confirmed_diagnosis_method_name nan metastatic_neoplasm_confirmed_diagnosis_method_name-2 nan metastatic_neoplasm_confirmed_diagnosis_method_text nan distant_metastasis_anatomic_site nan metastatic_neoplasm_initial_diagnosis_anatomic_site nan metastatic_neoplasm_initial_diagnosis_anatomic_site-2 nan metastatic_neoplasm_initial_diagnosis_anatomic_site-3 nan mitoses_count 5 number_of_lymphnodes_positive_by_he nan primary_lymph_node_presentation_assessment nan residual_tumor r0 tumor_tissue_site adrenal atypical_mitotic_figures atypical mitotic figures absent cytoplasm_presence_less_than_equal_25_percent cytoplasm presence <= 25% present diffuse_architecture diffuse architecture present invasion_of_tumor_capsule invasion of tumor capsule absent mitotic_rate mitotic rate > 5/50 hpf absent necrosis necrosis present nuclear_grade_iii_iv nuclear grade iii or iv absent sinusoid_invasion sinusoid invasion absent weiss_venous_invasion venous invasion absent weiss_score 3 year_of_initial_pathologic_diagnosis 2000",
"ONCOTREE_PRIMARY_DIAGNOSIS_COLOR": "Purple",
"ORD_PHYSICIAN_NPI": 0000,
"DISEASE_CENTER_DESCR": "Adrenal Gland oncology",
"REPORT_DATE": cur_dt,
"BIRTH_DATE": bir_dt,
"ALT_MRN": "b3164f7b-c826-4e08-9ee6-8ff96d29b913",
"REPORT_VERSION": 1,
"GENDER": "Male",
"PATHOLOGIST_NAME": "Kacie Smith [fake] M.D.",
'QUESTION1_YN': "Y",
'QUESTION3_YN': "Y",
'CRIS_YN': "Y",
'QUESTION2_YN': 'Y',
'BLOCK_NUMBER': '',
'QUESTION5_YN': 'Y',
'ORD_PHYSICIAN_EMAIL': '',
'QUESTION4_YN': '',
'QC_RESULT': '',
'data_push_id': '2017-01-01 05:00:00'
}
genomic = {
'_id': ObjectId(),
"TRANSCRIPT_SOURCE": "NM_153618.1",
"BESTEFFECT_TRANSCRIPT_EXON": 7,
"TRUE_ENTREZ_ID": "80031",
"TRUE_VARIANT_CLASSIFICATION": "Missense_Mutation",
"CANONICAL_VARIANT_CLASSIFICATION": "Missense_Mutation",
"TRUE_STRAND": "+",
"TRUE_PROTEIN_CHANGE": "p.I160V",
"VARIANT_CATEGORY": "MUTATION",
"BESTEFFECT_VARIANT_CLASSIFICATION": "Missense_Mutation",
"TIER": 4,
"ALLELE_FRACTION": 0.7977839374344301,
"CANONICAL_STRAND": "+",
"BEST": True,
"CANONICAL_CDNA_TRANSCRIPT_ID": "ENST00000316364.5",
"ALTERNATE_ALLELE": "A",
"CHROMOSOME": "15",
"POSITION": 48053888,
"WILDTYPE": False,
"CLINICAL_ID": ObjectId(clinical['_id']),
"CANONICAL_TRANSCRIPT_EXON": 7,
"BESTEFFECT_HUGO_SYMBOL": "SEMA6D",
"REFERENCE_ALLELE": "A",
"COVERAGE": 0,
"BESTEFFECT_CDNA_CHANGE": "c.478A>G",
"TRUE_CDNA_CHANGE": "c.478A>G",
"TRUE_CDNA_TRANSCRIPT_ID": "ENST00000316364.5",
"TRUE_HUGO_SYMBOL": "SEMA6D",
"CANONICAL_CDNA_CHANGE": "c.478A>G",
"SAMPLE_ID": "TCGA-OR-TEST1",
"BESTEFFECT_ENTREZ_ID": "80031",
"SOMATIC_STATUS": "Somatic",
"BESTEFFECT_CDNA_TRANSCRIPT_ID": "ENST00000316364.5",
"REFERENCE_GENOME": "37",
"BESTEFFECT_PROTEIN_CHANGE": "p.I160V",
"TRUE_TRANSCRIPT_EXON": 7,
"CANONICAL_PROTEIN_CHANGE": "p.I160V",
"CANONICAL_ENTREZ_ID": "80031",
"CANONICAL_HUGO_SYMBOL": "SEMA6D",
}
status = {
"_id": ObjectId(),
"updated_genomic": 0,
"new_genomic": 6063,
"silent": False,
"new_clinical": 137,
"updated_clinical": 17,
"last_update": datetime.datetime(year=1995, month=1, day=1)
}
match = {
"_id": ObjectId(),
"MATCH_STATUS": 0,
"VARIANT_CATEGORY": "MUTATION",
"CLINICAL_ID": clinical['_id'],
"TRUE_HUGO_SYMBOL": "BRAF",
"TEAM_ID": ObjectId(),
"FILTER_STATUS": 1,
"FILTER_NAME": "MEK Inhibitor",
"VARIANTS": [
ObjectId()
],
"ONCOTREE_PRIMARY_DIAGNOSIS_NAME": "Diffuse Glioma",
"USER_ID": ObjectId(),
"MMID": "D47CE1",
"FILTER_ID": ObjectId(),
"PATIENT_MRN": "XXXXXX",
"EMAIL_SUBJECT": "",
"EMAIL_ADDRESS": "*****@*****.**",
"EMAIL_BODY": "",
}
trial_status_fields = ['drugs', 'genes', 'tumor_types', 'sponsor', 'coordinating_center',
'phase_summary', 'accrual_goal', 'investigator', 'age_summary', 'protocol_number',
'disease_status', 'nct_number', 'disease_center']
def add_remaining_required_fields(self, table):
"""
Given a clinical database table as a list of dictionaries, this method will iterate through each entry
and add in the remaining missing required clinical fields from the default self.clinical values
"""
existing_keys = []
for item in table:
existing_keys += item.keys()
existing_keys = list(set(existing_keys))
required_keys = self.clinical.keys()
for key in required_keys:
if key == '_id':
continue
if key not in existing_keys:
for item in table:
if isinstance(self.clinical[key], ObjectId):
item[key] = str(ObjectId())
else:
item[key] = self.clinical[key]
return table
def _check_email(self):
email = list(self.db['email'].find())
assert len(email) == 1
self.db['email'].drop()
@staticmethod
def _rand_alphanum(length):
return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(length))
dbnames = ['clinical', 'genomic', 'trial', 'normalize', 'filter', 'match']
def _savedb(self):
"""
Loads important collections into memory so the database
can be wiped for the unit test and then restored afterwards
"""
self.clinicaldb = list(self.db['clinical'].find())
self.genomicdb = list(self.db['genomic'].find())
self.trialdb = list(self.db['trial'].find())
self.normalizedb = list(self.db['normalize'].find())
self.filterdb = list(self.db['filter'].find())
self.matchdb = list(self.db['match'].find())
for dbname in self.dbnames:
self.db[dbname].drop()
def _restoredb(self):
for db, dbname in zip(
[self.clinicaldb, self.genomicdb, self.trialdb, self.normalizedb, self.filterdb, self.matchdb], self.dbnames
):
if db:
self.db[dbname].insert_many(db)
class TestMinimal(pea.TestCase):
""" Start the building of the tests for the PulsePod API
by subclassing this class and provide proper settings
using :func:`setUp()`
"""
def setUp(self, config=None):
""" Prepare the test fixture
:param config: the name of the config class. Defaults
to `testing`.
"""
super(TestMinimal, self).setUp()
self.this_directory = os.path.dirname(os.path.realpath(__file__))
if config is None:
config = 'testing'
elif config in ('production', 'development', 'heroku'):
assert 0, "Unable to run tests with %s config" % config
self.connection = None
self.app = create_app('testing')
self.app_context = self.app.app_context()
self.app_context.push()
self.setupDB()
self.test_client = self.app.test_client()
self.domain = self.app.config['DOMAIN']
self.config = self.app.config
self.DataMessage = {
'message_content': u'020f5be1018446df5314512d41b6028446df5' +
'339ecb6427438df53cee69f4255038446df538dc656427438df530dbe' +
'9442642adf53011aa641'
}
self.DeployMessage = {
'message_content': u'030f5b310026032229c58b436b4003e1b655'
}
def tearDown(self):
super(TestMinimal, self).tearDown()
self.dropDB()
self.app_context.pop()
del self.app
def assert200(self, status):
self.assertEqual(status, 200)
def assert201(self, status):
self.assertEqual(status, 201)
def assert301(self, status):
self.assertEqual(status, 301)
def assert404(self, status):
self.assertEqual(status, 404)
def assert304(self, status):
self.assertEqual(status, 304)
def get(self, resource, query='', item=None, headers=None):
if resource in self.domain:
resource = self.domain[resource]['url']
if item:
request = '/%s/%s%s' % (resource, item, query)
else:
request = '/%s%s' % (resource, query)
r = self.test_client.get(request)
return self.parse_response(r)
def post(self, url, data, headers=None, content_type='application/json'):
if headers is None:
headers = []
headers.append(('Content-Type', content_type))
r = self.test_client.post(url, data=json.dumps(data), headers=headers)
return self.parse_response(r)
def put(self, url, data, headers=None):
if headers is None:
headers = []
headers.append(('Content-Type', 'application/json'))
r = self.test_client.put(url, data=json.dumps(data), headers=headers)
return self.parse_response(r)
def patch(self, url, data, headers=None):
if headers is None:
headers = []
headers.append(('Content-Type', 'application/json'))
r = self.test_client.patch(url, data=json.dumps(data), headers=headers)
return self.parse_response(r)
def delete(self, url, headers=None):
r = self.test_client.delete(url, headers=headers)
return self.parse_response(r)
def bulk_insert(self):
pass
def parse_response(self, r):
try:
v = json.loads(r.get_data())
except json.JSONDecodeError:
v = None
return v, r.status_code
def setupDB(self):
settings = self.app.config['MONGODB_SETTINGS']
self.connection = MongoClient(settings['HOST'],
settings['PORT'])
self.connection.drop_database(settings['DB'])
if 'USERNAME' in settings:
self.connection[settings['DB']].add_user(
settings['USERNAME'], settings['PASSWORD'])
self.Pod = models.Pod
self.User = models.User
self.Message = models.Message
self.Notebook = models.Notebook
self.Sensor = models.Sensor
self.Data = models.Data
self.bulk_insert()
def dropDB(self):
settings = self.app.config['MONGODB_SETTINGS']
self.connection = MongoClient(
settings['HOST'],
settings['PORT'])
self.connection.drop_database(settings['DB'])
self.connection.close()
class TestMinimal(TestBase):
def setUp(self):
""" Prepare the test fixture
"""
self.setupDB()
super(TestMinimal, self).setUp()
self.app = Eve(settings='settings.py', auth=run.Auth)
self.test_client = self.app.test_client()
self.domain = self.app.config['DOMAIN']
def tearDown(self):
self.dropDB()
def assert200(self, status):
self.assertEqual(status, 200)
def assert301(self, status):
self.assertEqual(status, 301)
def assert404(self, status):
self.assertEqual(status, 404)
def assert304(self, status):
self.assertEqual(status, 304)
def assert400(self, status):
self.assertEqual(status, 400)
def assert401(self, status):
self.assertEqual(status, 401)
def assert401or405(self, status):
self.assertTrue(status == 401 or 405)
def assert403(self, status):
self.assertEqual(status, 403)
def assert405(self, status):
self.assertEqual(status, 405)
def assert412(self, status):
self.assertEqual(status, 412)
def assert500(self, status):
self.assertEqual(status, 500)
def setupDB(self):
self.connection = MongoClient(MONGO_HOST, MONGO_PORT)
self.connection.drop_database(MONGO_DBNAME)
if MONGO_USERNAME:
self.connection[MONGO_DBNAME].add_user(MONGO_USERNAME,
MONGO_PASSWORD)
self.bulk_insert()
def bulk_insert(self):
accounts = [
{'u': '*****@*****.**', 'p': 'pw1', 't': 'token1', 'r': ['app']},
{'u': '*****@*****.**', 'p': 'pw1', 't': 'token2', 'r': ['user']},
]
_db = self.connection[MONGO_DBNAME]
_db.accounts.insert(accounts)
self.connection.close()
def dropDB(self):
self.connection = MongoClient(MONGO_HOST, MONGO_PORT)
self.connection.drop_database(MONGO_DBNAME)
self.connection.close()
def mongo_conn():
client = MongoClient(conn + ip_port0 + auth)
db = client[db_name]
return db
class TestMinimal(unittest.TestCase):
""" Start the building of the tests for an application
based on Eve by subclassing this class and provide proper settings
using :func:`setUp()`
"""
def setUp(self, settings_file='eve/tests/test_settings.py'):
""" Prepare the test fixture
:param settings_file: the name of the settings file. Defaults
to `eve/tests/test_settings.py`.
"""
self.known_resource_count = 101
self.setupDB()
self.settings_file = settings_file
self.app = eve.Eve(settings=self.settings_file)
self.test_client = self.app.test_client()
self.domain = self.app.config['DOMAIN']
def tearDown(self):
self.dropDB()
def assert200(self, status):
self.assertEqual(status, 200)
def assert301(self, status):
self.assertEqual(status, 301)
def assert404(self, status):
self.assertEqual(status, 404)
def assert304(self, status):
self.assertEqual(status, 304)
def get(self, resource, query='', item=None):
if resource in self.domain:
resource = self.domain[resource]['url']
if item:
request = '/%s/%s/' % (resource, item)
else:
request = '/%s/%s' % (resource, query)
r = self.test_client.get(request)
return self.parse_response(r)
def parse_response(self, r):
v = json.loads(r.get_data()) if r.status_code == 200 else None
return v, r.status_code
def assertValidationError(self, response, key, matches):
self.assertTrue(key in response)
k = response[key]
self.assertTrue('status' in k)
self.assertTrue(eve.STATUS_ERR in k['status'])
self.assertTrue('issues' in k)
issues = k['issues']
self.assertTrue(len(issues))
for match in matches:
self.assertTrue(match in issues[0])
def assertExpires(self, resource):
# TODO if we ever get access to response.date (it is None), compare
# it with Expires
r = self.test_client.get(resource)
expires = r.headers.get('Expires')
self.assertTrue(expires is not None)
def assertCacheControl(self, resource):
r = self.test_client.get(resource)
cache_control = r.headers.get('Cache-Control')
self.assertTrue(cache_control is not None)
self.assertEqual(cache_control,
self.domain[self.known_resource]['cache_control'])
def assertIfModifiedSince(self, resource):
r = self.test_client.get(resource)
last_modified = r.headers.get('Last-Modified')
self.assertTrue(last_modified is not None)
r = self.test_client.get(resource, headers=[('If-Modified-Since',
last_modified)])
self.assert304(r.status_code)
self.assertTrue(not r.get_data())
def assertItem(self, item):
self.assertEqual(type(item), dict)
_id = item.get(self.app.config['ID_FIELD'])
self.assertTrue(_id is not None)
match = re.compile(self.app.config['ITEM_URL']).match(_id)
self.assertTrue(match is not None)
self.assertEqual(match.group(), _id)
updated_on = item.get(self.app.config['LAST_UPDATED'])
self.assertTrue(updated_on is not None)
try:
datetime.strptime(updated_on, self.app.config['DATE_FORMAT'])
except Exception as e:
self.fail('Cannot convert field "%s" to datetime: %s' %
(self.app.config['LAST_UPDATED'], e))
link = item.get('_links')
self.assertItemLink(link, _id)
def assertHomeLink(self, links):
self.assertTrue('parent' in links)
link = links['parent']
self.assertTrue('title' in link)
self.assertTrue('href' in link)
self.assertEqual('home', link['title'])
self.assertEqual("%s" % self.app.config['SERVER_NAME'], link['href'])
def assertResourceLink(self, links, resource):
self.assertTrue('self' in links)
link = links['self']
self.assertTrue('title' in link)
self.assertTrue('href' in link)
url = self.domain[resource]['url']
self.assertEqual(url, link['title'])
self.assertEqual("%s/%s/" % (self.app.config['SERVER_NAME'], url),
link['href'])
def assertCollectionLink(self, links, resource):
self.assertTrue('collection' in links)
link = links['collection']
self.assertTrue('title' in link)
self.assertTrue('href' in link)
url = self.domain[resource]['url']
self.assertEqual(url, link['title'])
self.assertEqual("%s/%s/" % (self.app.config['SERVER_NAME'], url),
link['href'])
def assertNextLink(self, links, page):
self.assertTrue('next' in links)
link = links['next']
self.assertTrue('title' in link)
self.assertTrue('href' in link)
self.assertEqual('next page', link['title'])
self.assertTrue("page=%d" % page in link['href'])
def assertPrevLink(self, links, page):
self.assertTrue('prev' in links)
link = links['prev']
self.assertTrue('title' in link)
self.assertTrue('href' in link)
self.assertEqual('previous page', link['title'])
if page > 1:
self.assertTrue("page=%d" % page in link['href'])
def assertItemLink(self, links, item_id):
self.assertTrue('self' in links)
link = links['self']
#TODO we are too deep here to get a hold of the due title. Should fix.
self.assertTrue('title' in link)
self.assertTrue('href' in link)
self.assertTrue('/%s/' % item_id in link['href'])
def assertLastLink(self, links, page):
if page:
self.assertTrue('last' in links)
link = links['last']
self.assertTrue('title' in link)
self.assertTrue('href' in link)
self.assertEqual('last page', link['title'])
self.assertTrue("page=%d" % page in link['href'])
else:
self.assertTrue('last' not in links)
def assert400(self, status):
self.assertEqual(status, 400)
def assert401(self, status):
self.assertEqual(status, 401)
def assert401or405(self, status):
self.assertTrue(status == 401 or 405)
def assert403(self, status):
self.assertEqual(status, 403)
def assert405(self, status):
self.assertEqual(status, 405)
def assert412(self, status):
self.assertEqual(status, 412)
def assert500(self, status):
self.assertEqual(status, 500)
def setupDB(self):
self.connection = MongoClient(MONGO_HOST, MONGO_PORT)
self.connection.drop_database(MONGO_DBNAME)
if MONGO_USERNAME:
self.connection[MONGO_DBNAME].add_user(MONGO_USERNAME,
MONGO_PASSWORD)
self.bulk_insert()
def bulk_insert(self):
pass
def dropDB(self):
self.connection = MongoClient(MONGO_HOST, MONGO_PORT)
self.connection.drop_database(MONGO_DBNAME)
self.connection.close()
def bootstrap(ctx, forced=False):
# get the current time.
cur_dt = formatdate(time.mktime(datetime.now().timetuple()))
# connect to database.
#connection = MongoClient(MONGO_HOST, MONGO_PORT)
connection = MongoClient(MONGO_URI)
# establish connection.
if MONGO_USERNAME:
connection[MONGO_DBNAME].add_user(MONGO_USERNAME, MONGO_PASSWORD)
# setup the db pointer
db = connection[MONGO_DBNAME]
# check if the clinical and genomic empty.
if db[COLLECTION_CLINICAL].count() > 0 or db[COLLECTION_GENOMIC].count() > 0:
if not forced:
logging.warn("attempting to bootstrap the database without forcing")
return
logging.warn("forcibly resetting the database, may take some time")
# clear the database.
clear_db_completely()
connection.close()
# initialize user.
init_user_debug(ctx=ctx)
# load the clinical_df
clinical_df = clinical_load(DATA_CLINICAL_CSV)
# store unique keys for every sample.
keys = {}
# insert each clinical entry
new_clinical = 0
for entry in clinical_gen(clinical_df, clinical_schema=clinical['schema']):
# insert into db the clinical data.
resp, status_code = utility_post(ctx, 'clinical', entry, token='abc123')
if status_code != 201:
logging.error("unable to insert")
assert status_code == 201
new_clinical += 1
# get the unique key.
key = resp['_id']
# save it for later linking to genomics
keys[entry['SAMPLE_ID']] = key
# load the genomic_df.
genomic_df = genomic_load(DATA_GENOMIC_CSV)
# insert each genomic entry.
i = 0
new_genomic = 0
for entry in genomics_gen(genomic_df, genomic_schema=genomic['schema']):
# must have a clinical entry.
if entry['SAMPLE_ID'] not in keys:
continue
# hook entry with clinical_id.
id = keys[entry['SAMPLE_ID']]
entry['CLINICAL_ID'] = id
# insert it into db.
entry['WILDTYPE'] = bool(entry['WILDTYPE'])
resp, status_code = utility_post(ctx, 'genomic', entry, token='abc123')
if status_code != 201:
logging.error("unable to insert")
assert status_code == 201
#if i > 1000:
# break
i += 1
new_genomic += 1
# post the status update.
status = {
'last_update': cur_dt,
'new_clinical': new_clinical,
'updated_clinical': 0,
'new_genomic': new_genomic,
'updated_genomic': 0,
'silent': True
}
resp, status_code = utility_post(ctx, 'status', status, token='abc123')
if status_code != 201:
logging.error("unable to insert")
assert status_code == 201
class TestBase(TestMinimal):
DEVICES = 'devices'
DEVICE_EVENT = 'events/devices'
EVENTS = 'events'
PLACES = 'places'
SNAPSHOT = 'snapshot'
ACCOUNTS = 'accounts'
def setUp(self, settings_file=None, url_converters=None):
from ereuse_devicehub import default_settings as settings
self.set_settings(settings)
self.app = DeviceHub()
self.prepare()
@staticmethod
def set_settings(settings):
settings.MONGO_DBNAME = 'devicehubtest'
settings.DATABASES = 'dht1', 'dht2' # Some tests use 2 databases
settings.DHT1_DBNAME = 'dht1_'
settings.DHT2_DBNAME = 'dht2_'
settings.GRD_DEBUG = True # We do not want to actually fulfill GRD
settings.APP_NAME = 'DeviceHub'
settings.DEBUG = True
settings.TESTING = True
settings.LOG = True
settings.GRD = False
settings.AGENT_ACCOUNTS = {
'self': ('*****@*****.**', '12345')
}
def prepare(self):
self.MONGO_DBNAME = self.app.config['MONGO_DBNAME']
self.MONGO_HOST = self.app.config['MONGO_HOST']
self.MONGO_PORT = self.app.config['MONGO_PORT']
self.DATABASES = self.app.config['DATABASES']
self.connection = None
self.setupDB()
self.test_client = self.app.test_client()
self.domain = self.app.config['DOMAIN']
self.token = self._login()
self.auth_header = ('authorization', 'Basic ' + self.token)
def setupDB(self):
self.connection = MongoClient(self.MONGO_HOST, self.MONGO_PORT)
self.db = self.connection[self.MONGO_DBNAME]
self.drop_databases()
self.create_dummy_user()
self.create_self_machine_account()
# We call the method again as we have erased the DB
self.app.grd_submitter_caller = SubmitterCaller(self.app, GRDSubmitter)
# self.app.grd_submitter_caller.token = self.app.grd_submitter_caller.prepare_user(self.app)
# self.app.grd_submitter_caller.process = None
def create_dummy_user(self):
self.db.accounts.insert(
{
'email': "[email protected]",
'password': AccountDomain.encrypt_password('1234'),
'role': 'admin',
'token': 'NOFATDNNUB',
'databases': self.app.config['DATABASES'],
'defaultDatabase': self.app.config['DATABASES'][0],
'@type': 'Account'
}
)
self.account = self.db.accounts.find_one({'email': '[email protected]'})
def create_self_machine_account(self):
email, password = self.app.config['AGENT_ACCOUNTS']['self']
self.db.accounts.insert(
{
'role': 'superuser',
'token': 'QYADFBPNZZDFJEWAFGGF',
'databases': self.app.config['DATABASES'],
'@type': 'Account',
'email': email,
'password': AccountDomain.encrypt_password(password)
}
)
def tearDown(self):
self.dropDB()
del self.app
def drop_databases(self):
self.connection.drop_database(self.MONGO_DBNAME)
for database in self.DATABASES:
self.connection.drop_database(self.app.config[database.upper().replace('-', '') + '_DBNAME'])
def dropDB(self):
self.drop_databases()
self.connection.close()
def full(self, resource_name: str, resource: dict or str or ObjectId) -> dict:
return resource if type(resource) is dict else self.get(resource_name, '', str(resource))[0]
def select_database(self, url):
if 'accounts' in url:
return ''
else:
return self.app.config['DATABASES'][0]
def get(self, resource, query='', item=None, authorize=True, database=None):
if resource in self.domain:
resource = self.domain[resource]['url']
if item:
request = '/%s/%s%s' % (resource, item, query)
else:
request = '/%s%s' % (resource, query)
database = database or self.select_database(resource)
return self._get(database + request, self.token if authorize else None)
def _get(self, url, token=None, **kwargs):
environ_base = {'HTTP_AUTHORIZATION': 'Basic ' + token} if token else {}
environ_base.update(kwargs.pop('environ_base', {}))
r = self.test_client.get(url, environ_base=environ_base, **kwargs)
return self.parse_response(r)
def post(self, url, data, headers=None, content_type='application/json'):
full_url = self.select_database(url) + '/' + url
headers = headers or []
return self._post(full_url, data, self.token, headers, content_type)
def _post(self, url, data, token, headers=None, content_type='application/json'):
headers = headers or []
headers.append(('authorization', 'Basic ' + token))
if type(data) is str:
headers.append(('Content-Type', content_type))
r = self.test_client.post(url, data=data, headers=headers)
return self.parse_response(r)
return super(TestBase, self).post(url, data, headers, content_type)
def patch(self, url, data, headers=None):
headers = headers or []
return self._patch(self.select_database(url) + '/' + url, data, self.token, headers)
def _patch(self, url, data, token, headers=None):
headers = headers or []
headers.append(('authorization', 'Basic ' + token))
return super(TestBase, self).patch(url, data, headers)
def put(self, url, data, headers=None):
headers = headers or []
return super(TestBase, self).put(self.select_database(url) + '/' + url, data, headers + [self.auth_header])
def delete(self, url, headers=None):
headers = headers or []
return super(TestBase, self).delete(self.select_database(url) + '/' + url, headers + [self.auth_header])
def _login(self) -> str:
return super(TestBase, self).post('/login', {"email": "[email protected]", "password": "******"})[0]['token']
def assert308(self, status):
self.assertEqual(status, 308)
def dropDB2(self):
self.connection = MongoClient()
self.connection.drop_database(MONGO1_DBNAME)
self.connection.close()
from flask import Flask
from flask.ext.pymongo import PyMongo, MongoClient
import base64
app = Flask(__name__)
client = MongoClient('<YOUR-MONGO-URI>')
mongo = client['<YOUR-MONGODB-NAME']
""" Routes """
@app.route("/")
def hello():
headers = create_headers()
columns = create_columns()
data = {'some_id': 1, 'headers': headers, 'columns': columns}
user = mongo.db.users
user.insert(data)
return 'added data'
@app.route("/get")
def get():
user = mongo.db.users
data = user.find_one({'some_id': 1})
headers = decode_headers(data['headers'])
columns = decode_columns(data['columns'])
print headers
for col in columns:
print '*' * 60
print col
def init_user_debug(db=None, ctx=None):
# only activate if necessary.
if db is None:
# connect to database.
#connection = MongoClient(MONGO_HOST, MONGO_PORT)
connection = MongoClient(MONGO_URI)
# establish connection.
if MONGO_USERNAME:
connection[MONGO_DBNAME].add_user(MONGO_USERNAME, MONGO_PASSWORD)
# setup the db pointer
db = connection[MONGO_DBNAME]
# add testing user.
user_db = db['user']
team_db = db['team']
# add dummy first user.
user = {
"_id": ObjectId("4444ecb48a6ba828126f4444"),
"first_name": 'Jean-luc',
"last_name": 'Picard',
"title": "Captain",
"email": '*****@*****.**',
"token": 'abc123',
"roles": ["admin"],
'teams': []
}
user_id = user['_id']
user_token = user['token']
user_db.insert_one(user)
# insert the team first.
team = {
"_id": ObjectId("66a52871a8c829842fbe618b"),
"name": "test team"
}
# insert it.
team_db.insert_one(team)
team_id = team['_id']
# add user.
user = {
"_id": ObjectId("5697ecb48a6ba828126f8128"),
"first_name": 'Jean-luc2',
"last_name": 'Picard2',
"title": "Captain2",
"email": '*****@*****.**',
"token": 'abc1234',
"roles": ["user"],
'teams': []
}
user_db.insert_one(user)
# insert the team first.
team = {
"name": "bernd team"
}
# insert it.
r, status = utility_post(ctx, 'team', team, token=user_token)
ctx.assert201(status)
# add user.
user = {
"_id": ObjectId("5697ecb48a6ba828126f8128"),
"first_name": 'Jean-luc3',
"last_name": 'Picard3',
"title": "Captain3",
"email": '*****@*****.**',
"token": 'abc1234',
"user_name": "NULL",
"roles": ["user"],
'teams': [r['_id']]
}
r, status = utility_post(ctx, 'user', user, token=user_token)
ctx.assert201(status)
# close db.
if db is None:
connection.close()
# return results.
return user_id, user_token, team_id
def dropDB(self):
self.connection = MongoClient(MONGO_HOST, MONGO_PORT)
self.connection.drop_database(MONGO_DBNAME)
self.connection.close()
class TestBase(unittest.TestCase):
def setUp(self):
self.setupDB()
self.settings_file = 'eve/tests/test_settings.py'
self.app = Eve(settings=self.settings_file)
self.test_client = self.app.test_client()
self.domain = self.app.config['DOMAIN']
self.known_resource = 'contacts'
self.known_resource_url = ('/%s/' %
self.domain[self.known_resource]['url'])
self.empty_resource = 'empty'
self.empty_resource_url = '/%s/' % self.empty_resource
self.unknown_resource = 'unknown'
self.unknown_resource_url = '/%s/' % self.unknown_resource
self.unknown_item_id = '4f46445fc88e201858000000'
self.unknown_item_name = 'unknown'
self.unknown_item_id_url = (
'/%s/%s/' %
(self.domain[self.known_resource]['url'], self.unknown_item_id))
self.unknown_item_name_url = (
'/%s/%s/' %
(self.domain[self.known_resource]['url'], self.unknown_item_name))
self.readonly_resource = 'payments'
self.readonly_resource_url = (
'/%s/' % self.domain[self.readonly_resource]['url'])
self.different_resource = 'users'
self.different_resource_url = (
'/%s/' % self.domain[self.different_resource]['url'])
def tearDown(self):
self.dropDB()
def assert200(self, status):
self.assertEqual(status, 200)
def assert301(self, status):
self.assertEqual(status, 301)
def assert404(self, status):
self.assertEqual(status, 404)
def assert304(self, status):
self.assertEqual(status, 304)
def setupDB(self):
self.connection = MongoClient()
self.connection.drop_database(MONGO_DBNAME)
self.connection[MONGO_DBNAME].add_user(MONGO_USERNAME, MONGO_PASSWORD)
self.bulk_insert()
def bulk_insert(self):
_db = self.connection[MONGO_DBNAME]
_db.contacts.insert(self.random_contacts(100))
_db.contacts.insert(self.random_users(2))
_db.payments.insert(self.random_payments(10))
_db.invoices.insert(self.random_invoices(1))
self.connection.close()
def dropDB(self):
self.connection = MongoClient()
self.connection.drop_database(MONGO_DBNAME)
self.connection.close()
def random_contacts(self, num, standard_date_fields=True):
schema = DOMAIN['contacts']['schema']
contacts = []
for i in range(num):
dt = datetime.now()
contact = {
'ref': self.random_string(schema['ref']['maxlength']),
'prog': i,
'role': random.choice(schema['role']['allowed']),
'rows': self.random_rows(random.randint(0, 5)),
'alist': self.random_list(random.randint(0, 5)),
'location': {
'address': 'address ' + self.random_string(5),
'city': 'city ' + self.random_string(3),
},
'born':
datetime.today() + timedelta(days=random.randint(-10, 10)),
'tid': ObjectId(),
}
if standard_date_fields:
contact[eve.LAST_UPDATED] = dt
contact[eve.DATE_CREATED] = dt
contacts.append(contact)
return contacts
def random_users(self, num):
users = self.random_contacts(num)
for user in users:
user['username'] = self.random_string(10)
return users
def random_payments(self, num):
payments = []
for i in range(num):
dt = datetime.now()
payment = {
'a_string': self.random_string(10),
'a_number': i,
eve.LAST_UPDATED: dt,
eve.DATE_CREATED: dt,
}
payments.append(payment)
return payments
def random_invoices(self, num):
invoices = []
for i in range(num):
dt = datetime.now()
invoice = {
'inv_number': self.random_string(10),
eve.LAST_UPDATED: dt,
eve.DATE_CREATED: dt,
}
invoices.append(invoice)
return invoices
def random_string(self, num):
return (''.join(
random.choice(string.ascii_uppercase) for x in range(num)))
def random_list(self, num):
alist = []
for i in range(num):
alist.append(['string' + str(i), random.randint(1000, 9999)])
return alist
def random_rows(self, num):
schema = DOMAIN['contacts']['schema']['rows']['schema']['schema']
rows = []
for i in range(num):
rows.append({
'sku': self.random_string(schema['sku']['maxlength']),
'price': random.randint(100, 1000),
})
return rows
from app import app
from flask.ext.pymongo import PyMongo, MongoClient
from py2neo import Graph
# connect to MongoDB with the defaults
mongo1 = PyMongo(app)
## connect to another MongoDB database on the same host
app.config['MONGO2_DBNAME'] = 'evgroio-dev'
mongo2 = PyMongo(app, config_prefix='MONGO2')
app.config['MONGO3_DBNAME'] = 'test'
mongo3 = PyMongo(app, config_prefix='MONGO3')
# Affective Computing Databases
app.config['ANALYSIS_DBNAME'] = 'affect-analysis-interceptor'
affect_analysis = PyMongo(app, config_prefix='ANALYSIS')
#remoteDB1
mongolab_uri = 'mongodb://*****:*****@ds041238.mongolab.com:41238/heroku_app36697506'
client = MongoClient(mongolab_uri,
connectTimeoutMS=30000,
socketTimeoutMS=None,
socketKeepAlive=True)
remoteDB1 = client.get_default_database()
## connect to default instance of neo4j
secure_graph1 = Graph("http://*****:*****@0.0.0.0:7474/db/data/")
def setupDB2(self):
self.connection = MongoClient()
self.connection.drop_database(MONGO1_DBNAME)
self.connection[MONGO1_DBNAME].add_user(MONGO1_USERNAME,
MONGO1_PASSWORD)
self.bulk_insert2()
def dropDB(self):
self.connection = MongoClient(MONGO_HOST, MONGO_PORT)
self.connection.drop_database(MONGO_DBNAME)
self.connection.close()
class CBioEngine(object):
connection = None
clinical_df = None
genomic_df = None
match_df = None
living_genomic = 0
def __init__(self, mongo_uri, mongo_dbname, match_schema, muser=None, mpass=None,
collection_clinical="clinical", collection_genomic="genomic"):
# save these commmon variables.
self.mongo_uri = mongo_uri
self.mongo_dbname = mongo_dbname
self.muser = muser
self.mpass = mpass
self.collection_clinical = collection_clinical
self.collection_genomic = collection_genomic
# establish database connection.
self._setup_db()
# set variables.
self.match_schema = match_schema
def match(self, c=None, g=None, c_fields=None, g_fields=None):
logging.info("matching: num_clinical=%d, num_genomic=%d" % (self._c.count(), self._g.count()))
self.match_clinical(c=c, c_fields=c_fields)
self.match_genomic(g=g, g_fields=g_fields)
if self.clinical_df.shape[0] != 0 and self.genomic_df.shape[0] != 0:
match_df = pd.merge(self.clinical_df,
self.genomic_df,
how='inner',
left_on=['_id', 'SAMPLE_ID'],
right_on=['CLINICAL_ID', 'SAMPLE_ID'])
match_all_df = match_df.copy()
alive_query = {"VITAL_STATUS": "alive"}
alive_patients = list(self._c.find(alive_query))
alive_ids = [item['SAMPLE_ID'] for item in alive_patients]
match_df.fillna(value='alive', inplace=True)
match_df = match_df[match_df['SAMPLE_ID'].isin(alive_ids)]
self.genomic_df = self.genomic_df[self.genomic_df['SAMPLE_ID'].isin(alive_ids)]
self.living_genomic = self.genomic_df.shape[0]
else:
match_df = None
match_all_df = None
alive_ids = self._c.find({}).distinct("SAMPLE_ID")
if self.genomic_df.shape[0] == 0:
self.living_genomic = 0
else:
self.genomic_df = self.genomic_df[self.genomic_df['SAMPLE_ID'].isin(alive_ids)]
self.living_genomic = self.genomic_df.shape[0]
self.match_df = match_df
self.match_all_df = match_all_df
return self.match_df
def match_clinical(self, c=None, c_fields=None):
# fix empty.
if c is None:
c = {}
# query each table.
if c_fields is None:
results = self._c.find(c, {'_id': 1, 'SAMPLE_ID': 1, 'REPORT_DATE': 1, 'VITAL_STATUS': 1})
else:
c_fields['SAMPLE_ID'] = 1
c_fields['VITAL_STATUS'] = 1
c_fields['_id'] = 1
results = self._c.find(c, c_fields)
# convert to dataframe.
clinical_df = pd.DataFrame(list(results))
# save as global var.
self.clinical_df = clinical_df
# return it.
return self.clinical_df
def match_genomic(self, g=None, g_fields=None):
# fix empty.
if g is None:
g = {}
# query each table.
if g_fields is None:
results = self._g.find(g, {'CLINICAL_ID': 1, 'SAMPLE_ID': 1})
else:
g_fields['SAMPLE_ID'] = 1
g_fields['CLINICAL_ID'] = 1
results = self._g.find(g, g_fields)
# convert to dataframe.
genomic_df = pd.DataFrame(list(results))
# save as global var.
self.genomic_df = genomic_df
# return it.
return self.genomic_df
def match_iter(self):
# short circuit on empty
if self.match_df is None:
return
# create validator.
v = Validator(self.match_schema)
# names cols.
cis = zip(range(len(self.match_df.columns)), list(self.match_df.columns))
# strip out underscore fields.
tmp = []
for a, b in cis:
if b[0] == "_" and b != "_id_y":
continue
tmp.append((a,b))
cis = tmp
# loop over each entry.
for row in self.match_df.itertuples():
# create dictionary.
tmp = {}
for i, key in cis:
# extract value.
val = row[i+1]
# convert nan
if pd.isnull(val):
val = None
# convert special key.
if key == "_id_y":
key = "GENOMIC_ID"
# save it to dict.
tmp[key] = val
# yield the match
yield tmp
def _setup_db(self):
# connect to database.
self.connection = MongoClient(self.mongo_uri)
# make user.
if self.muser is not None:
self.connection[self.mongo_dbname].add_user(self.muser, self.mpass)
# establish the collection interface.
self._c = self.connection[self.mongo_dbname][self.collection_clinical]
self._g = self.connection[self.mongo_dbname][self.collection_genomic]
def _close_db(self):
self.connection.close()
class CBioEngine(object):
connection = None
clinical_df = None
genomic_df = None
match_df = None
living_genomic = 0
def __init__(self,
mongo_uri,
mongo_dbname,
match_schema,
muser=None,
mpass=None,
collection_clinical="clinical",
collection_genomic="genomic"):
# save these commmon variables.
self.mongo_uri = mongo_uri
self.mongo_dbname = mongo_dbname
self.muser = muser
self.mpass = mpass
self.collection_clinical = collection_clinical
self.collection_genomic = collection_genomic
# establish database connection.
self._setup_db()
# set variables.
self.match_schema = match_schema
def match(self, c=None, g=None, c_fields=None, g_fields=None):
logging.info("matching: num_clinical=%d, num_genomic=%d" %
(self._c.count(), self._g.count()))
self.match_clinical(c=c, c_fields=c_fields)
self.match_genomic(g=g, g_fields=g_fields)
if self.clinical_df.shape[0] != 0 and self.genomic_df.shape[0] != 0:
match_df = pd.merge(self.clinical_df,
self.genomic_df,
how='inner',
left_on=['_id', 'SAMPLE_ID'],
right_on=['CLINICAL_ID', 'SAMPLE_ID'])
match_all_df = match_df.copy()
alive_query = {"VITAL_STATUS": "alive"}
alive_patients = list(self._c.find(alive_query))
alive_ids = [item['SAMPLE_ID'] for item in alive_patients]
match_df.fillna(value='alive', inplace=True)
match_df = match_df[match_df['SAMPLE_ID'].isin(alive_ids)]
self.genomic_df = self.genomic_df[
self.genomic_df['SAMPLE_ID'].isin(alive_ids)]
self.living_genomic = self.genomic_df.shape[0]
else:
match_df = None
match_all_df = None
alive_ids = self._c.find({}).distinct("SAMPLE_ID")
if self.genomic_df.shape[0] == 0:
self.living_genomic = 0
else:
self.genomic_df = self.genomic_df[
self.genomic_df['SAMPLE_ID'].isin(alive_ids)]
self.living_genomic = self.genomic_df.shape[0]
self.match_df = match_df
self.match_all_df = match_all_df
return self.match_df
def match_clinical(self, c=None, c_fields=None):
# fix empty.
if c is None:
c = {}
# query each table.
if c_fields is None:
results = self._c.find(c, {
'_id': 1,
'SAMPLE_ID': 1,
'REPORT_DATE': 1,
'VITAL_STATUS': 1
})
else:
c_fields['SAMPLE_ID'] = 1
c_fields['VITAL_STATUS'] = 1
c_fields['_id'] = 1
results = self._c.find(c, c_fields)
# convert to dataframe.
clinical_df = pd.DataFrame(list(results))
# save as global var.
self.clinical_df = clinical_df
# return it.
return self.clinical_df
def match_genomic(self, g=None, g_fields=None):
# fix empty.
if g is None:
g = {}
# query each table.
if g_fields is None:
results = self._g.find(g, {'CLINICAL_ID': 1, 'SAMPLE_ID': 1})
else:
g_fields['SAMPLE_ID'] = 1
g_fields['CLINICAL_ID'] = 1
results = self._g.find(g, g_fields)
# convert to dataframe.
genomic_df = pd.DataFrame(list(results))
# save as global var.
self.genomic_df = genomic_df
# return it.
return self.genomic_df
def match_iter(self):
# short circuit on empty
if self.match_df is None:
return
# create validator.
v = Validator(self.match_schema)
# names cols.
cis = zip(range(len(self.match_df.columns)),
list(self.match_df.columns))
# strip out underscore fields.
tmp = []
for a, b in cis:
if b[0] == "_" and b != "_id_y":
continue
tmp.append((a, b))
cis = tmp
# loop over each entry.
for row in self.match_df.itertuples():
# create dictionary.
tmp = {}
for i, key in cis:
# extract value.
val = row[i + 1]
# convert nan
if pd.isnull(val):
val = None
# convert special key.
if key == "_id_y":
key = "GENOMIC_ID"
# save it to dict.
tmp[key] = val
# yield the match
yield tmp
def _setup_db(self):
# connect to database.
self.connection = MongoClient(self.mongo_uri)
# make user.
if self.muser is not None:
self.connection[self.mongo_dbname].add_user(self.muser, self.mpass)
# establish the collection interface.
self._c = self.connection[self.mongo_dbname][self.collection_clinical]
self._g = self.connection[self.mongo_dbname][self.collection_genomic]
def _close_db(self):
self.connection.close()
#!/usr/bin/python
#coding: utf-8
from flask import Flask
from flask.ext.pymongo import MongoClient
from flask.ext.httpauth import HTTPBasicAuth
# from flask.ext.sqlalchemy import SQLAlchemy
from flask.ext.security import SQLAlchemyUserDatastore, Security
from App import config
app = Flask(__name__)
app.config.from_object(config)
# db = SQLAlchemy(app)
client = MongoClient()
auth = HTTPBasicAuth()
from App.models import User #, Role, roles_users
from App.routes import api
# # Flask-Security
# user_datastore = SQLAlchemyUserDatastore(db, User, Role)
# security = Security().init_app(app, user_datastore, register_blueprint=False)
#
# # init database data
# try:
# # db.create_all()
# # db.session.add(User("test1", "test1"))
# # db.session.add(User("test2", "test2"))
# # db.session.add(Role("admin", "管理员"))
# # db.session.commit()
# # db.engine.execute(roles_users.insert(), user_id=1, role_id=1)