def post(self):
form = ThirdPartyData()
form.survey.choices = [(x, x) for x in db.list_external_surveys()]
msg = ''
seperators = {'comma': ',', 'tab': '\t', 'space': ' '}
args = {a: v[0] for a, v in viewitems(self.request.arguments)}
form.process(data=args)
# Validate form and make sure upload happened
if not form.validate() or 'file_in' not in self.request.files:
self.render("ag_third_party.html", the_form=form,
errors=msg)
return
# Format file for stringIO
file_body = self.request.files['file_in'][0]['body'].replace(
"\r\n", "\n").replace("\r", "\n")
file_body = StringIO(unicode(file_body), newline=None)
try:
count = db.store_external_survey(
file_body, form.survey.data,
separator=seperators[form.seperator.data],
survey_id_col=form.survey_id.data, trim=form.trim.data)
except KeyError as e:
msg = 'Header column not found: %s' % str(e)
except Exception as e:
# Print any error that happens to the page
msg = str(e)
else:
msg = "%d surveys added to '%s' successfully" % \
(count, form.survey.data)
self.render("ag_third_party.html", the_form=form,
errors=msg)
def test_pulldown_third_party(self):
# Add survey answers
with open(self.ext_survey_fp, 'rU') as f:
obs = db.store_external_survey(
f, 'Vioscreen', separator=',', survey_id_col='SubjectId',
trim='-160')
self.assertEqual(obs, 3)
barcodes = ['000029429', '000018046', '000023299', '000023300']
# Test without third party
obs, _ = db.pulldown(barcodes)
survey = obs[1]
self.assertFalse('VIOSCREEN' in survey)
obs, _ = db.pulldown(barcodes, blanks=['BLANK.01'])
survey = obs[1]
self.assertFalse('VIOSCREEN' in survey)
self.assertTrue('BLANK.01' in survey)
# Test with third party
obs, _ = db.pulldown(barcodes, external=['Vioscreen'])
survey = obs[1]
self.assertTrue('VIOSCREEN' in survey)
obs, _ = db.pulldown(barcodes, blanks=['BLANK.01'],
external=['Vioscreen'])
survey = obs[1]
self.assertTrue('VIOSCREEN' in survey)
self.assertTrue('BLANK.01' in survey)
def test_pulldown_third_party(self):
# Add survey answers
with open(self.ext_survey_fp, 'rU') as f:
obs = db.store_external_survey(f,
'Vioscreen',
separator=',',
survey_id_col='SubjectId',
trim='-160')
self.assertEqual(obs, 3)
barcodes = ['000029429', '000018046', '000023299', '000023300']
# Test without third party
obs, _ = db.pulldown(barcodes)
survey = obs[1]
self.assertFalse('VIOSCREEN' in survey)
obs, _ = db.pulldown(barcodes, blanks=['BLANK.01'])
survey = obs[1]
self.assertFalse('VIOSCREEN' in survey)
self.assertTrue('BLANK.01' in survey)
# Test with third party
obs, _ = db.pulldown(barcodes, external=['Vioscreen'])
survey = obs[1]
self.assertTrue('VIOSCREEN' in survey)
obs, _ = db.pulldown(barcodes,
blanks=['BLANK.01'],
external=['Vioscreen'])
survey = obs[1]
self.assertTrue('VIOSCREEN' in survey)
self.assertTrue('BLANK.01' in survey)
def post(self):
form = ThirdPartyData()
form.survey.choices = [(x, x) for x in db.list_external_surveys()]
msg = ''
seperators = {'comma': ',', 'tab': '\t', 'space': ' '}
args = {a: v[0] for a, v in viewitems(self.request.arguments)}
form.process(data=args)
# Validate form and make sure upload happened
if not form.validate() or 'file_in' not in self.request.files:
self.render("ag_third_party.html", the_form=form, errors=msg)
return
# Format file for stringIO
file_body = self.request.files['file_in'][0]['body'].replace(
"\r\n", "\n").replace("\r", "\n")
file_body = StringIO(unicode(file_body), newline=None)
try:
count = db.store_external_survey(
file_body,
form.survey.data,
separator=seperators[form.seperator.data],
survey_id_col=form.survey_id.data,
trim=form.trim.data)
except KeyError as e:
msg = 'Header column not found: %s' % str(e)
except Exception as e:
# Print any error that happens to the page
msg = str(e)
else:
msg = "%d surveys added to '%s' successfully" % \
(count, form.survey.data)
self.render("ag_third_party.html", the_form=form, errors=msg)
def test_pulldown_third_party(self):
# Add survey answers
with open(self.ext_survey_fp, 'rU') as f:
obs = db.store_external_survey(f,
'Vioscreen',
separator=',',
survey_id_col='SubjectId',
trim='-160')
self.assertEqual(obs, 3)
barcodes = ['000029429', '000018046', '000023299', '000023300']
# Test without third party
obs, _ = db.pulldown(barcodes)
# Parse the metadata into a pandas dataframe to test some invariants
# This tests does not ensure that the columns have the exact value
# but at least ensure that the contents looks as expected
survey_df = pd.read_csv(StringIO(obs[1]),
delimiter='\t',
dtype=str,
encoding='utf-8')
survey_df.set_index('sample_name', inplace=True, drop=True)
# Make sure that the prohibited columns from EBI are not in the
# pulldown
self.assertEqual(
set(survey_df.columns).intersection(ebi_remove), set())
freq_accepted_vals = {
'Never', 'Rarely (a few times/month)',
'Regularly (3-5 times/week)', 'Occasionally (1-2 times/week)',
'Unspecified', 'Daily'
}
freq_cols = [
'ALCOHOL_FREQUENCY', 'PROBIOTIC_FREQUENCY',
'ONE_LITER_OF_WATER_A_DAY_FREQUENCY', 'POOL_FREQUENCY',
'FLOSSING_FREQUENCY', 'COSMETICS_FREQUENCY'
]
for col in freq_cols:
vals = set(survey_df[col])
self.assertTrue(all([x in freq_accepted_vals for x in vals]))
# This astype is making sure that the values in the BMI column are
# values that can be casted to float.
survey_df[survey_df.BMI != 'Unspecified'].BMI.astype(float)
body_product_values = set(survey_df.BODY_PRODUCT)
self.assertTrue(
all([
x.startswith('UBERON') or x == 'Unspecified'
for x in body_product_values
]))
survey = obs[1]
self.assertFalse('VIOSCREEN' in survey)
obs, _ = db.pulldown(barcodes, blanks=['BLANK.01'])
survey = obs[1]
self.assertFalse('VIOSCREEN' in survey)
self.assertTrue('BLANK.01' in survey)
# Test with third party
obs, _ = db.pulldown(barcodes, external=['Vioscreen'])
survey = obs[1]
self.assertTrue('VIOSCREEN' in survey)
obs, _ = db.pulldown(barcodes,
blanks=['BLANK.01'],
external=['Vioscreen'])
survey = obs[1]
self.assertTrue('VIOSCREEN' in survey)
self.assertTrue('BLANK.01' in survey)
def test_pulldown_third_party(self):
# Add survey answers
with open(self.ext_survey_fp, "rU") as f:
obs = db.store_external_survey(f, "Vioscreen", separator=",", survey_id_col="SubjectId", trim="-160")
self.assertEqual(obs, 3)
barcodes = ["000029429", "000018046", "000023299", "000023300"]
# Test without third party
obs, _ = db.pulldown(barcodes)
# Parse the metadata into a pandas dataframe to test some invariants
# This tests does not ensure that the columns have the exact value
# but at least ensure that the contents looks as expected
survey_df = pd.read_csv(StringIO(obs[1]), delimiter="\t", dtype=str, encoding="utf-8")
survey_df.set_index("sample_name", inplace=True, drop=True)
# Make sure that the prohibited columns from EBI are not in the
# pulldown
self.assertEqual(set(survey_df.columns).intersection(ebi_remove), set())
freq_accepted_vals = {
"Never",
"Rarely (a few times/month)",
"Regularly (3-5 times/week)",
"Occasionally (1-2 times/week)",
"Unspecified",
"Daily",
}
freq_cols = [
"ALCOHOL_FREQUENCY",
"PROBIOTIC_FREQUENCY",
"ONE_LITER_OF_WATER_A_DAY_FREQUENCY",
"POOL_FREQUENCY",
"FLOSSING_FREQUENCY",
"COSMETICS_FREQUENCY",
]
for col in freq_cols:
vals = set(survey_df[col])
self.assertTrue(all([x in freq_accepted_vals for x in vals]))
# This astype is making sure that the values in the BMI column are
# values that can be casted to float.
survey_df.BMI.astype(float)
body_product_values = set(survey_df.BODY_PRODUCT)
self.assertTrue(all([x.startswith("UBERON") or x == "Unspecified" for x in body_product_values]))
survey = obs[1]
self.assertFalse("VIOSCREEN" in survey)
obs, _ = db.pulldown(barcodes, blanks=["BLANK.01"])
survey = obs[1]
self.assertFalse("VIOSCREEN" in survey)
self.assertTrue("BLANK.01" in survey)
# Test with third party
obs, _ = db.pulldown(barcodes, external=["Vioscreen"])
survey = obs[1]
self.assertTrue("VIOSCREEN" in survey)
obs, _ = db.pulldown(barcodes, blanks=["BLANK.01"], external=["Vioscreen"])
survey = obs[1]
self.assertTrue("VIOSCREEN" in survey)
self.assertTrue("BLANK.01" in survey)
def test_pulldown_third_party(self):
# Add survey answers
with open(self.ext_survey_fp, 'rU') as f:
obs = db.store_external_survey(
f, 'Vioscreen', separator=',', survey_id_col='SubjectId',
trim='-160')
self.assertEqual(obs, 3)
barcodes = ['000029429', '000018046', '000023299', '000023300']
# Test without third party
obs, _ = db.pulldown(barcodes)
# Parse the metadata into a pandas dataframe to test some invariants
# This tests does not ensure that the columns have the exact value
# but at least ensure that the contents looks as expected
survey_df = pd.read_csv(
StringIO(obs[1]), delimiter='\t', dtype=str, encoding='utf-8')
survey_df.set_index('sample_name', inplace=True, drop=True)
# Make sure that the prohibited columns from EBI are not in the
# pulldown
self.assertEqual(set(survey_df.columns).intersection(ebi_remove),
set())
freq_accepted_vals = {
'Never', 'Rarely (a few times/month)',
'Regularly (3-5 times/week)', 'Occasionally (1-2 times/week)',
'Unspecified', 'Daily'}
freq_cols = ['ALCOHOL_FREQUENCY', 'PROBIOTIC_FREQUENCY',
'ONE_LITER_OF_WATER_A_DAY_FREQUENCY', 'POOL_FREQUENCY',
'FLOSSING_FREQUENCY', 'COSMETICS_FREQUENCY']
for col in freq_cols:
vals = set(survey_df[col])
self.assertTrue(all([x in freq_accepted_vals for x in vals]))
# This astype is making sure that the values in the BMI column are
# values that can be casted to float.
survey_df[survey_df.BMI != 'Unspecified'] .BMI.astype(float)
body_product_values = set(survey_df.BODY_PRODUCT)
self.assertTrue(all([x.startswith('UBERON') or x == 'Unspecified'
for x in body_product_values]))
survey = obs[1]
self.assertFalse('VIOSCREEN' in survey)
obs, _ = db.pulldown(barcodes, blanks=['BLANK.01'])
survey = obs[1]
self.assertFalse('VIOSCREEN' in survey)
self.assertTrue('BLANK.01' in survey)
# Test with third party
obs, _ = db.pulldown(barcodes, external=['Vioscreen'])
survey = obs[1]
self.assertTrue('VIOSCREEN' in survey)
obs, _ = db.pulldown(barcodes, blanks=['BLANK.01'],
external=['Vioscreen'])
survey = obs[1]
self.assertTrue('VIOSCREEN' in survey)
self.assertTrue('BLANK.01' in survey)
