def test_importConditions(self):
standard_files = []
standard_files.append(join(fixturesPath, 'trialTypes.xlsx'))
#standard_files.append(join(fixturesPath, 'trialTypes.xls')) # xls is depreciated
standard_files.append(join(fixturesPath, 'trialTypes.csv'))
standard_files.append(join(fixturesPath, 'trialTypes_eu.csv'))
standard_files.append(join(fixturesPath, 'trialTypes.tsv'))
# some extra formats (expected fails)
fileName_pkl = join(fixturesPath, 'trialTypes.pkl')
fileName_docx = join(fixturesPath, 'trialTypes.docx')
expected_cond = utils.OrderedDict([('text', 'red'), ('congruent', 1),
('corrAns', 1),
('letterColor', 'red'), ('n', 2),
('float', 1.1)])
# check import worked for standard file formats
for filename in standard_files:
conds = utils.importConditions(filename)
assert conds[0] == expected_cond, (
"Did not correctly import for '{}': "
"expected({}) != imported({})".format(filename, expected_cond,
conds[0]))
# test for None in filename with _assertValidVarNames
assert utils.importConditions(fileName=None) == []
assert utils.importConditions(fileName=None,
returnFieldNames=True) == ([], [])
# Test value error for non-existent file
with pytest.raises(ValueError) as errMsg:
utils.importConditions(fileName='raiseErrorfileName')
assert 'Conditions file not found: %s' % os.path.abspath(
'raiseErrorfileName') in str(errMsg.value)
conds = utils.importConditions(fileName_pkl)
assert conds[0] == expected_cond
# trialTypes.pkl saved in list of list format (see trialTypes.docx)
# test assertion for invalid file type
with pytest.raises(IOError) as errMsg:
utils.importConditions(fileName_docx)
assert ('Your conditions file should be an '
'xlsx, csv, dlm, tsv or pkl file') == str(errMsg.value)
# test random selection of conditions
all_conditions = utils.importConditions(standard_files[0])
assert len(all_conditions) == 6
num_selected_conditions = 1001
selected_conditions = utils.importConditions(
standard_files[0],
selection=(np.concatenate(
([0.9], np.random.random(num_selected_conditions - 1) *
len(all_conditions)))))
assert selected_conditions[0] == expected_cond
assert len(selected_conditions) == num_selected_conditions
def test_importConditions(self):
fileName_xlsx = os.path.join(fixturesPath, 'trialTypes.xlsx')
fileName_xls = os.path.join(fixturesPath, 'trialTypes.xls')
fileName_csv = os.path.join(fixturesPath, 'trialTypes.csv')
fileName_pkl = os.path.join(fixturesPath, 'trialTypes.pkl')
fileName_docx = os.path.join(fixturesPath, 'trialTypes.docx')
expected_cond = utils.OrderedDict(
[('text', 'red'),
('congruent', 1),
('corrAns', 1),
('letterColor', 'red'),
('n', 2),
('float', 1.1)])
conds = utils.importConditions(fileName_xlsx)
assert conds[0] == expected_cond
# test for None in filename with _assertValidVarNames
assert utils.importConditions(fileName=None) == []
assert utils.importConditions(fileName=None, returnFieldNames=True) == ([], [])
# Test value error for non-existent file
with pytest.raises(ValueError) as errMsg:
utils.importConditions(fileName='raiseErrorfileName')
assert 'Conditions file not found: %s' % os.path.abspath('raiseErrorfileName') in str(errMsg.value)
# Check file extensions in nested pandasToDictList()
conds = utils.importConditions(fileName_csv)
assert conds[0] == expected_cond
conds = utils.importConditions(fileName_xls)
assert conds[0] == expected_cond
if PY3:
conds = utils.importConditions(fileName_pkl)
assert conds[0] == expected_cond
else:
with pytest.raises((IOError)) as errMsg:
utils.importConditions(fileName_pkl)
assert ('Could not open %s as conditions' % fileName_pkl) == str(errMsg.value)
# trialTypes.pkl saved in list of list format (see trialTypes.docx)
# test assertion for invalid file type
with pytest.raises(IOError) as errMsg:
utils.importConditions(fileName_docx)
assert ('Your conditions file should be an ''xlsx, csv or pkl file') == str(errMsg.value)
def importItems(self, items):
"""Import items from csv or excel sheet and convert to list of dicts.
Will also accept a list of dicts.
Note, for csv and excel files, 'options' must contain comma separated values,
e.g., one, two, three. No parenthesis, or quotation marks required.
Parameters
----------
items : Excel or CSV file, list of dicts
Items used to populate the Form
Returns
-------
List of dicts
A list of dicts, where each list entry is a dict containing all fields for a single Form item
"""
def _checkSynonyms(items, fieldNames):
"""Checks for updated names for fields (i.e. synonyms)"""
replacedFields = set()
for field in _synonyms:
synonym = _synonyms[field]
for item in items:
if synonym in item:
# convert to new name
item[field] = item[synonym]
del item[synonym]
replacedFields.add(field)
for field in replacedFields:
fieldNames.append(field)
fieldNames.remove(_synonyms[field])
logging.warning("Form {} included field no longer used {}. "
"Replacing with new name '{}'".format(
self.name, _synonyms[field], field))
def _checkRequiredFields(fieldNames):
"""Checks for required headings (do this after checking synonyms)"""
for hdr in _knownFields:
# is it required and/or present?
if _knownFields[hdr] == _REQUIRED and hdr not in fieldNames:
raise ValueError("Missing header ({}) in Form ({}). "
"Headers found were: {}".format(
hdr, self.name, fieldNames))
def _checkTypes(types, itemText):
"""A nested function for testing the number of options given
Raises ValueError if n Options not > 1
"""
itemDiff = set([types]) - set(_knownRespTypes)
for incorrItemType in itemDiff:
if incorrItemType == _REQUIRED:
if self._itemsFile:
itemsFileStr = ("in items file '{}'".format(
self._itemsFile))
else:
itemsFileStr = ""
msg = ("Item {}{} is missing a required "
"value for its response type. Permitted types are "
"{}.".format(itemText, itemsFileStr,
_knownRespTypes))
if self.autoLog:
logging.error(msg)
raise ValueError(msg)
def _addDefaultItems(items):
"""
Adds default items when missing. Works in-place.
Parameters
----------
items : List of dicts
headers : List of column headers for each item
"""
def isPresent(d, field):
# check if the field is there and not empty on this row
return (field in d and d[field] not in [None, ''])
missingHeaders = []
defaultValues = _knownFields
for index, item in enumerate(items):
defaultValues['index'] = index
for header in defaultValues:
# if header is missing of val is None or ''
if not isPresent(item, header):
oldHeader = header.replace('item', 'question')
if isPresent(item, oldHeader):
item[header] = item[oldHeader]
logging.warning(
"{} is a deprecated heading for Forms. "
"Use {} instead".format(oldHeader, header))
continue
# Default to colour scheme if specified
if defaultValues[header] in ['fg', 'bg', 'em']:
item[header] = self.colorScheme[
defaultValues[header]]
else:
item[header] = defaultValues[header]
missingHeaders.append(header)
msg = "Using default values for the following headers: {}".format(
missingHeaders)
if self.autoLog:
logging.info(msg)
if self.autoLog:
logging.info("Importing items...")
if not isinstance(items, list):
# items is a conditions file
self._itemsFile = Path(items)
items, fieldNames = importConditions(items, returnFieldNames=True)
else: # we already have a list so lets find the fieldnames
fieldNames = set()
for item in items:
fieldNames = fieldNames.union(item)
fieldNames = list(fieldNames) # convert to list at the end
self._itemsFile = None
_checkSynonyms(items, fieldNames)
_checkRequiredFields(fieldNames)
# Add default values if entries missing
_addDefaultItems(items)
# Convert options to list of strings
for idx, item in enumerate(items):
if item['ticks']:
item['ticks'] = listFromString(item['ticks'])
if 'tickLabels' in item and item['tickLabels']:
item['tickLabels'] = listFromString(item['tickLabels'])
if 'options' in item and item['options']:
item['options'] = listFromString(item['options'])
# Check types
[_checkTypes(item['type'], item['itemText']) for item in items]
# Check N options > 1
# Randomise items if requested
if self.randomize:
shuffle(items)
return items
def test_import_blankColumns(self):
fileName_blanks = join(fixturesPath, 'trialsBlankCols.xlsx')
conds = utils.importConditions(fileName_blanks)
assert len(conds) == 6
assert len(list(conds[0].keys())) == 6
def importItems(self, items):
"""Import items from csv or excel sheet and convert to list of dicts.
Will also accept a list of dicts.
Note, for csv and excel files, 'options' must contain comma separated values,
e.g., one, two, three. No parenthesis, or quotation marks required.
Parameters
----------
items : Excel or CSV file, list of dicts
Items used to populate the Form
Returns
-------
List of dicts
A list of dicts, where each list entry is a dict containing all fields for a single Form item
"""
def _checkOptions(options):
"""A nested function for testing the number of options given
Raises ValueError if n Options not > 1
"""
if not len(options) > 1:
msg = "Provide at least two possible options for your item responses."
if self.autoLog:
psychopy.logging.error(msg)
raise ValueError(msg)
def _checkTypes(types):
"""A nested function for testing the number of options given
Raises ValueError if n Options not > 1
"""
allowedTypes = ['rating', 'slider', 'textbox', 'radio']
itemDiff = set([types]) - set(allowedTypes)
if len(itemDiff) > 0:
msg = (
"In Forms, {} is not allowed. You can only use type {}. "
"Please amend your item types in your item list").format(
itemDiff, allowedTypes)
if self.autoLog:
psychopy.logging.error(msg)
raise ValueError(msg)
def _checkHeaders(fields):
"""A nested function for testing the names of fields in any given set of items
Raises NameError if fields do not match required survey fields
"""
surveyFields = [
'index', 'responseWidth', 'layout', 'questionText', 'type',
'questionWidth', 'options'
]
if not set(surveyFields) == set(fields):
msg = "Use the following fields/column names for Forms...\n{}".format(
surveyFields)
if self.autoLog:
psychopy.logging.error(msg)
raise NameError(msg)
if self.autoLog:
psychopy.logging.info("Importing items...")
if not isinstance(items, list):
items, returnFieldNames = importConditions(items,
returnFieldNames=True)
# Check fieldnames are correct
_checkHeaders(returnFieldNames)
else:
for item in items:
_checkHeaders(item.keys())
# Convert options to list of strings
for idx, item in enumerate(items):
if isinstance(item['options'], str):
items[idx]['options'] = item['options'].split(',')
# Check types
[_checkTypes(item['type']) for item in items]
# Check N options > 1
[_checkOptions(item['options']) for item in items]
return self.randomizeItems(items)
def importItems(self, items):
"""Import items from csv or excel sheet and convert to list of dicts.
Will also accept a list of dicts.
Note, for csv and excel files, 'options' must contain comma separated values,
e.g., one, two, three. No parenthesis, or quotation marks required.
Parameters
----------
items : Excel or CSV file, list of dicts
Items used to populate the Form
Returns
-------
List of dicts
A list of dicts, where each list entry is a dict containing all fields for a single Form item
"""
def _checkOptions(options):
"""A nested function for testing the number of options given
Raises ValueError if n Options not > 1
"""
if not len(options) > 1:
msg = "Provide at least two possible options for your item responses."
if self.autoLog:
psychopy.logging.error(msg)
raise ValueError(msg)
def _checkTypes(types):
"""A nested function for testing the number of options given
Raises ValueError if n Options not > 1
"""
allowedTypes = ['rating', 'slider', 'textbox', 'radio']
itemDiff = set([types])-set(allowedTypes)
if len(itemDiff) > 0:
msg = ("In Forms, {} is not allowed. You can only use type {}. "
"Please amend your item types in your item list").format(itemDiff,
allowedTypes)
if self.autoLog:
psychopy.logging.error(msg)
raise ValueError(msg)
def _checkHeaders(fields):
"""A nested function for checking the names of fields in any given set of items
Returns
-------
missingHeaders : Set of missing headers, or None if no missing headers
"""
surveyFields = {'index', 'responseWidth', 'layout', 'questionText',
'type', 'questionWidth', 'options',
'questionColor', 'responseColor'}
fields = set(fields)
if not surveyFields == fields:
missingHeaders = surveyFields.difference(fields)
msg = ("Missing headers: {}. "
"Note, headers are case sensitive and must match: {}"
.format(missingHeaders, surveyFields))
if self.autoLog:
psychopy.logging.warning(msg)
return missingHeaders
def _addDefaultItems(items, missingHeaders):
"""
Adds default items when missing. Works in-place.
Parameters
----------
items : List of dicts
headers : List of column headers for each item
"""
if missingHeaders is None:
return
defaultValues = {'index': 0,
'responseWidth': .3,
'questionWidth': .7,
'layout': 'horiz',
'questionText': 'Default question',
'type': 'rating',
'options': 'Yes, No',
'questionColor': 'white',
'responseColor': 'white'}
msg = "Using default values for the following headers: {}".format(missingHeaders)
if self.autoLog:
psychopy.logging.warning(msg)
for index, item in enumerate(items):
defaultValues['index'] = index
for header in missingHeaders:
items[index][header] = defaultValues[header]
if self.autoLog:
psychopy.logging.info("Importing items...")
if not isinstance(items, list):
# items is a conditions file
items, returnFieldNames = importConditions(items, returnFieldNames=True)
# Check fieldnames are correct
missingHeaders = _checkHeaders(returnFieldNames)
# Add default values if headers are missing
_addDefaultItems(items, missingHeaders)
else:
# items is a list of dicts
for item in items:
# Check fieldnames are correct
missingHeaders = _checkHeaders(item.keys())
# Add default values if headers are missing
_addDefaultItems(items, missingHeaders)
# Convert options to list of strings
for idx, item in enumerate(items):
if PY3:
if isinstance(item['options'], str):
items[idx]['options'] = item['options'].split(',')
else: # Python2
if isinstance(item['options'], basestring):
items[idx]['options'] = item['options'].split(',')
# Check types
[_checkTypes(item['type']) for item in items]
# Check N options > 1
[_checkOptions(item['options']) for item in items]
# Randomise items if requested
self.randomizeItems(items)
return items
def test_importConditions(self):
fileName_xlsx = os.path.join(fixturesPath, 'trialTypes.xlsx')
fileName_xls = os.path.join(fixturesPath, 'trialTypes.xls')
fileName_csv = os.path.join(fixturesPath, 'trialTypes.csv')
fileName_pkl = os.path.join(fixturesPath, 'trialTypes.pkl')
fileName_docx = os.path.join(fixturesPath, 'trialTypes.docx')
expected_cond = utils.OrderedDict([('text', 'red'), ('congruent', 1),
('corrAns', 1),
('letterColor', 'red'), ('n', 2),
('float', 1.1)])
conds = utils.importConditions(fileName_xlsx)
assert conds[0] == expected_cond
# test for None in filename with _assertValidVarNames
assert utils.importConditions(fileName=None) == []
assert utils.importConditions(fileName=None,
returnFieldNames=True) == ([], [])
# Test value error for non-existent file
with pytest.raises(ValueError) as errMsg:
utils.importConditions(fileName='raiseErrorfileName')
assert 'Conditions file not found: %s' % os.path.abspath(
'raiseErrorfileName') in str(errMsg.value)
# Check file extensions in nested pandasToDictList()
conds = utils.importConditions(fileName_csv)
assert conds[0] == expected_cond
conds = utils.importConditions(fileName_xls)
assert conds[0] == expected_cond
if PY3:
conds = utils.importConditions(fileName_pkl)
assert conds[0] == expected_cond
else:
with pytest.raises((IOError)) as errMsg:
utils.importConditions(fileName_pkl)
assert ('Could not open %s as conditions' % fileName_pkl) == str(
errMsg.value)
# trialTypes.pkl saved in list of list format (see trialTypes.docx)
# test assertion for invalid file type
with pytest.raises(IOError) as errMsg:
utils.importConditions(fileName_docx)
assert ('Your conditions file should be an '
'xlsx, csv or pkl file') == str(errMsg.value)
def importItems(self, items):
"""Import items from csv or excel sheet and convert to list of dicts.
Will also accept a list of dicts.
Note, for csv and excel files, 'options' must contain comma separated values,
e.g., one, two, three. No parenthesis, or quotation marks required.
Parameters
----------
items : Excel or CSV file, list of dicts
Items used to populate the Form
Returns
-------
List of dicts
A list of dicts, where each list entry is a dict containing all fields for a single Form item
"""
def _checkOptions(options):
"""A nested function for testing the number of options given
Raises ValueError if n Options not > 1
"""
if not len(options) > 1:
msg = "Provide at least two possible options for your item responses."
if self.autoLog:
psychopy.logging.error(msg)
raise ValueError(msg)
def _checkTypes(types):
"""A nested function for testing the number of options given
Raises ValueError if n Options not > 1
"""
allowedTypes = ['rating', 'slider', 'textbox', 'radio']
itemDiff = set([types])-set(allowedTypes)
if len(itemDiff) > 0:
msg = ("In Forms, {} is not allowed. You can only use type {}. "
"Please amend your item types in your item list").format(itemDiff,
allowedTypes)
if self.autoLog:
psychopy.logging.error(msg)
raise ValueError(msg)
def _checkHeaders(fields):
"""A nested function for testing the names of fields in any given set of items
Raises NameError if fields do not match required survey fields
"""
surveyFields = ['index', 'responseWidth', 'layout', 'questionText', 'type', 'questionWidth', 'options']
if not set(surveyFields) == set(fields):
msg = "Use the following fields/column names for Forms...\n{}".format(surveyFields)
if self.autoLog:
psychopy.logging.error(msg)
raise NameError(msg)
if self.autoLog:
psychopy.logging.info("Importing items...")
if not isinstance(items, list):
items, returnFieldNames = importConditions(items, returnFieldNames=True)
# Check fieldnames are correct
_checkHeaders(returnFieldNames)
else:
for item in items:
_checkHeaders(item.keys())
# Convert options to list of strings
for idx, item in enumerate(items):
if PY3:
if isinstance(item['options'], str):
items[idx]['options'] = item['options'].split(',')
else: # Python2
if isinstance(item['options'], basestring):
items[idx]['options'] = item['options'].split(',')
# Check types
[_checkTypes(item['type']) for item in items]
# Check N options > 1
[_checkOptions(item['options']) for item in items]
return self.randomizeItems(items)
