def test_importConditions(self): standard_files = [] standard_files.append(join(fixturesPath, 'trialTypes.xlsx')) #standard_files.append(join(fixturesPath, 'trialTypes.xls')) # xls is depreciated standard_files.append(join(fixturesPath, 'trialTypes.csv')) standard_files.append(join(fixturesPath, 'trialTypes_eu.csv')) standard_files.append(join(fixturesPath, 'trialTypes.tsv')) # some extra formats (expected fails) fileName_pkl = join(fixturesPath, 'trialTypes.pkl') fileName_docx = join(fixturesPath, 'trialTypes.docx') expected_cond = utils.OrderedDict([('text', 'red'), ('congruent', 1), ('corrAns', 1), ('letterColor', 'red'), ('n', 2), ('float', 1.1)]) # check import worked for standard file formats for filename in standard_files: conds = utils.importConditions(filename) assert conds[0] == expected_cond, ( "Did not correctly import for '{}': " "expected({}) != imported({})".format(filename, expected_cond, conds[0])) # test for None in filename with _assertValidVarNames assert utils.importConditions(fileName=None) == [] assert utils.importConditions(fileName=None, returnFieldNames=True) == ([], []) # Test value error for non-existent file with pytest.raises(ValueError) as errMsg: utils.importConditions(fileName='raiseErrorfileName') assert 'Conditions file not found: %s' % os.path.abspath( 'raiseErrorfileName') in str(errMsg.value) conds = utils.importConditions(fileName_pkl) assert conds[0] == expected_cond # trialTypes.pkl saved in list of list format (see trialTypes.docx) # test assertion for invalid file type with pytest.raises(IOError) as errMsg: utils.importConditions(fileName_docx) assert ('Your conditions file should be an ' 'xlsx, csv, dlm, tsv or pkl file') == str(errMsg.value) # test random selection of conditions all_conditions = utils.importConditions(standard_files[0]) assert len(all_conditions) == 6 num_selected_conditions = 1001 selected_conditions = utils.importConditions( standard_files[0], selection=(np.concatenate( ([0.9], np.random.random(num_selected_conditions - 1) * len(all_conditions))))) assert selected_conditions[0] == expected_cond assert len(selected_conditions) == num_selected_conditions
def test_importConditions(self): fileName_xlsx = os.path.join(fixturesPath, 'trialTypes.xlsx') fileName_xls = os.path.join(fixturesPath, 'trialTypes.xls') fileName_csv = os.path.join(fixturesPath, 'trialTypes.csv') fileName_pkl = os.path.join(fixturesPath, 'trialTypes.pkl') fileName_docx = os.path.join(fixturesPath, 'trialTypes.docx') expected_cond = utils.OrderedDict( [('text', 'red'), ('congruent', 1), ('corrAns', 1), ('letterColor', 'red'), ('n', 2), ('float', 1.1)]) conds = utils.importConditions(fileName_xlsx) assert conds[0] == expected_cond # test for None in filename with _assertValidVarNames assert utils.importConditions(fileName=None) == [] assert utils.importConditions(fileName=None, returnFieldNames=True) == ([], []) # Test value error for non-existent file with pytest.raises(ValueError) as errMsg: utils.importConditions(fileName='raiseErrorfileName') assert 'Conditions file not found: %s' % os.path.abspath('raiseErrorfileName') in str(errMsg.value) # Check file extensions in nested pandasToDictList() conds = utils.importConditions(fileName_csv) assert conds[0] == expected_cond conds = utils.importConditions(fileName_xls) assert conds[0] == expected_cond if PY3: conds = utils.importConditions(fileName_pkl) assert conds[0] == expected_cond else: with pytest.raises((IOError)) as errMsg: utils.importConditions(fileName_pkl) assert ('Could not open %s as conditions' % fileName_pkl) == str(errMsg.value) # trialTypes.pkl saved in list of list format (see trialTypes.docx) # test assertion for invalid file type with pytest.raises(IOError) as errMsg: utils.importConditions(fileName_docx) assert ('Your conditions file should be an ''xlsx, csv or pkl file') == str(errMsg.value)
def importItems(self, items): """Import items from csv or excel sheet and convert to list of dicts. Will also accept a list of dicts. Note, for csv and excel files, 'options' must contain comma separated values, e.g., one, two, three. No parenthesis, or quotation marks required. Parameters ---------- items : Excel or CSV file, list of dicts Items used to populate the Form Returns ------- List of dicts A list of dicts, where each list entry is a dict containing all fields for a single Form item """ def _checkSynonyms(items, fieldNames): """Checks for updated names for fields (i.e. synonyms)""" replacedFields = set() for field in _synonyms: synonym = _synonyms[field] for item in items: if synonym in item: # convert to new name item[field] = item[synonym] del item[synonym] replacedFields.add(field) for field in replacedFields: fieldNames.append(field) fieldNames.remove(_synonyms[field]) logging.warning("Form {} included field no longer used {}. " "Replacing with new name '{}'".format( self.name, _synonyms[field], field)) def _checkRequiredFields(fieldNames): """Checks for required headings (do this after checking synonyms)""" for hdr in _knownFields: # is it required and/or present? if _knownFields[hdr] == _REQUIRED and hdr not in fieldNames: raise ValueError("Missing header ({}) in Form ({}). " "Headers found were: {}".format( hdr, self.name, fieldNames)) def _checkTypes(types, itemText): """A nested function for testing the number of options given Raises ValueError if n Options not > 1 """ itemDiff = set([types]) - set(_knownRespTypes) for incorrItemType in itemDiff: if incorrItemType == _REQUIRED: if self._itemsFile: itemsFileStr = ("in items file '{}'".format( self._itemsFile)) else: itemsFileStr = "" msg = ("Item {}{} is missing a required " "value for its response type. Permitted types are " "{}.".format(itemText, itemsFileStr, _knownRespTypes)) if self.autoLog: logging.error(msg) raise ValueError(msg) def _addDefaultItems(items): """ Adds default items when missing. Works in-place. Parameters ---------- items : List of dicts headers : List of column headers for each item """ def isPresent(d, field): # check if the field is there and not empty on this row return (field in d and d[field] not in [None, '']) missingHeaders = [] defaultValues = _knownFields for index, item in enumerate(items): defaultValues['index'] = index for header in defaultValues: # if header is missing of val is None or '' if not isPresent(item, header): oldHeader = header.replace('item', 'question') if isPresent(item, oldHeader): item[header] = item[oldHeader] logging.warning( "{} is a deprecated heading for Forms. " "Use {} instead".format(oldHeader, header)) continue # Default to colour scheme if specified if defaultValues[header] in ['fg', 'bg', 'em']: item[header] = self.colorScheme[ defaultValues[header]] else: item[header] = defaultValues[header] missingHeaders.append(header) msg = "Using default values for the following headers: {}".format( missingHeaders) if self.autoLog: logging.info(msg) if self.autoLog: logging.info("Importing items...") if not isinstance(items, list): # items is a conditions file self._itemsFile = Path(items) items, fieldNames = importConditions(items, returnFieldNames=True) else: # we already have a list so lets find the fieldnames fieldNames = set() for item in items: fieldNames = fieldNames.union(item) fieldNames = list(fieldNames) # convert to list at the end self._itemsFile = None _checkSynonyms(items, fieldNames) _checkRequiredFields(fieldNames) # Add default values if entries missing _addDefaultItems(items) # Convert options to list of strings for idx, item in enumerate(items): if item['ticks']: item['ticks'] = listFromString(item['ticks']) if 'tickLabels' in item and item['tickLabels']: item['tickLabels'] = listFromString(item['tickLabels']) if 'options' in item and item['options']: item['options'] = listFromString(item['options']) # Check types [_checkTypes(item['type'], item['itemText']) for item in items] # Check N options > 1 # Randomise items if requested if self.randomize: shuffle(items) return items
def test_import_blankColumns(self): fileName_blanks = join(fixturesPath, 'trialsBlankCols.xlsx') conds = utils.importConditions(fileName_blanks) assert len(conds) == 6 assert len(list(conds[0].keys())) == 6
def importItems(self, items): """Import items from csv or excel sheet and convert to list of dicts. Will also accept a list of dicts. Note, for csv and excel files, 'options' must contain comma separated values, e.g., one, two, three. No parenthesis, or quotation marks required. Parameters ---------- items : Excel or CSV file, list of dicts Items used to populate the Form Returns ------- List of dicts A list of dicts, where each list entry is a dict containing all fields for a single Form item """ def _checkOptions(options): """A nested function for testing the number of options given Raises ValueError if n Options not > 1 """ if not len(options) > 1: msg = "Provide at least two possible options for your item responses." if self.autoLog: psychopy.logging.error(msg) raise ValueError(msg) def _checkTypes(types): """A nested function for testing the number of options given Raises ValueError if n Options not > 1 """ allowedTypes = ['rating', 'slider', 'textbox', 'radio'] itemDiff = set([types]) - set(allowedTypes) if len(itemDiff) > 0: msg = ( "In Forms, {} is not allowed. You can only use type {}. " "Please amend your item types in your item list").format( itemDiff, allowedTypes) if self.autoLog: psychopy.logging.error(msg) raise ValueError(msg) def _checkHeaders(fields): """A nested function for testing the names of fields in any given set of items Raises NameError if fields do not match required survey fields """ surveyFields = [ 'index', 'responseWidth', 'layout', 'questionText', 'type', 'questionWidth', 'options' ] if not set(surveyFields) == set(fields): msg = "Use the following fields/column names for Forms...\n{}".format( surveyFields) if self.autoLog: psychopy.logging.error(msg) raise NameError(msg) if self.autoLog: psychopy.logging.info("Importing items...") if not isinstance(items, list): items, returnFieldNames = importConditions(items, returnFieldNames=True) # Check fieldnames are correct _checkHeaders(returnFieldNames) else: for item in items: _checkHeaders(item.keys()) # Convert options to list of strings for idx, item in enumerate(items): if isinstance(item['options'], str): items[idx]['options'] = item['options'].split(',') # Check types [_checkTypes(item['type']) for item in items] # Check N options > 1 [_checkOptions(item['options']) for item in items] return self.randomizeItems(items)
def importItems(self, items): """Import items from csv or excel sheet and convert to list of dicts. Will also accept a list of dicts. Note, for csv and excel files, 'options' must contain comma separated values, e.g., one, two, three. No parenthesis, or quotation marks required. Parameters ---------- items : Excel or CSV file, list of dicts Items used to populate the Form Returns ------- List of dicts A list of dicts, where each list entry is a dict containing all fields for a single Form item """ def _checkOptions(options): """A nested function for testing the number of options given Raises ValueError if n Options not > 1 """ if not len(options) > 1: msg = "Provide at least two possible options for your item responses." if self.autoLog: psychopy.logging.error(msg) raise ValueError(msg) def _checkTypes(types): """A nested function for testing the number of options given Raises ValueError if n Options not > 1 """ allowedTypes = ['rating', 'slider', 'textbox', 'radio'] itemDiff = set([types])-set(allowedTypes) if len(itemDiff) > 0: msg = ("In Forms, {} is not allowed. You can only use type {}. " "Please amend your item types in your item list").format(itemDiff, allowedTypes) if self.autoLog: psychopy.logging.error(msg) raise ValueError(msg) def _checkHeaders(fields): """A nested function for checking the names of fields in any given set of items Returns ------- missingHeaders : Set of missing headers, or None if no missing headers """ surveyFields = {'index', 'responseWidth', 'layout', 'questionText', 'type', 'questionWidth', 'options', 'questionColor', 'responseColor'} fields = set(fields) if not surveyFields == fields: missingHeaders = surveyFields.difference(fields) msg = ("Missing headers: {}. " "Note, headers are case sensitive and must match: {}" .format(missingHeaders, surveyFields)) if self.autoLog: psychopy.logging.warning(msg) return missingHeaders def _addDefaultItems(items, missingHeaders): """ Adds default items when missing. Works in-place. Parameters ---------- items : List of dicts headers : List of column headers for each item """ if missingHeaders is None: return defaultValues = {'index': 0, 'responseWidth': .3, 'questionWidth': .7, 'layout': 'horiz', 'questionText': 'Default question', 'type': 'rating', 'options': 'Yes, No', 'questionColor': 'white', 'responseColor': 'white'} msg = "Using default values for the following headers: {}".format(missingHeaders) if self.autoLog: psychopy.logging.warning(msg) for index, item in enumerate(items): defaultValues['index'] = index for header in missingHeaders: items[index][header] = defaultValues[header] if self.autoLog: psychopy.logging.info("Importing items...") if not isinstance(items, list): # items is a conditions file items, returnFieldNames = importConditions(items, returnFieldNames=True) # Check fieldnames are correct missingHeaders = _checkHeaders(returnFieldNames) # Add default values if headers are missing _addDefaultItems(items, missingHeaders) else: # items is a list of dicts for item in items: # Check fieldnames are correct missingHeaders = _checkHeaders(item.keys()) # Add default values if headers are missing _addDefaultItems(items, missingHeaders) # Convert options to list of strings for idx, item in enumerate(items): if PY3: if isinstance(item['options'], str): items[idx]['options'] = item['options'].split(',') else: # Python2 if isinstance(item['options'], basestring): items[idx]['options'] = item['options'].split(',') # Check types [_checkTypes(item['type']) for item in items] # Check N options > 1 [_checkOptions(item['options']) for item in items] # Randomise items if requested self.randomizeItems(items) return items
def test_importConditions(self): fileName_xlsx = os.path.join(fixturesPath, 'trialTypes.xlsx') fileName_xls = os.path.join(fixturesPath, 'trialTypes.xls') fileName_csv = os.path.join(fixturesPath, 'trialTypes.csv') fileName_pkl = os.path.join(fixturesPath, 'trialTypes.pkl') fileName_docx = os.path.join(fixturesPath, 'trialTypes.docx') expected_cond = utils.OrderedDict([('text', 'red'), ('congruent', 1), ('corrAns', 1), ('letterColor', 'red'), ('n', 2), ('float', 1.1)]) conds = utils.importConditions(fileName_xlsx) assert conds[0] == expected_cond # test for None in filename with _assertValidVarNames assert utils.importConditions(fileName=None) == [] assert utils.importConditions(fileName=None, returnFieldNames=True) == ([], []) # Test value error for non-existent file with pytest.raises(ValueError) as errMsg: utils.importConditions(fileName='raiseErrorfileName') assert 'Conditions file not found: %s' % os.path.abspath( 'raiseErrorfileName') in str(errMsg.value) # Check file extensions in nested pandasToDictList() conds = utils.importConditions(fileName_csv) assert conds[0] == expected_cond conds = utils.importConditions(fileName_xls) assert conds[0] == expected_cond if PY3: conds = utils.importConditions(fileName_pkl) assert conds[0] == expected_cond else: with pytest.raises((IOError)) as errMsg: utils.importConditions(fileName_pkl) assert ('Could not open %s as conditions' % fileName_pkl) == str( errMsg.value) # trialTypes.pkl saved in list of list format (see trialTypes.docx) # test assertion for invalid file type with pytest.raises(IOError) as errMsg: utils.importConditions(fileName_docx) assert ('Your conditions file should be an ' 'xlsx, csv or pkl file') == str(errMsg.value)
def importItems(self, items): """Import items from csv or excel sheet and convert to list of dicts. Will also accept a list of dicts. Note, for csv and excel files, 'options' must contain comma separated values, e.g., one, two, three. No parenthesis, or quotation marks required. Parameters ---------- items : Excel or CSV file, list of dicts Items used to populate the Form Returns ------- List of dicts A list of dicts, where each list entry is a dict containing all fields for a single Form item """ def _checkOptions(options): """A nested function for testing the number of options given Raises ValueError if n Options not > 1 """ if not len(options) > 1: msg = "Provide at least two possible options for your item responses." if self.autoLog: psychopy.logging.error(msg) raise ValueError(msg) def _checkTypes(types): """A nested function for testing the number of options given Raises ValueError if n Options not > 1 """ allowedTypes = ['rating', 'slider', 'textbox', 'radio'] itemDiff = set([types])-set(allowedTypes) if len(itemDiff) > 0: msg = ("In Forms, {} is not allowed. You can only use type {}. " "Please amend your item types in your item list").format(itemDiff, allowedTypes) if self.autoLog: psychopy.logging.error(msg) raise ValueError(msg) def _checkHeaders(fields): """A nested function for testing the names of fields in any given set of items Raises NameError if fields do not match required survey fields """ surveyFields = ['index', 'responseWidth', 'layout', 'questionText', 'type', 'questionWidth', 'options'] if not set(surveyFields) == set(fields): msg = "Use the following fields/column names for Forms...\n{}".format(surveyFields) if self.autoLog: psychopy.logging.error(msg) raise NameError(msg) if self.autoLog: psychopy.logging.info("Importing items...") if not isinstance(items, list): items, returnFieldNames = importConditions(items, returnFieldNames=True) # Check fieldnames are correct _checkHeaders(returnFieldNames) else: for item in items: _checkHeaders(item.keys()) # Convert options to list of strings for idx, item in enumerate(items): if PY3: if isinstance(item['options'], str): items[idx]['options'] = item['options'].split(',') else: # Python2 if isinstance(item['options'], basestring): items[idx]['options'] = item['options'].split(',') # Check types [_checkTypes(item['type']) for item in items] # Check N options > 1 [_checkOptions(item['options']) for item in items] return self.randomizeItems(items)