def _clean_validate_template(cls, md_template, study_id,
current_columns=None):
"""Takes care of all validation and cleaning of metadata templates
Parameters
----------
md_template : DataFrame
The metadata template file contents indexed by sample ids
study_id : int
The study to which the metadata template belongs to.
current_columns : iterable of str, optional
The current list of metadata columns
Returns
-------
md_template : DataFrame
Cleaned copy of the input md_template
Raises
------
QiitaDBColumnError
If the sample names in md_template contains invalid names
QiitaDBWarning
If there are missing columns required for some functionality
"""
cls._check_subclass()
invalid_ids = qdb.metadata_template.util.get_invalid_sample_names(
md_template.index)
if invalid_ids:
raise qdb.exceptions.QiitaDBColumnError(
"The following sample names in the template contain invalid "
"characters (only alphanumeric characters or periods are "
"allowed): %s." % ", ".join(invalid_ids))
if len(set(md_template.index)) != len(md_template.index):
raise qdb.exceptions.QiitaDBDuplicateSamplesError(
find_duplicates(md_template.index))
# We are going to modify the md_template. We create a copy so
# we don't modify the user one
md_template = md_template.copy(deep=True)
# In the database, all the column headers are lowercase
md_template.columns = [c.lower() for c in md_template.columns]
# validating pgsql reserved words not to be column headers
current_headers = set(md_template.columns.values)
qdb.metadata_template.util.validate_invalid_column_names(
current_headers)
# Prefix the sample names with the study_id
qdb.metadata_template.util.prefix_sample_names_with_id(md_template,
study_id)
# Check that we don't have duplicate columns
if len(set(md_template.columns)) != len(md_template.columns):
raise qdb.exceptions.QiitaDBDuplicateHeaderError(
find_duplicates(md_template.columns))
return md_template
def test_empty_input(self):
def empty_gen():
raise StopIteration()
yield
for empty in [], (), '', set(), {}, empty_gen():
self.assertEqual(find_duplicates(empty), set())
def _build_mapping_file(self, samples):
"""Builds the combined mapping file for all samples
Code modified slightly from qiime.util.MetadataMap.__add__"""
conn_handler = SQLConnectionHandler()
all_sample_ids = set()
sql = """SELECT filepath_id, filepath
FROM qiita.filepath
JOIN qiita.prep_template_filepath USING (filepath_id)
JOIN qiita.prep_template_preprocessed_data
USING (prep_template_id)
JOIN qiita.preprocessed_processed_data
USING (preprocessed_data_id)
JOIN qiita.filepath_type USING (filepath_type_id)
WHERE processed_data_id = %s
AND filepath_type = 'qiime_map'
ORDER BY filepath_id DESC"""
_id, fp = get_mountpoint('templates')[0]
to_concat = []
for pid, samples in viewitems(samples):
if len(samples) != len(set(samples)):
duplicates = find_duplicates(samples)
raise QiitaDBError("Duplicate sample ids found: %s"
% ', '.join(duplicates))
# Get the QIIME mapping file
qiime_map_fp = conn_handler.execute_fetchall(sql, (pid,))[0][1]
# Parse the mapping file
qiime_map = pd.read_csv(
join(fp, qiime_map_fp), sep='\t', keep_default_na=False,
na_values=['unknown'], index_col=False,
converters=defaultdict(lambda: str))
qiime_map.set_index('#SampleID', inplace=True, drop=True)
qiime_map = qiime_map.loc[samples]
duplicates = all_sample_ids.intersection(qiime_map.index)
if duplicates or len(samples) != len(set(samples)):
# Duplicate samples so raise error
raise QiitaDBError("Duplicate sample ids found: %s"
% ', '.join(duplicates))
all_sample_ids.update(qiime_map.index)
to_concat.append(qiime_map)
merged_map = pd.concat(to_concat)
cols = merged_map.columns.values.tolist()
cols.remove('BarcodeSequence')
cols.remove('LinkerPrimerSequence')
cols.remove('Description')
new_cols = ['BarcodeSequence', 'LinkerPrimerSequence']
new_cols.extend(cols)
new_cols.append('Description')
merged_map = merged_map[new_cols]
# Save the mapping file
_, base_fp = get_mountpoint(self._table)[0]
mapping_fp = join(base_fp, "%d_analysis_mapping.txt" % self._id)
merged_map.to_csv(mapping_fp, index_label='#SampleID',
na_rep='unknown', sep='\t')
self._add_file("%d_analysis_mapping.txt" % self._id, "plain_text")
def test_empty_input(self):
def empty_gen():
raise StopIteration()
yield
for empty in [], (), '', set(), {}, empty_gen():
self.assertEqual(find_duplicates(empty), set())
def test_empty_input(self):
def empty_gen():
return
yield
for empty in [], (), '', set(), {}, empty_gen():
self.assertEqual(find_duplicates(empty), set())
def test_empty_input(self):
def empty_gen():
return
yield
for empty in [], (), '', set(), {}, empty_gen():
self.assertEqual(find_duplicates(empty), set())
def _validate_ids(self, data, ids):
"""Validate the IDs.
Checks that IDs are unique and that the
number of IDs matches the number of rows/cols in the data array.
Subclasses can override this method to perform different/more specific
validation.
Notes
-----
Accepts arguments instead of inspecting instance attributes to avoid
creating an invalid dissimilarity matrix before raising an error.
Otherwise, the invalid dissimilarity matrix could be used after the
exception is caught and handled.
"""
duplicates = find_duplicates(ids)
if duplicates:
formatted_duplicates = ', '.join(repr(e) for e in duplicates)
raise DissimilarityMatrixError("IDs must be unique. Found the "
"following duplicate IDs: %s" %
formatted_duplicates)
if 0 == len(ids):
raise DissimilarityMatrixError("IDs must be at least 1 in "
"size.")
if len(ids) != data.shape[0]:
raise DissimilarityMatrixError("The number of IDs (%d) must match "
"the number of rows/columns in the "
"data (%d)." %
(len(ids), data.shape[0]))
def _validate(self, data, ids):
"""Validate the data array and IDs.
Checks that the data is at least 1x1 in size, 2D, square, hollow, and
contains only floats. Also checks that IDs are unique and that the
number of IDs matches the number of rows/cols in the data array.
Subclasses can override this method to perform different/more specific
validation (e.g., see `DistanceMatrix`).
Notes
-----
Accepts arguments instead of inspecting instance attributes to avoid
creating an invalid dissimilarity matrix before raising an error.
Otherwise, the invalid dissimilarity matrix could be used after the
exception is caught and handled.
"""
if 0 in data.shape:
raise DissimilarityMatrixError("Data must be at least 1x1 in "
"size.")
if len(data.shape) != 2:
raise DissimilarityMatrixError("Data must have exactly two "
"dimensions.")
if data.shape[0] != data.shape[1]:
raise DissimilarityMatrixError("Data must be square (i.e., have "
"the same number of rows and "
"columns).")
if data.dtype != np.double:
raise DissimilarityMatrixError("Data must contain only floating "
"point values.")
if np.trace(data) != 0:
raise DissimilarityMatrixError("Data must be hollow (i.e., the "
"diagonal can only contain zeros).")
duplicates = find_duplicates(ids)
if duplicates:
formatted_duplicates = ', '.join(repr(e) for e in duplicates)
raise DissimilarityMatrixError("IDs must be unique. Found the "
"following duplicate IDs: %s" %
formatted_duplicates)
if len(ids) != data.shape[0]:
raise DissimilarityMatrixError("The number of IDs (%d) must match "
"the number of rows/columns in the "
"data (%d)." %
(len(ids), data.shape[0]))
def _validate(self, data, ids):
"""Validate the data array and IDs.
Checks that the data is at least 1x1 in size, 2D, square, hollow, and
contains only floats. Also checks that IDs are unique and that the
number of IDs matches the number of rows/cols in the data array.
Subclasses can override this method to perform different/more specific
validation (e.g., see `DistanceMatrix`).
Notes
-----
Accepts arguments instead of inspecting instance attributes to avoid
creating an invalid dissimilarity matrix before raising an error.
Otherwise, the invalid dissimilarity matrix could be used after the
exception is caught and handled.
"""
if 0 in data.shape:
raise DissimilarityMatrixError("Data must be at least 1x1 in "
"size.")
if len(data.shape) != 2:
raise DissimilarityMatrixError("Data must have exactly two "
"dimensions.")
if data.shape[0] != data.shape[1]:
raise DissimilarityMatrixError("Data must be square (i.e., have "
"the same number of rows and "
"columns).")
if data.dtype != np.double:
raise DissimilarityMatrixError("Data must contain only floating "
"point values.")
if np.trace(data) != 0:
raise DissimilarityMatrixError("Data must be hollow (i.e., the "
"diagonal can only contain zeros).")
duplicates = find_duplicates(ids)
if duplicates:
formatted_duplicates = ', '.join(repr(e) for e in duplicates)
raise DissimilarityMatrixError("IDs must be unique. Found the "
"following duplicate IDs: %s" %
formatted_duplicates)
if len(ids) != data.shape[0]:
raise DissimilarityMatrixError("The number of IDs (%d) must match "
"the number of rows/columns in the "
"data (%d)." %
(len(ids), data.shape[0]))
def _check_duplicated_columns(prep_cols, sample_cols):
r"""Check for duplicated colums in the prep_cols and sample_cols
Parameters
----------
prep_cols : list of str
Column names in the prep info file
sample_cols : list of str
Column names in the sample info file
Raises
------
QiitaDBColumnError
If there are duplicated columns names in the sample and the prep
"""
prep_cols.extend(sample_cols)
dups = find_duplicates(prep_cols)
if dups:
raise qdb.exceptions.QiitaDBColumnError(
'Duplicated column names in the sample and prep info '
'files: %s. You need to delete that duplicated field' %
','.join(dups))
def _check_duplicated_columns(prep_cols, sample_cols):
r"""Check for duplicated colums in the prep_cols and sample_cols
Parameters
----------
prep_cols : list of str
Column names in the prep info file
sample_cols : list of str
Column names in the sample info file
Raises
------
QiitaDBColumnError
If there are duplicated columns names in the sample and the prep
"""
prep_cols.extend(sample_cols)
dups = find_duplicates(prep_cols)
if dups:
raise qdb.exceptions.QiitaDBColumnError(
'Duplicated column names in the sample and prep info '
'files: %s. You need to delete that duplicated field' %
','.join(dups))
def test_all_duplicates(self):
self.assertEqual(
find_duplicates(('a', 'bc', 'bc', 'def', 'a', 'def', 'def')),
set(['a', 'bc', 'def']))
def test_many_duplicates(self):
self.assertEqual(find_duplicates(['a', 'bc', 'bc', 'def', 'a']),
set(['a', 'bc']))
def test_no_duplicates(self):
self.assertEqual(find_duplicates(['a', 'bc', 'def', 'A']), set())
def reindex(self, key=None, keys=None):
"""Reassign keys to sequences in the MSA.
Parameters
----------
key : callable or metadata key, optional
If provided, defines a unique, hashable key for each sequence in
the MSA. Can either be a callable accepting a single argument (each
sequence) or a key into each sequence's ``metadata`` attribute.
keys : iterable, optional
An iterable of the same length as the number of sequences in the
MSA. `keys` must contain unique, hashable elements. Each element
will be used as the respective key for the sequences in the MSA.
Raises
------
ValueError
If `key` and `keys` are both provided.
ValueError
If `keys` is not the same length as the number of sequences in the
MSA.
UniqueError
If keys are not unique.
See Also
--------
keys
has_keys
Notes
-----
If `key` or `keys` are not provided, keys will not be set and certain
operations requiring keys will raise an ``OperationError``.
Examples
--------
Create a ``TabularMSA`` object without keys:
>>> from skbio import DNA, TabularMSA
>>> seqs = [DNA('ACG', metadata={'id': 'a'}),
... DNA('AC-', metadata={'id': 'b'})]
>>> msa = TabularMSA(seqs)
>>> msa.has_keys()
False
Set keys on the MSA, using each sequence's ID:
>>> msa.reindex(key='id')
>>> msa.has_keys()
True
>>> msa.keys
array(['a', 'b'], dtype=object)
Remove keys from the MSA:
>>> msa.reindex()
>>> msa.has_keys()
False
Alternatively, an iterable of keys may be passed via `keys`:
>>> msa.reindex(keys=['a', 'b'])
>>> msa.keys
array(['a', 'b'], dtype=object)
"""
if key is not None and keys is not None:
raise ValueError(
"Cannot use both `key` and `keys` at the same time.")
keys_ = None
if key is not None:
keys_ = [resolve_key(seq, key) for seq in self._seqs]
elif keys is not None:
keys = list(keys)
if len(keys) != len(self):
raise ValueError(
"Number of elements in `keys` must match number of "
"sequences: %d != %d" % (len(keys), len(self)))
keys_ = keys
if keys_ is not None:
# Hashability of keys is implicitly checked here.
duplicates = find_duplicates(keys_)
if duplicates:
raise UniqueError(
"Keys must be unique. Duplicate keys: %r" % duplicates)
# Create an immutable ndarray to ensure key invariants are
# preserved. Use object dtype to preserve original key types. This
# is important, for example, because np.array(['a', 42]) will
# upcast to ['a', '42'].
keys_ = np.array(keys_, dtype=object, copy=True)
keys_.flags.writeable = False
self._keys = keys_
def reindex(self, key=None, keys=None):
"""Reassign keys to sequences in the MSA.
Parameters
----------
key : callable or metadata key, optional
If provided, defines a unique, hashable key for each sequence in
the MSA. Can either be a callable accepting a single argument (each
sequence) or a key into each sequence's ``metadata`` attribute.
keys : iterable, optional
An iterable of the same length as the number of sequences in the
MSA. `keys` must contain unique, hashable elements. Each element
will be used as the respective key for the sequences in the MSA.
Raises
------
ValueError
If `key` and `keys` are both provided.
ValueError
If `keys` is not the same length as the number of sequences in the
MSA.
UniqueError
If keys are not unique.
See Also
--------
keys
has_keys
Notes
-----
If `key` or `keys` are not provided, keys will not be set and certain
operations requiring keys will raise an ``OperationError``.
Examples
--------
Create a ``TabularMSA`` object without keys:
>>> from skbio import DNA, TabularMSA
>>> seqs = [DNA('ACG', metadata={'id': 'a'}),
... DNA('AC-', metadata={'id': 'b'})]
>>> msa = TabularMSA(seqs)
>>> msa.has_keys()
False
Set keys on the MSA, using each sequence's ID:
>>> msa.reindex(key='id')
>>> msa.has_keys()
True
>>> msa.keys
array(['a', 'b'], dtype=object)
Remove keys from the MSA:
>>> msa.reindex()
>>> msa.has_keys()
False
Alternatively, an iterable of keys may be passed via `keys`:
>>> msa.reindex(keys=['a', 'b'])
>>> msa.keys
array(['a', 'b'], dtype=object)
"""
if key is not None and keys is not None:
raise ValueError(
"Cannot use both `key` and `keys` at the same time.")
keys_ = None
if key is not None:
keys_ = [resolve_key(seq, key) for seq in self._seqs]
elif keys is not None:
keys = list(keys)
if len(keys) != len(self):
raise ValueError(
"Number of elements in `keys` must match number of "
"sequences: %d != %d" % (len(keys), len(self)))
keys_ = keys
if keys_ is not None:
# Hashability of keys is implicitly checked here.
duplicates = find_duplicates(keys_)
if duplicates:
raise UniqueError("Keys must be unique. Duplicate keys: %r" %
duplicates)
# Create an immutable ndarray to ensure key invariants are
# preserved. Use object dtype to preserve original key types. This
# is important, for example, because np.array(['a', 42]) will
# upcast to ['a', '42'].
keys_ = np.array(keys_, dtype=object, copy=True)
keys_.flags.writeable = False
self._keys = keys_
def test_all_duplicates(self):
self.assertEqual(
find_duplicates(('a', 'bc', 'bc', 'def', 'a', 'def', 'def')),
set(['a', 'bc', 'def']))
def test_no_duplicates(self):
self.assertEqual(find_duplicates(['a', 'bc', 'def', 'A']), set())
def load_template_to_dataframe(fn, strip_whitespace=True, index='sample_name'):
"""Load a sample/prep template or a QIIME mapping file into a data frame
Parameters
----------
fn : str or file-like object
filename of the template to load, or an already open template file
strip_whitespace : bool, optional
Defaults to True. Whether or not to strip whitespace from values in the
input file
index : str, optional
Defaults to 'sample_name'. The index to use in the loaded information
Returns
-------
DataFrame
Pandas dataframe with the loaded information
Raises
------
ValueError
Empty file passed
QiitaDBColumnError
If the sample_name column is not present in the template.
If there's a value in one of the reserved columns that cannot be cast
to the needed type.
QiitaDBWarning
When columns are dropped because they have no content for any sample.
QiitaDBError
When non UTF-8 characters are found in the file.
QiitaDBDuplicateHeaderError
If duplicate columns are present in the template
Notes
-----
The index attribute of the DataFrame will be forced to be 'sample_name'
and will be cast to a string. Additionally rows that start with a '\t'
character will be ignored and columns that are empty will be removed. Empty
sample names will be removed from the DataFrame.
The following table describes the data type per column that will be
enforced in `fn`. Column names are case-insensitive but will be lowercased
on addition to the database.
+-----------------------+--------------+
| Column Name | Python Type |
+=======================+==============+
| sample_name | str |
+-----------------------+--------------+
| #SampleID | str |
+-----------------------+--------------+
| physical_location | str |
+-----------------------+--------------+
| has_physical_specimen | bool |
+-----------------------+--------------+
| has_extracted_data | bool |
+-----------------------+--------------+
| sample_type | str |
+-----------------------+--------------+
| host_subject_id | str |
+-----------------------+--------------+
| description | str |
+-----------------------+--------------+
| latitude | float |
+-----------------------+--------------+
| longitude | float |
+-----------------------+--------------+
"""
# Load in file lines
holdfile = None
with open_file(fn, mode='U') as f:
holdfile = f.readlines()
if not holdfile:
raise ValueError('Empty file passed!')
# Strip all values in the cells in the input file, if requested
if strip_whitespace:
for pos, line in enumerate(holdfile):
holdfile[pos] = '\t'.join(d.strip(" \r\x0b\x0c")
for d in line.split('\t'))
# get and clean the controlled columns
cols = holdfile[0].split('\t')
controlled_cols = {'sample_name'}
controlled_cols.update(CONTROLLED_COLS)
holdfile[0] = '\t'.join(c.lower() if c.lower() in controlled_cols else c
for c in cols)
if index == "#SampleID":
# We're going to parse a QIIME mapping file. We are going to first
# parse it with the QIIME function so we can remove the comments
# easily and make sure that QIIME will accept this as a mapping file
data, headers, comments = _parse_mapping_file(holdfile)
holdfile = ["%s\n" % '\t'.join(d) for d in data]
holdfile.insert(0, "%s\n" % '\t'.join(headers))
# The QIIME parser fixes the index and removes the #
index = 'SampleID'
# index_col:
# is set as False, otherwise it is cast as a float and we want a string
# keep_default:
# is set as False, to avoid inferring empty/NA values with the defaults
# that Pandas has.
# na_values:
# the values that should be considered as empty
# true_values:
# the values that should be considered "True" for boolean columns
# false_values:
# the values that should be considered "False" for boolean columns
# converters:
# ensure that sample names are not converted into any other types but
# strings and remove any trailing spaces. Don't let pandas try to guess
# the dtype of the other columns, force them to be a str.
# comment:
# using the tab character as "comment" we remove rows that are
# constituted only by delimiters i. e. empty rows.
try:
template = pd.read_csv(StringIO(''.join(holdfile)), sep='\t',
encoding='utf-8', infer_datetime_format=True,
keep_default_na=False, na_values=NA_VALUES,
true_values=TRUE_VALUES,
false_values=FALSE_VALUES,
parse_dates=True, index_col=False, comment='\t',
mangle_dupe_cols=False, converters={
index: lambda x: str(x).strip(),
# required sample template information
'physical_location': str,
'sample_type': str,
# collection_timestamp is not added here
'host_subject_id': str,
'description': str,
# common prep template information
'center_name': str,
'center_projct_name': str})
except UnicodeDecodeError:
# Find row number and col number for utf-8 encoding errors
headers = holdfile[0].strip().split('\t')
errors = defaultdict(list)
for row, line in enumerate(holdfile, 1):
for col, cell in enumerate(line.split('\t')):
try:
cell.encode('utf-8')
except UnicodeError:
errors[headers[col]].append(row)
lines = ['%s: row(s) %s' % (header, ', '.join(map(str, rows)))
for header, rows in viewitems(errors)]
raise QiitaDBError('Non UTF-8 characters found in columns:\n' +
'\n'.join(lines))
# Check that we don't have duplicate columns
if len(set(template.columns)) != len(template.columns):
raise QiitaDBDuplicateHeaderError(find_duplicates(template.columns))
# let pandas infer the dtypes of these columns, if the inference is
# not correct, then we have to raise an error
columns_to_dtype = [(['latitude', 'longitude'], (np.int, np.float),
'integer or decimal'),
(['has_physical_specimen', 'has_extracted_data'],
np.bool_, 'boolean')]
for columns, c_dtype, english_desc in columns_to_dtype:
for n in columns:
if n in template.columns and not all([isinstance(val, c_dtype)
for val in template[n]]):
raise QiitaDBColumnError("The '%s' column includes values "
"that cannot be cast into a %s "
"value " % (n, english_desc))
initial_columns = set(template.columns)
if index not in template.columns:
raise QiitaDBColumnError("The '%s' column is missing from "
"your template, this file cannot be parsed."
% index)
# remove rows that have no sample identifier but that may have other data
# in the rest of the columns
template.dropna(subset=[index], how='all', inplace=True)
# set the sample name as the index
template.set_index(index, inplace=True)
# it is not uncommon to find templates that have empty columns
template.dropna(how='all', axis=1, inplace=True)
initial_columns.remove(index)
dropped_cols = initial_columns - set(template.columns)
if dropped_cols:
warnings.warn('The following column(s) were removed from the template '
'because all their values are empty: '
'%s' % ', '.join(dropped_cols), QiitaDBWarning)
# Pandas represents data with np.nan rather than Nones, change it to None
# because psycopg2 knows that a None is a Null in SQL, while it doesn't
# know what to do with NaN
template = template.where((pd.notnull(template)), None)
return template
def load_template_to_dataframe(fn, index='sample_name'):
"""Load a sample/prep template or a QIIME mapping file into a data frame
Parameters
----------
fn : str or file-like object
filename of the template to load, or an already open template file
index : str, optional
Defaults to 'sample_name'. The index to use in the loaded information
Returns
-------
DataFrame
Pandas dataframe with the loaded information
Raises
------
ValueError
Empty file passed
QiitaDBColumnError
If the sample_name column is not present in the template.
QiitaDBWarning
When columns are dropped because they have no content for any sample.
QiitaDBError
When non UTF-8 characters are found in the file.
QiitaDBDuplicateHeaderError
If duplicate columns are present in the template
Notes
-----
The index attribute of the DataFrame will be forced to be 'sample_name'
and will be cast to a string. Additionally rows that start with a '\t'
character will be ignored and columns that are empty will be removed. Empty
sample names will be removed from the DataFrame.
Column names are case-insensitive but will be lowercased on addition to
the database
Everything in the DataFrame will be read and managed as string
"""
# Load in file lines
holdfile = None
with qdb.util.open_file(fn, newline=None,
encoding="utf8", errors='ignore') as f:
holdfile = f.readlines()
if not holdfile:
raise ValueError('Empty file passed!')
if index == "#SampleID":
# We're going to parse a QIIME mapping file. We are going to first
# parse it with the QIIME function so we can remove the comments
# easily and make sure that QIIME will accept this as a mapping file
data, headers, comments = _parse_mapping_file(holdfile)
holdfile = ["%s\n" % '\t'.join(d) for d in data]
holdfile.insert(0, "%s\n" % '\t'.join(headers))
# The QIIME parser fixes the index and removes the #
index = 'SampleID'
# Strip all values in the cells in the input file
for pos, line in enumerate(holdfile):
cols = line.split('\t')
if pos == 0 and index != 'SampleID':
# get and clean the controlled columns
ccols = {'sample_name'}
ccols.update(qdb.metadata_template.constants.CONTROLLED_COLS)
newcols = [
c.lower().strip() if c.lower().strip() in ccols
else c.strip()
for c in cols]
# while we are here, let's check for duplicate columns headers
ncols = set(newcols)
if len(ncols) != len(newcols):
if '' in ncols:
raise ValueError(
'Your file has empty columns headers.')
raise qdb.exceptions.QiitaDBDuplicateHeaderError(
find_duplicates(newcols))
else:
# .strip will remove odd chars, newlines, tabs and multiple
# spaces but we need to read a new line at the end of the
# line(+'\n')
newcols = [d.strip(" \r\n") for d in cols]
holdfile[pos] = '\t'.join(newcols) + '\n'
# index_col:
# is set as False, otherwise it is cast as a float and we want a string
# keep_default:
# is set as False, to avoid inferring empty/NA values with the defaults
# that Pandas has.
# comment:
# using the tab character as "comment" we remove rows that are
# constituted only by delimiters i. e. empty rows.
template = pd.read_csv(
StringIO(''.join(holdfile)),
sep='\t',
dtype=str,
encoding='utf-8',
infer_datetime_format=False,
keep_default_na=False,
index_col=False,
comment='\t',
converters={index: lambda x: str(x).strip()})
# remove newlines and tabs from fields
template.replace(to_replace='[\t\n\r\x0b\x0c]+', value='',
regex=True, inplace=True)
# removing columns with empty values
template.dropna(axis='columns', how='all', inplace=True)
if template.empty:
raise ValueError("The template is empty")
initial_columns = set(template.columns)
if index not in template.columns:
raise qdb.exceptions.QiitaDBColumnError(
"The '%s' column is missing from your template, this file cannot "
"be parsed." % index)
# remove rows that have no sample identifier but that may have other data
# in the rest of the columns
template.dropna(subset=[index], how='all', inplace=True)
# set the sample name as the index
template.set_index(index, inplace=True)
# it is not uncommon to find templates that have empty columns so let's
# find the columns that are all ''
columns = np.where(np.all(template.applymap(lambda x: x == ''), axis=0))
template.drop(template.columns[columns], axis=1, inplace=True)
initial_columns.remove(index)
dropped_cols = initial_columns - set(template.columns)
if dropped_cols:
warnings.warn(
'The following column(s) were removed from the template because '
'all their values are empty: %s'
% ', '.join(dropped_cols), qdb.exceptions.QiitaDBWarning)
# Pandas represents data with np.nan rather than Nones, change it to None
# because psycopg2 knows that a None is a Null in SQL, while it doesn't
# know what to do with NaN
template = template.where((pd.notnull(template)), None)
return template
def test_mixed_types(self):
def gen():
yield from ('a', 1, 'bc', 2, 'a', 2, 2, 3.0)
self.assertEqual(find_duplicates(gen()), set(['a', 2]))
def _clean_validate_template(cls, md_template, study_id, restriction_dict):
"""Takes care of all validation and cleaning of metadata templates
Parameters
----------
md_template : DataFrame
The metadata template file contents indexed by sample ids
study_id : int
The study to which the metadata template belongs to.
restriction_dict : dict of {str: Restriction}
A dictionary with the restrictions that apply to the metadata
Returns
-------
md_template : DataFrame
Cleaned copy of the input md_template
Raises
------
QiitaDBColumnError
If the sample names in md_template contains invalid names
QiitaDBWarning
If there are missing columns required for some functionality
"""
cls._check_subclass()
invalid_ids = get_invalid_sample_names(md_template.index)
if invalid_ids:
raise QiitaDBColumnError("The following sample names in the "
"template contain invalid characters "
"(only alphanumeric characters or periods"
" are allowed): %s." %
", ".join(invalid_ids))
# We are going to modify the md_template. We create a copy so
# we don't modify the user one
md_template = deepcopy(md_template)
# Prefix the sample names with the study_id
prefix_sample_names_with_id(md_template, study_id)
# In the database, all the column headers are lowercase
md_template.columns = [c.lower() for c in md_template.columns]
# Check that we don't have duplicate columns
if len(set(md_template.columns)) != len(md_template.columns):
raise QiitaDBDuplicateHeaderError(
find_duplicates(md_template.columns))
# Check if we have the columns required for some functionality
warning_msg = []
for key, restriction in viewitems(restriction_dict):
missing = set(restriction.columns).difference(md_template)
if missing:
warning_msg.append(
"%s: %s" % (restriction.error_msg, ', '.join(missing)))
if warning_msg:
warnings.warn(
"Some functionality will be disabled due to missing "
"columns:\n\t%s.\nCheck https://github.com/biocore/qiita/wiki"
"/Preparing-Qiita-template-files for a description of these "
"fields." % ";\n\t".join(warning_msg),
QiitaDBWarning)
return md_template
def _clean_validate_template(cls, md_template, study_id, restriction_dict,
current_columns=None):
"""Takes care of all validation and cleaning of metadata templates
Parameters
----------
md_template : DataFrame
The metadata template file contents indexed by sample ids
study_id : int
The study to which the metadata template belongs to.
restriction_dict : dict of {str: Restriction}
A dictionary with the restrictions that apply to the metadata
current_columns : iterable of str, optional
The current list of metadata columns
Returns
-------
md_template : DataFrame
Cleaned copy of the input md_template
Raises
------
QiitaDBColumnError
If the sample names in md_template contains invalid names
QiitaDBWarning
If there are missing columns required for some functionality
"""
cls._check_subclass()
invalid_ids = qdb.metadata_template.util.get_invalid_sample_names(
md_template.index)
if invalid_ids:
raise qdb.exceptions.QiitaDBColumnError(
"The following sample names in the template contain invalid "
"characters (only alphanumeric characters or periods are "
"allowed): %s." % ", ".join(invalid_ids))
if len(set(md_template.index)) != len(md_template.index):
raise qdb.exceptions.QiitaDBDuplicateSamplesError(
find_duplicates(md_template.index))
# We are going to modify the md_template. We create a copy so
# we don't modify the user one
md_template = md_template.copy(deep=True)
# Prefix the sample names with the study_id
qdb.metadata_template.util.prefix_sample_names_with_id(md_template,
study_id)
# In the database, all the column headers are lowercase
md_template.columns = [c.lower() for c in md_template.columns]
# Check that we don't have duplicate columns
if len(set(md_template.columns)) != len(md_template.columns):
raise qdb.exceptions.QiitaDBDuplicateHeaderError(
find_duplicates(md_template.columns))
# Check if we have the columns required for some functionality
warning_msg = []
columns = set(md_template.columns)
if current_columns:
columns.update(current_columns)
for key, restriction in viewitems(restriction_dict):
missing = set(restriction.columns).difference(columns)
if missing:
warning_msg.append(
"%s: %s" % (restriction.error_msg, ', '.join(missing)))
if warning_msg:
warnings.warn(
"Some functionality will be disabled due to missing "
"columns:\n\t%s.\nSee the Templates tutorial for a description"
" of these fields." % ";\n\t".join(warning_msg),
qdb.exceptions.QiitaDBWarning)
return md_template
def test_mixed_types(self):
def gen():
for e in 'a', 1, 'bc', 2, 'a', 2, 2, 3.0:
yield e
self.assertEqual(find_duplicates(gen()), set(['a', 2]))
def _clean_validate_template(cls, md_template, study_id, obj,
conn_handler=None):
"""Takes care of all validation and cleaning of metadata templates
Parameters
----------
md_template : DataFrame
The metadata template file contents indexed by sample ids
study_id : int
The study to which the metadata template belongs to.
obj : object
Any extra object needed by the template to perform any extra check
Returns
-------
md_template : DataFrame
Cleaned copy of the input md_template
Raises
------
QiitaDBColumnError
If the sample names in md_template contains invalid names
QiitaDBDuplicateHeaderError
If md_template contains duplicate headers
QiitaDBColumnError
If md_template is missing a required column
"""
cls._check_subclass()
invalid_ids = get_invalid_sample_names(md_template.index)
if invalid_ids:
raise QiitaDBColumnError("The following sample names in the "
"template contain invalid characters "
"(only alphanumeric characters or periods"
" are allowed): %s." %
", ".join(invalid_ids))
# We are going to modify the md_template. We create a copy so
# we don't modify the user one
md_template = deepcopy(md_template)
# Prefix the sample names with the study_id
prefix_sample_names_with_id(md_template, study_id)
# In the database, all the column headers are lowercase
md_template.columns = [c.lower() for c in md_template.columns]
# Check that we don't have duplicate columns
if len(set(md_template.columns)) != len(md_template.columns):
raise QiitaDBDuplicateHeaderError(
find_duplicates(md_template.columns))
# We need to check for some special columns, that are not present on
# the database, but depending on the data type are required.
missing = cls._check_special_columns(md_template, obj)
conn_handler = conn_handler if conn_handler else SQLConnectionHandler()
# Get the required columns from the DB
db_cols = get_table_cols(cls._table, conn_handler)
# Remove the sample_id and study_id columns
db_cols.remove('sample_id')
db_cols.remove(cls._id_column)
# Retrieve the headers of the metadata template
headers = list(md_template.keys())
# Check that md_template has the required columns
remaining = set(db_cols).difference(headers)
missing = missing.union(remaining)
missing = missing.difference(cls.translate_cols_dict)
if missing:
raise QiitaDBColumnError("Missing columns: %s"
% ', '.join(missing))
return md_template
def _clean_validate_template(cls,
md_template,
study_id,
current_columns=None):
"""Takes care of all validation and cleaning of metadata templates
Parameters
----------
md_template : DataFrame
The metadata template file contents indexed by sample ids
study_id : int
The study to which the metadata template belongs to.
current_columns : iterable of str, optional
The current list of metadata columns
Returns
-------
md_template : DataFrame
Cleaned copy of the input md_template
Raises
------
QiitaDBColumnError
If the sample names in md_template contains invalid names
QiitaDBWarning
If there are missing columns required for some functionality
"""
cls._check_subclass()
invalid_ids = qdb.metadata_template.util.get_invalid_sample_names(
md_template.index)
if invalid_ids:
raise qdb.exceptions.QiitaDBColumnError(
"The following sample names in the template contain invalid "
"characters (only alphanumeric characters or periods are "
"allowed): %s." % ", ".join(invalid_ids))
if len(set(md_template.index)) != len(md_template.index):
raise qdb.exceptions.QiitaDBDuplicateSamplesError(
find_duplicates(md_template.index))
# We are going to modify the md_template. We create a copy so
# we don't modify the user one
md_template = md_template.copy(deep=True)
# In the database, all the column headers are lowercase
md_template.columns = [c.lower() for c in md_template.columns]
# validating pgsql reserved words not to be column headers
current_headers = set(md_template.columns.values)
qdb.metadata_template.util.validate_invalid_column_names(
current_headers)
# Prefix the sample names with the study_id
qdb.metadata_template.util.prefix_sample_names_with_id(
md_template, study_id)
# Check that we don't have duplicate columns
if len(set(md_template.columns)) != len(md_template.columns):
raise qdb.exceptions.QiitaDBDuplicateHeaderError(
find_duplicates(md_template.columns))
return md_template
def test_many_duplicates(self):
self.assertEqual(find_duplicates(['a', 'bc', 'bc', 'def', 'a']),
set(['a', 'bc']))
def load_template_to_dataframe(fn, strip_whitespace=True, index='sample_name'):
"""Load a sample/prep template or a QIIME mapping file into a data frame
Parameters
----------
fn : str or file-like object
filename of the template to load, or an already open template file
strip_whitespace : bool, optional
Defaults to True. Whether or not to strip whitespace from values in the
input file
index : str, optional
Defaults to 'sample_name'. The index to use in the loaded information
Returns
-------
DataFrame
Pandas dataframe with the loaded information
Raises
------
ValueError
Empty file passed
QiitaDBColumnError
If the sample_name column is not present in the template.
QiitaDBWarning
When columns are dropped because they have no content for any sample.
QiitaDBError
When non UTF-8 characters are found in the file.
QiitaDBDuplicateHeaderError
If duplicate columns are present in the template
Notes
-----
The index attribute of the DataFrame will be forced to be 'sample_name'
and will be cast to a string. Additionally rows that start with a '\t'
character will be ignored and columns that are empty will be removed. Empty
sample names will be removed from the DataFrame.
Column names are case-insensitive but will be lowercased on addition to
the database
Everything in the DataFrame will be read and managed as string
"""
# Load in file lines
holdfile = None
with open_file(fn, mode='U') as f:
holdfile = f.readlines()
if not holdfile:
raise ValueError('Empty file passed!')
# Strip all values in the cells in the input file, if requested
if strip_whitespace:
for pos, line in enumerate(holdfile):
holdfile[pos] = '\t'.join(d.strip(" \r\x0b\x0c")
for d in line.split('\t'))
# get and clean the controlled columns
cols = holdfile[0].split('\t')
controlled_cols = {'sample_name'}
controlled_cols.update(qdb.metadata_template.constants.CONTROLLED_COLS)
holdfile[0] = '\t'.join(c.lower() if c.lower() in controlled_cols else c
for c in cols)
if index == "#SampleID":
# We're going to parse a QIIME mapping file. We are going to first
# parse it with the QIIME function so we can remove the comments
# easily and make sure that QIIME will accept this as a mapping file
data, headers, comments = _parse_mapping_file(holdfile)
holdfile = ["%s\n" % '\t'.join(d) for d in data]
holdfile.insert(0, "%s\n" % '\t'.join(headers))
# The QIIME parser fixes the index and removes the #
index = 'SampleID'
# Check that we don't have duplicate columns
col_names = [c.lower() for c in holdfile[0].strip().split('\t')]
if len(set(col_names)) != len(col_names):
raise qdb.exceptions.QiitaDBDuplicateHeaderError(
find_duplicates(col_names))
# index_col:
# is set as False, otherwise it is cast as a float and we want a string
# keep_default:
# is set as False, to avoid inferring empty/NA values with the defaults
# that Pandas has.
# comment:
# using the tab character as "comment" we remove rows that are
# constituted only by delimiters i. e. empty rows.
try:
template = pd.read_csv(
StringIO(''.join(holdfile)),
sep='\t',
dtype=str,
encoding='utf-8',
infer_datetime_format=False,
keep_default_na=False,
index_col=False,
comment='\t',
converters={index: lambda x: str(x).strip()})
except UnicodeDecodeError:
# Find row number and col number for utf-8 encoding errors
headers = holdfile[0].strip().split('\t')
errors = defaultdict(list)
for row, line in enumerate(holdfile, 1):
for col, cell in enumerate(line.split('\t')):
try:
cell.encode('utf-8')
except UnicodeError:
errors[headers[col]].append(row)
lines = ['%s: row(s) %s' % (header, ', '.join(map(str, rows)))
for header, rows in viewitems(errors)]
raise qdb.exceptions.QiitaDBError(
'Non UTF-8 characters found in columns:\n' + '\n'.join(lines))
initial_columns = set(template.columns)
if index not in template.columns:
raise qdb.exceptions.QiitaDBColumnError(
"The '%s' column is missing from your template, this file cannot "
"be parsed." % index)
# remove rows that have no sample identifier but that may have other data
# in the rest of the columns
template.dropna(subset=[index], how='all', inplace=True)
# set the sample name as the index
template.set_index(index, inplace=True)
# it is not uncommon to find templates that have empty columns so let's
# find the columns that are all ''
columns = np.where(np.all(template.applymap(lambda x: x == ''), axis=0))
template.drop(template.columns[columns], axis=1, inplace=True)
initial_columns.remove(index)
dropped_cols = initial_columns - set(template.columns)
if dropped_cols:
warnings.warn(
'The following column(s) were removed from the template because '
'all their values are empty: %s'
% ', '.join(dropped_cols), qdb.exceptions.QiitaDBWarning)
# Pandas represents data with np.nan rather than Nones, change it to None
# because psycopg2 knows that a None is a Null in SQL, while it doesn't
# know what to do with NaN
template = template.where((pd.notnull(template)), None)
return template
def test_mixed_types(self):
def gen():
for e in 'a', 1, 'bc', 2, 'a', 2, 2, 3.0:
yield e
self.assertEqual(find_duplicates(gen()), set(['a', 2]))
def load_template_to_dataframe(fn, strip_whitespace=True, index='sample_name'):
"""Load a sample/prep template or a QIIME mapping file into a data frame
Parameters
----------
fn : str or file-like object
filename of the template to load, or an already open template file
strip_whitespace : bool, optional
Defaults to True. Whether or not to strip whitespace from values in the
input file
index : str, optional
Defaults to 'sample_name'. The index to use in the loaded information
Returns
-------
DataFrame
Pandas dataframe with the loaded information
Raises
------
ValueError
Empty file passed
QiitaDBColumnError
If the sample_name column is not present in the template.
QiitaDBWarning
When columns are dropped because they have no content for any sample.
QiitaDBError
When non UTF-8 characters are found in the file.
QiitaDBDuplicateHeaderError
If duplicate columns are present in the template
Notes
-----
The index attribute of the DataFrame will be forced to be 'sample_name'
and will be cast to a string. Additionally rows that start with a '\t'
character will be ignored and columns that are empty will be removed. Empty
sample names will be removed from the DataFrame.
Column names are case-insensitive but will be lowercased on addition to
the database
Everything in the DataFrame will be read and managed as string
"""
# Load in file lines
holdfile = None
with open_file(fn, mode='U') as f:
holdfile = f.readlines()
if not holdfile:
raise ValueError('Empty file passed!')
# Strip all values in the cells in the input file, if requested
if strip_whitespace:
for pos, line in enumerate(holdfile):
holdfile[pos] = '\t'.join(
d.strip(" \r\x0b\x0c") for d in line.split('\t'))
# get and clean the controlled columns
cols = holdfile[0].split('\t')
controlled_cols = {'sample_name'}
controlled_cols.update(qdb.metadata_template.constants.CONTROLLED_COLS)
holdfile[0] = '\t'.join(c.lower() if c.lower() in controlled_cols else c
for c in cols)
if index == "#SampleID":
# We're going to parse a QIIME mapping file. We are going to first
# parse it with the QIIME function so we can remove the comments
# easily and make sure that QIIME will accept this as a mapping file
data, headers, comments = _parse_mapping_file(holdfile)
holdfile = ["%s\n" % '\t'.join(d) for d in data]
holdfile.insert(0, "%s\n" % '\t'.join(headers))
# The QIIME parser fixes the index and removes the #
index = 'SampleID'
# Check that we don't have duplicate columns
col_names = [c.lower() for c in holdfile[0].strip().split('\t')]
if len(set(col_names)) != len(col_names):
raise qdb.exceptions.QiitaDBDuplicateHeaderError(
find_duplicates(col_names))
# index_col:
# is set as False, otherwise it is cast as a float and we want a string
# keep_default:
# is set as False, to avoid inferring empty/NA values with the defaults
# that Pandas has.
# comment:
# using the tab character as "comment" we remove rows that are
# constituted only by delimiters i. e. empty rows.
try:
template = pd.read_csv(StringIO(''.join(holdfile)),
sep='\t',
dtype=str,
encoding='utf-8',
infer_datetime_format=False,
keep_default_na=False,
index_col=False,
comment='\t',
converters={index: lambda x: str(x).strip()})
except UnicodeDecodeError:
# Find row number and col number for utf-8 encoding errors
headers = holdfile[0].strip().split('\t')
errors = defaultdict(list)
for row, line in enumerate(holdfile, 1):
for col, cell in enumerate(line.split('\t')):
try:
cell.encode('utf-8')
except UnicodeError:
errors[headers[col]].append(row)
lines = [
'%s: row(s) %s' % (header, ', '.join(map(str, rows)))
for header, rows in viewitems(errors)
]
raise qdb.exceptions.QiitaDBError(
'Non UTF-8 characters found in columns:\n' + '\n'.join(lines))
initial_columns = set(template.columns)
if index not in template.columns:
raise qdb.exceptions.QiitaDBColumnError(
"The '%s' column is missing from your template, this file cannot "
"be parsed." % index)
# remove rows that have no sample identifier but that may have other data
# in the rest of the columns
template.dropna(subset=[index], how='all', inplace=True)
# set the sample name as the index
template.set_index(index, inplace=True)
# it is not uncommon to find templates that have empty columns so let's
# find the columns that are all ''
columns = np.where(np.all(template.applymap(lambda x: x == ''), axis=0))
template.drop(template.columns[columns], axis=1, inplace=True)
initial_columns.remove(index)
dropped_cols = initial_columns - set(template.columns)
if dropped_cols:
warnings.warn(
'The following column(s) were removed from the template because '
'all their values are empty: %s' % ', '.join(dropped_cols),
qdb.exceptions.QiitaDBWarning)
# Pandas represents data with np.nan rather than Nones, change it to None
# because psycopg2 knows that a None is a Null in SQL, while it doesn't
# know what to do with NaN
template = template.where((pd.notnull(template)), None)
return template
def test_mixed_types(self):
def gen():
yield from ('a', 1, 'bc', 2, 'a', 2, 2, 3.0)
self.assertEqual(find_duplicates(gen()), set(['a', 2]))
def load_template_to_dataframe(fn, strip_whitespace=True, index='sample_name'):
"""Load a sample/prep template or a QIIME mapping file into a data frame
Parameters
----------
fn : str or file-like object
filename of the template to load, or an already open template file
strip_whitespace : bool, optional
Defaults to True. Whether or not to strip whitespace from values in the
input file
index : str, optional
Defaults to 'sample_name'. The index to use in the loaded information
Returns
-------
DataFrame
Pandas dataframe with the loaded information
Raises
------
ValueError
Empty file passed
QiitaDBColumnError
If the sample_name column is not present in the template.
If there's a value in one of the reserved columns that cannot be cast
to the needed type.
QiitaDBWarning
When columns are dropped because they have no content for any sample.
QiitaDBError
When non UTF-8 characters are found in the file.
QiitaDBDuplicateHeaderError
If duplicate columns are present in the template
Notes
-----
The index attribute of the DataFrame will be forced to be 'sample_name'
and will be cast to a string. Additionally rows that start with a '\t'
character will be ignored and columns that are empty will be removed. Empty
sample names will be removed from the DataFrame.
The following table describes the data type per column that will be
enforced in `fn`. Column names are case-insensitive but will be lowercased
on addition to the database.
+-----------------------+--------------+
| Column Name | Python Type |
+=======================+==============+
| sample_name | str |
+-----------------------+--------------+
| #SampleID | str |
+-----------------------+--------------+
| physical_location | str |
+-----------------------+--------------+
| has_physical_specimen | bool |
+-----------------------+--------------+
| has_extracted_data | bool |
+-----------------------+--------------+
| sample_type | str |
+-----------------------+--------------+
| host_subject_id | str |
+-----------------------+--------------+
| description | str |
+-----------------------+--------------+
| latitude | float |
+-----------------------+--------------+
| longitude | float |
+-----------------------+--------------+
"""
# Load in file lines
holdfile = None
with open_file(fn, mode='U') as f:
holdfile = f.readlines()
if not holdfile:
raise ValueError('Empty file passed!')
# Strip all values in the cells in the input file, if requested
if strip_whitespace:
for pos, line in enumerate(holdfile):
holdfile[pos] = '\t'.join(
d.strip(" \r\x0b\x0c") for d in line.split('\t'))
# get and clean the controlled columns
cols = holdfile[0].split('\t')
controlled_cols = {'sample_name'}
controlled_cols.update(qdb.metadata_template.constants.CONTROLLED_COLS)
holdfile[0] = '\t'.join(c.lower() if c.lower() in controlled_cols else c
for c in cols)
if index == "#SampleID":
# We're going to parse a QIIME mapping file. We are going to first
# parse it with the QIIME function so we can remove the comments
# easily and make sure that QIIME will accept this as a mapping file
data, headers, comments = _parse_mapping_file(holdfile)
holdfile = ["%s\n" % '\t'.join(d) for d in data]
holdfile.insert(0, "%s\n" % '\t'.join(headers))
# The QIIME parser fixes the index and removes the #
index = 'SampleID'
# index_col:
# is set as False, otherwise it is cast as a float and we want a string
# keep_default:
# is set as False, to avoid inferring empty/NA values with the defaults
# that Pandas has.
# na_values:
# the values that should be considered as empty
# true_values:
# the values that should be considered "True" for boolean columns
# false_values:
# the values that should be considered "False" for boolean columns
# converters:
# ensure that sample names are not converted into any other types but
# strings and remove any trailing spaces. Don't let pandas try to guess
# the dtype of the other columns, force them to be a str.
# comment:
# using the tab character as "comment" we remove rows that are
# constituted only by delimiters i. e. empty rows.
try:
template = pd.read_csv(
StringIO(''.join(holdfile)),
sep='\t',
encoding='utf-8',
infer_datetime_format=True,
keep_default_na=False,
na_values=qdb.metadata_template.constants.NA_VALUES,
true_values=qdb.metadata_template.constants.TRUE_VALUES,
false_values=qdb.metadata_template.constants.FALSE_VALUES,
parse_dates=True,
index_col=False,
comment='\t',
mangle_dupe_cols=False,
converters={
index: lambda x: str(x).strip(),
# required sample template information
'physical_location': str,
'sample_type': str,
# collection_timestamp is not added here
'host_subject_id': str,
'description': str,
# common prep template information
'center_name': str,
'center_projct_name': str
})
except UnicodeDecodeError:
# Find row number and col number for utf-8 encoding errors
headers = holdfile[0].strip().split('\t')
errors = defaultdict(list)
for row, line in enumerate(holdfile, 1):
for col, cell in enumerate(line.split('\t')):
try:
cell.encode('utf-8')
except UnicodeError:
errors[headers[col]].append(row)
lines = [
'%s: row(s) %s' % (header, ', '.join(map(str, rows)))
for header, rows in viewitems(errors)
]
raise qdb.exceptions.QiitaDBError(
'Non UTF-8 characters found in columns:\n' + '\n'.join(lines))
# Check that we don't have duplicate columns
if len(set(template.columns)) != len(template.columns):
raise qdb.exceptions.QiitaDBDuplicateHeaderError(
find_duplicates(template.columns))
# let pandas infer the dtypes of these columns, if the inference is
# not correct, then we have to raise an error
columns_to_dtype = [
(['latitude', 'longitude'], (np.int, np.float), 'integer or decimal'),
(['has_physical_specimen', 'has_extracted_data'], np.bool_, 'boolean')
]
for columns, c_dtype, english_desc in columns_to_dtype:
for n in columns:
if n in template.columns and not all(
[isinstance(val, c_dtype) for val in template[n]]):
raise qdb.exceptions.QiitaDBColumnError(
"The '%s' column includes values that cannot be cast "
"into a %s value " % (n, english_desc))
initial_columns = set(template.columns)
if index not in template.columns:
raise qdb.exceptions.QiitaDBColumnError(
"The '%s' column is missing from your template, this file cannot "
"be parsed." % index)
# remove rows that have no sample identifier but that may have other data
# in the rest of the columns
template.dropna(subset=[index], how='all', inplace=True)
# set the sample name as the index
template.set_index(index, inplace=True)
# it is not uncommon to find templates that have empty columns
template.dropna(how='all', axis=1, inplace=True)
initial_columns.remove(index)
dropped_cols = initial_columns - set(template.columns)
if dropped_cols:
warnings.warn(
'The following column(s) were removed from the template because '
'all their values are empty: %s' % ', '.join(dropped_cols),
qdb.exceptions.QiitaDBWarning)
# Pandas represents data with np.nan rather than Nones, change it to None
# because psycopg2 knows that a None is a Null in SQL, while it doesn't
# know what to do with NaN
template = template.where((pd.notnull(template)), None)
return template
def load_template_to_dataframe(fn, index='sample_name'):
"""Load a sample/prep template or a QIIME mapping file into a data frame
Parameters
----------
fn : str or file-like object
filename of the template to load, or an already open template file
index : str, optional
Defaults to 'sample_name'. The index to use in the loaded information
Returns
-------
DataFrame
Pandas dataframe with the loaded information
Raises
------
ValueError
Empty file passed
QiitaDBColumnError
If the sample_name column is not present in the template.
QiitaDBWarning
When columns are dropped because they have no content for any sample.
QiitaDBError
When non UTF-8 characters are found in the file.
QiitaDBDuplicateHeaderError
If duplicate columns are present in the template
Notes
-----
The index attribute of the DataFrame will be forced to be 'sample_name'
and will be cast to a string. Additionally rows that start with a '\t'
character will be ignored and columns that are empty will be removed. Empty
sample names will be removed from the DataFrame.
Column names are case-insensitive but will be lowercased on addition to
the database
Everything in the DataFrame will be read and managed as string
"""
# Load in file lines
holdfile = None
with qdb.util.open_file(fn, mode='U') as f:
errors = defaultdict(list)
holdfile = f.readlines()
# here we are checking for non UTF-8 chars
for row, line in enumerate(holdfile):
for col, block in enumerate(line.split('\t')):
try:
tblock = block.encode('utf-8')
except UnicodeDecodeError:
tblock = unicode(block, errors='replace')
tblock = tblock.replace(u'\ufffd', '🐾')
errors[tblock].append('(%d, %d)' % (row, col))
if bool(errors):
raise ValueError(
"There are invalid (non UTF-8) characters in your information "
"file. The offending fields and their location (row, column) "
"are listed below, invalid characters are represented using "
"🐾: %s" % '; '.join([
'"%s" = %s' % (k, ', '.join(v))
for k, v in viewitems(errors)
]))
if not holdfile:
raise ValueError('Empty file passed!')
if index == "#SampleID":
# We're going to parse a QIIME mapping file. We are going to first
# parse it with the QIIME function so we can remove the comments
# easily and make sure that QIIME will accept this as a mapping file
data, headers, comments = _parse_mapping_file(holdfile)
holdfile = ["%s\n" % '\t'.join(d) for d in data]
holdfile.insert(0, "%s\n" % '\t'.join(headers))
# The QIIME parser fixes the index and removes the #
index = 'SampleID'
# Strip all values in the cells in the input file
for pos, line in enumerate(holdfile):
cols = line.split('\t')
if pos == 0 and index != 'SampleID':
# get and clean the controlled columns
ccols = {'sample_name'}
ccols.update(qdb.metadata_template.constants.CONTROLLED_COLS)
newcols = [
c.lower().strip() if c.lower().strip() in ccols else c.strip()
for c in cols
]
# while we are here, let's check for duplicate columns headers
ncols = set(newcols)
if len(ncols) != len(newcols):
if '' in ncols:
raise ValueError('Your file has empty columns headers.')
raise qdb.exceptions.QiitaDBDuplicateHeaderError(
find_duplicates(newcols))
else:
# .strip will remove odd chars, newlines, tabs and multiple
# spaces but we need to read a new line at the end of the
# line(+'\n')
newcols = [d.strip(" \r\n") for d in cols]
holdfile[pos] = '\t'.join(newcols) + '\n'
# index_col:
# is set as False, otherwise it is cast as a float and we want a string
# keep_default:
# is set as False, to avoid inferring empty/NA values with the defaults
# that Pandas has.
# comment:
# using the tab character as "comment" we remove rows that are
# constituted only by delimiters i. e. empty rows.
template = pd.read_csv(StringIO(''.join(holdfile)),
sep='\t',
dtype=str,
encoding='utf-8',
infer_datetime_format=False,
keep_default_na=False,
index_col=False,
comment='\t',
converters={index: lambda x: str(x).strip()})
# remove newlines and tabs from fields
template.replace(to_replace='[\t\n\r\x0b\x0c]+',
value='',
regex=True,
inplace=True)
# removing columns with empty values
template.dropna(axis='columns', how='all', inplace=True)
if template.empty:
raise ValueError("The template is empty")
initial_columns = set(template.columns)
if index not in template.columns:
raise qdb.exceptions.QiitaDBColumnError(
"The '%s' column is missing from your template, this file cannot "
"be parsed." % index)
# remove rows that have no sample identifier but that may have other data
# in the rest of the columns
template.dropna(subset=[index], how='all', inplace=True)
# set the sample name as the index
template.set_index(index, inplace=True)
# it is not uncommon to find templates that have empty columns so let's
# find the columns that are all ''
columns = np.where(np.all(template.applymap(lambda x: x == ''), axis=0))
template.drop(template.columns[columns], axis=1, inplace=True)
initial_columns.remove(index)
dropped_cols = initial_columns - set(template.columns)
if dropped_cols:
warnings.warn(
'The following column(s) were removed from the template because '
'all their values are empty: %s' % ', '.join(dropped_cols),
qdb.exceptions.QiitaDBWarning)
# Pandas represents data with np.nan rather than Nones, change it to None
# because psycopg2 knows that a None is a Null in SQL, while it doesn't
# know what to do with NaN
template = template.where((pd.notnull(template)), None)
return template
def load_template_to_dataframe(fn, index='sample_name'):
"""Load a sample/prep template or a QIIME mapping file into a data frame
Parameters
----------
fn : str or file-like object
filename of the template to load, or an already open template file
index : str, optional
Defaults to 'sample_name'. The index to use in the loaded information
Returns
-------
DataFrame
Pandas dataframe with the loaded information
Raises
------
ValueError
Empty file passed
QiitaDBColumnError
If the sample_name column is not present in the template.
QiitaDBWarning
When columns are dropped because they have no content for any sample.
QiitaDBError
When non UTF-8 characters are found in the file.
QiitaDBDuplicateHeaderError
If duplicate columns are present in the template
Notes
-----
The index attribute of the DataFrame will be forced to be 'sample_name'
and will be cast to a string. Additionally rows that start with a '\t'
character will be ignored and columns that are empty will be removed. Empty
sample names will be removed from the DataFrame.
Column names are case-insensitive but will be lowercased on addition to
the database
Everything in the DataFrame will be read and managed as string
While reading the file via pandas, it's possible that it will raise a
'tokenizing' pd.errors.ParserError which is confusing for users; thus,
rewriting the error with an explanation of what it means and how to fix.
"""
# Load in file lines
holdfile = None
with qdb.util.open_file(fn, newline=None, encoding="utf8",
errors='ignore') as f:
holdfile = f.readlines()
if not holdfile:
raise ValueError('Empty file passed!')
if index == "#SampleID":
# We're going to parse a QIIME mapping file. We are going to first
# parse it with the QIIME function so we can remove the comments
# easily and make sure that QIIME will accept this as a mapping file
data, headers, comments = _parse_mapping_file(holdfile)
holdfile = ["%s\n" % '\t'.join(d) for d in data]
holdfile.insert(0, "%s\n" % '\t'.join(headers))
# The QIIME parser fixes the index and removes the #
index = 'SampleID'
# Strip all values in the cells in the input file
for pos, line in enumerate(holdfile):
cols = line.split('\t')
if pos == 0 and index != 'SampleID':
# get and clean the controlled columns
ccols = {'sample_name'}
ccols.update(qdb.metadata_template.constants.CONTROLLED_COLS)
newcols = [
c.lower().strip() if c.lower().strip() in ccols else c.strip()
for c in cols
]
# while we are here, let's check for duplicate columns headers
ncols = set(newcols)
if len(ncols) != len(newcols):
if '' in ncols:
raise ValueError('Your file has empty columns headers.')
raise qdb.exceptions.QiitaDBDuplicateHeaderError(
find_duplicates(newcols))
else:
# .strip will remove odd chars, newlines, tabs and multiple
# spaces but we need to read a new line at the end of the
# line(+'\n')
newcols = [d.strip(" \r\n") for d in cols]
holdfile[pos] = '\t'.join(newcols) + '\n'
# index_col:
# is set as False, otherwise it is cast as a float and we want a string
# keep_default:
# is set as False, to avoid inferring empty/NA values with the defaults
# that Pandas has.
# comment:
# using the tab character as "comment" we remove rows that are
# constituted only by delimiters i. e. empty rows.
try:
template = pd.read_csv(StringIO(''.join(holdfile)),
sep='\t',
dtype=str,
encoding='utf-8',
infer_datetime_format=False,
keep_default_na=False,
index_col=False,
comment='\t',
converters={index: lambda x: str(x).strip()})
except pd.errors.ParserError as e:
if 'tokenizing' in str(e):
msg = ('Your file has more columns with values than headers. To '
'fix, make sure to delete any extra rows or columns; they '
'might look empty because they have spaces. Then upload '
'and try again.')
raise RuntimeError(msg)
else:
raise e
# remove newlines and tabs from fields
template.replace(to_replace='[\t\n\r\x0b\x0c]+',
value='',
regex=True,
inplace=True)
# removing columns with empty values
template.dropna(axis='columns', how='all', inplace=True)
if template.empty:
raise ValueError("The template is empty")
initial_columns = set(template.columns)
if index not in template.columns:
raise qdb.exceptions.QiitaDBColumnError(
"The '%s' column is missing from your template, this file cannot "
"be parsed." % index)
# remove rows that have no sample identifier but that may have other data
# in the rest of the columns
template.dropna(subset=[index], how='all', inplace=True)
# set the sample name as the index
template.set_index(index, inplace=True)
# it is not uncommon to find templates that have empty columns so let's
# find the columns that are all ''
columns = np.where(np.all(template.applymap(lambda x: x == ''), axis=0))
template.drop(template.columns[columns], axis=1, inplace=True)
initial_columns.remove(index)
dropped_cols = initial_columns - set(template.columns)
if dropped_cols:
warnings.warn(
'The following column(s) were removed from the template because '
'all their values are empty: %s' % ', '.join(dropped_cols),
qdb.exceptions.QiitaDBWarning)
# removing 'sample-id' and 'sample_id' as per issue #2906
sdrop = []
if 'sample-id' in template.columns:
sdrop.append('sample-id')
if 'sample_id' in template.columns:
sdrop.append('sample_id')
if sdrop:
template.drop(columns=sdrop, inplace=True)
warnings.warn(
'The following column(s) were removed from the template because '
'they will cause conflicts with sample_name: %s' %
', '.join(sdrop), qdb.exceptions.QiitaDBWarning)
# Pandas represents data with np.nan rather than Nones, change it to None
# because psycopg2 knows that a None is a Null in SQL, while it doesn't
# know what to do with NaN
template = template.where((pd.notnull(template)), None)
return template
def _build_mapping_file(self, samples):
"""Builds the combined mapping file for all samples
Code modified slightly from qiime.util.MetadataMap.__add__"""
with qdb.sql_connection.TRN:
all_sample_ids = set()
sql = """SELECT filepath_id, filepath
FROM qiita.filepath
JOIN qiita.prep_template_filepath USING (filepath_id)
JOIN qiita.prep_template USING (prep_template_id)
JOIN qiita.filepath_type USING (filepath_type_id)
WHERE filepath_type = 'qiime_map'
AND artifact_id IN (SELECT *
FROM qiita.find_artifact_roots(%s))
ORDER BY filepath_id DESC"""
_id, fp = qdb.util.get_mountpoint('templates')[0]
to_concat = []
for pid, samples in viewitems(samples):
if len(samples) != len(set(samples)):
duplicates = find_duplicates(samples)
raise qdb.exceptions.QiitaDBError(
"Duplicate sample ids found: %s"
% ', '.join(duplicates))
# Get the QIIME mapping file
qdb.sql_connection.TRN.add(sql, [pid])
qiime_map_fp = \
qdb.sql_connection.TRN.execute_fetchindex()[0][1]
# Parse the mapping file
qiime_map = pd.read_csv(
join(fp, qiime_map_fp), sep='\t', keep_default_na=False,
na_values=['unknown'], index_col=False,
converters=defaultdict(lambda: str))
qiime_map.set_index('#SampleID', inplace=True, drop=True)
qiime_map = qiime_map.loc[samples]
duplicates = all_sample_ids.intersection(qiime_map.index)
if duplicates or len(samples) != len(set(samples)):
# Duplicate samples so raise error
raise qdb.exceptions.QiitaDBError(
"Duplicate sample ids found: %s"
% ', '.join(duplicates))
all_sample_ids.update(qiime_map.index)
to_concat.append(qiime_map)
merged_map = pd.concat(to_concat)
cols = merged_map.columns.values.tolist()
cols.remove('BarcodeSequence')
cols.remove('LinkerPrimerSequence')
cols.remove('Description')
new_cols = ['BarcodeSequence', 'LinkerPrimerSequence']
new_cols.extend(cols)
new_cols.append('Description')
merged_map = merged_map[new_cols]
# Save the mapping file
_, base_fp = qdb.util.get_mountpoint(self._table)[0]
mapping_fp = join(base_fp, "%d_analysis_mapping.txt" % self._id)
merged_map.to_csv(mapping_fp, index_label='#SampleID',
na_rep='unknown', sep='\t')
self._add_file("%d_analysis_mapping.txt" % self._id, "plain_text")
def _clean_validate_template(cls,
md_template,
study_id,
restriction_dict,
current_columns=None):
"""Takes care of all validation and cleaning of metadata templates
Parameters
----------
md_template : DataFrame
The metadata template file contents indexed by sample ids
study_id : int
The study to which the metadata template belongs to.
restriction_dict : dict of {str: Restriction}
A dictionary with the restrictions that apply to the metadata
current_columns : iterable of str, optional
The current list of metadata columns
Returns
-------
md_template : DataFrame
Cleaned copy of the input md_template
Raises
------
QiitaDBColumnError
If the sample names in md_template contains invalid names
QiitaDBWarning
If there are missing columns required for some functionality
"""
cls._check_subclass()
invalid_ids = qdb.metadata_template.util.get_invalid_sample_names(
md_template.index)
if invalid_ids:
raise qdb.exceptions.QiitaDBColumnError(
"The following sample names in the template contain invalid "
"characters (only alphanumeric characters or periods are "
"allowed): %s." % ", ".join(invalid_ids))
if len(set(md_template.index)) != len(md_template.index):
raise qdb.exceptions.QiitaDBDuplicateSamplesError(
find_duplicates(md_template.index))
# We are going to modify the md_template. We create a copy so
# we don't modify the user one
md_template = md_template.copy(deep=True)
# Prefix the sample names with the study_id
qdb.metadata_template.util.prefix_sample_names_with_id(
md_template, study_id)
# In the database, all the column headers are lowercase
md_template.columns = [c.lower() for c in md_template.columns]
# Check that we don't have duplicate columns
if len(set(md_template.columns)) != len(md_template.columns):
raise qdb.exceptions.QiitaDBDuplicateHeaderError(
find_duplicates(md_template.columns))
# Check if we have the columns required for some functionality
warning_msg = []
columns = set(md_template.columns)
if current_columns:
columns.update(current_columns)
for key, restriction in viewitems(restriction_dict):
missing = set(restriction.columns).difference(columns)
if missing:
warning_msg.append(
"%s: %s" %
(restriction.error_msg, ', '.join(sorted(missing))))
if warning_msg:
warnings.warn(
"Some functionality will be disabled due to missing "
"columns:\n\t%s.\nSee the Templates tutorial for a description"
" of these fields." % ";\n\t".join(warning_msg),
qdb.exceptions.QiitaDBWarning)
return md_template
