def test_insert_filepaths_queue(self):
fd, fp = mkstemp()
close(fd)
with open(fp, "w") as f:
f.write("\n")
self.files_to_remove.append(fp)
# create and populate queue
self.conn_handler.create_queue("toy_queue")
self.conn_handler.add_to_queue(
"toy_queue", "INSERT INTO qiita.qiita_user (email, name, password,"
"phone) VALUES (%s, %s, %s, %s)",
['*****@*****.**', 'Toy', 'pass', '111-111-1111'])
exp_new_id = 1 + self.conn_handler.execute_fetchone(
"SELECT count(1) FROM qiita.filepath")[0]
insert_filepaths([(fp, "raw_forward_seqs")],
1,
"raw_data",
"filepath",
self.conn_handler,
queue='toy_queue')
self.conn_handler.add_to_queue(
"toy_queue", "INSERT INTO qiita.raw_filepath (raw_data_id, "
"filepath_id) VALUES (1, %s)", ['{0}'])
self.conn_handler.execute_queue("toy_queue")
# check that the user was added to the DB
obs = self.conn_handler.execute_fetchall(
"SELECT * from qiita.qiita_user WHERE email = %s",
['*****@*****.**'])
exp = [[
'*****@*****.**', 5, 'pass', 'Toy', None, None, '111-111-1111',
None, None, None
]]
self.assertEqual(obs, exp)
# Check that the filepaths have been added to the DB
obs = self.conn_handler.execute_fetchall(
"SELECT * FROM qiita.filepath WHERE filepath_id=%d" % exp_new_id)
exp_fp = "1_%s" % basename(fp)
exp = [[exp_new_id, exp_fp, 1, '852952723', 1, 5]]
self.assertEqual(obs, exp)
# check that raw_filpath data was added to the DB
obs = self.conn_handler.execute_fetchall("""SELECT *
FROM qiita.raw_filepath
WHERE filepath_id=%d""" % exp_new_id)
exp_fp = "1_%s" % basename(fp)
exp = [[1, exp_new_id]]
self.assertEqual(obs, exp)
def test_insert_filepaths_string(self):
fd, fp = mkstemp()
close(fd)
with open(fp, "w") as f:
f.write("\n")
self.files_to_remove.append(fp)
exp_new_id = 1 + self.conn_handler.execute_fetchone(
"SELECT count(1) FROM qiita.filepath")[0]
obs = insert_filepaths([(fp, "raw_forward_seqs")], 1, "raw_data",
"filepath", self.conn_handler)
self.assertEqual(obs, [exp_new_id])
# Check that the files have been copied correctly
exp_fp = join(get_db_files_base_dir(), "raw_data",
"1_%s" % basename(fp))
self.assertTrue(exists(exp_fp))
self.files_to_remove.append(exp_fp)
# Check that the filepaths have been added to the DB
obs = self.conn_handler.execute_fetchall(
"SELECT * FROM qiita.filepath WHERE filepath_id=%d" % exp_new_id)
exp_fp = "1_%s" % basename(fp)
exp = [[exp_new_id, exp_fp, 1, '852952723', 1, 5]]
self.assertEqual(obs, exp)
def test_insert_filepaths_string(self):
fd, fp = mkstemp()
close(fd)
with open(fp, "w") as f:
f.write("\n")
self.files_to_remove.append(fp)
exp_new_id = 1 + self.conn_handler.execute_fetchone(
"SELECT count(1) FROM qiita.filepath")[0]
obs = insert_filepaths([(fp, "raw_forward_seqs")], 1, "raw_data",
"filepath", self.conn_handler)
self.assertEqual(obs, [exp_new_id])
# Check that the files have been copied correctly
exp_fp = join(get_db_files_base_dir(), "raw_data",
"1_%s" % basename(fp))
self.assertTrue(exists(exp_fp))
self.files_to_remove.append(exp_fp)
# Check that the filepaths have been added to the DB
obs = self.conn_handler.execute_fetchall(
"SELECT * FROM qiita.filepath WHERE filepath_id=%d" % exp_new_id)
exp_fp = "1_%s" % basename(fp)
exp = [[exp_new_id, exp_fp, 1, '852952723', 1, 5]]
self.assertEqual(obs, exp)
def test_insert_filepaths_queue(self):
fd, fp = mkstemp()
close(fd)
with open(fp, "w") as f:
f.write("\n")
self.files_to_remove.append(fp)
# create and populate queue
self.conn_handler.create_queue("toy_queue")
self.conn_handler.add_to_queue(
"toy_queue", "INSERT INTO qiita.qiita_user (email, name, password,"
"phone) VALUES (%s, %s, %s, %s)",
['*****@*****.**', 'Toy', 'pass', '111-111-1111'])
exp_new_id = 1 + self.conn_handler.execute_fetchone(
"SELECT count(1) FROM qiita.filepath")[0]
insert_filepaths([(fp, "raw_forward_seqs")], 1, "raw_data",
"filepath", self.conn_handler, queue='toy_queue')
self.conn_handler.add_to_queue(
"toy_queue", "INSERT INTO qiita.raw_filepath (raw_data_id, "
"filepath_id) VALUES (1, %s)", ['{0}'])
self.conn_handler.execute_queue("toy_queue")
# check that the user was added to the DB
obs = self.conn_handler.execute_fetchall(
"SELECT * from qiita.qiita_user WHERE email = %s",
['*****@*****.**'])
exp = [['*****@*****.**', 5, 'pass', 'Toy', None, None, '111-111-1111',
None, None, None]]
self.assertEqual(obs, exp)
# Check that the filepaths have been added to the DB
obs = self.conn_handler.execute_fetchall(
"SELECT * FROM qiita.filepath WHERE filepath_id=%d" % exp_new_id)
exp_fp = "1_%s" % basename(fp)
exp = [[exp_new_id, exp_fp, 1, '852952723', 1, 5]]
self.assertEqual(obs, exp)
# check that raw_filpath data was added to the DB
obs = self.conn_handler.execute_fetchall(
"""SELECT *
FROM qiita.raw_filepath
WHERE filepath_id=%d""" % exp_new_id)
exp_fp = "1_%s" % basename(fp)
exp = [[1, exp_new_id]]
self.assertEqual(obs, exp)
def add_filepath(self, filepath, fp_id=None):
r"""Populates the DB tables for storing the filepath and connects the
`self` objects with this filepath"""
with TRN:
fp_id = self._fp_id if fp_id is None else fp_id
try:
fpp_id = insert_filepaths([(filepath, fp_id)], None, "templates", "filepath", move_files=False)[0]
sql = """INSERT INTO qiita.{0} ({1}, filepath_id)
VALUES (%s, %s)""".format(
self._filepath_table, self._id_column
)
TRN.add(sql, [self._id, fpp_id])
TRN.execute()
except Exception as e:
LogEntry.create("Runtime", str(e), info={self.__class__.__name__: self.id})
raise e
def add_filepath(self, filepath, fp_id=None):
r"""Populates the DB tables for storing the filepath and connects the
`self` objects with this filepath"""
# Check that this function has been called from a subclass
self._check_subclass()
# Check if the connection handler has been provided. Create a new
# one if not.
conn_handler = SQLConnectionHandler()
fp_id = self._fp_id if fp_id is None else fp_id
try:
fpp_id = insert_filepaths([(filepath, fp_id)], None,
"templates", "filepath", conn_handler,
move_files=False)[0]
values = (self._id, fpp_id)
conn_handler.execute(
"INSERT INTO qiita.{0} ({1}, filepath_id) "
"VALUES (%s, %s)".format(
self._filepath_table, self._id_column), values)
except Exception as e:
LogEntry.create('Runtime', str(e),
info={self.__class__.__name__: self.id})
raise e
def add_filepath(self, filepath, conn_handler=None):
r"""Populates the DB tables for storing the filepath and connects the
`self` objects with this filepath"""
# Check that this function has been called from a subclass
self._check_subclass()
# Check if the connection handler has been provided. Create a new
# one if not.
conn_handler = conn_handler if conn_handler else SQLConnectionHandler()
if self._table == 'required_sample_info':
fp_id = convert_to_id("sample_template", "filepath_type",
conn_handler)
table = 'sample_template_filepath'
column = 'study_id'
elif self._table == 'common_prep_info':
fp_id = convert_to_id("prep_template", "filepath_type",
conn_handler)
table = 'prep_template_filepath'
column = 'prep_template_id'
else:
raise QiitaDBNotImplementedError(
'add_filepath for %s' % self._table)
try:
fpp_id = insert_filepaths([(filepath, fp_id)], None, "templates",
"filepath", conn_handler,
move_files=False)[0]
values = (self._id, fpp_id)
conn_handler.execute(
"INSERT INTO qiita.{0} ({1}, filepath_id) "
"VALUES (%s, %s)".format(table, column), values)
except Exception as e:
LogEntry.create('Runtime', str(e),
info={self.__class__.__name__: self.id})
raise e
if fp_type == 'directory':
# removing / from the path if it exists
to_tgz = fp[:-1] if fp[-1] == '/' else fp
break
if to_tgz is None:
continue
tgz = to_tgz + '.tgz'
if not exists(tgz):
with taropen(tgz, "w:gz") as tar:
tar.add(to_tgz, arcname=basename(to_tgz))
a_id = a.id
# Add the new tgz file to the artifact.
fp_ids = insert_filepaths([(tgz, tgz_id)], a_id, a.artifact_type,
move_files=False)
sql = """INSERT INTO qiita.artifact_filepath
(artifact_id, filepath_id)
VALUES (%s, %s)"""
sql_args = [[a_id, fp_id] for fp_id in fp_ids]
TRN.add(sql, sql_args, many=True)
TRN.execute()
#
# Generating compressed files for analysis
#
TRN.add("SELECT analysis_id FROM qiita.analysis")
for result in TRN.execute_fetchindex():
analysis_id = result[0]
# retrieving all analysis filepaths, we could have used
# Analysis.all_associated_filepath_ids but we could run into the
# removing / from the path if it exists
to_tgz = x['fp'][:-1] if x['fp'][-1] == '/' else x['fp']
break
if to_tgz is None:
continue
tgz = to_tgz + '.tgz'
if not exists(tgz):
with taropen(tgz, "w:gz") as tar:
tar.add(to_tgz, arcname=basename(to_tgz))
a_id = a.id
# Add the new tgz file to the artifact.
fp_ids = insert_filepaths([(tgz, tgz_id)],
a_id,
a.artifact_type,
move_files=False)
sql = """INSERT INTO qiita.artifact_filepath
(artifact_id, filepath_id)
VALUES (%s, %s)"""
sql_args = [[a_id, fp_id] for fp_id in fp_ids]
TRN.add(sql, sql_args, many=True)
TRN.execute()
#
# Generating compressed files for analysis
#
TRN.add("SELECT analysis_id FROM qiita.analysis")
for result in TRN.execute_fetchindex():
analysis_id = result[0]
# retrieving all analysis filepaths, we could have used