def reset_test_db():
"""Resets the test database"""
with labman.db.sql_connection.TRN as TRN:
TRN.add("SELECT test FROM settings")
if not TRN.execute_fetchlast():
raise RuntimeError(
"Working on a production environment. Not executing "
"tests to protect the production database.")
# Reset the test database to enforce test independence at the class
# level. The client id and client secret are hardcoded because these
# should only be used in the test environment. If this fails, it would
# mean that the Qiita installation is not a test installation
client_id = '19ndkO3oMKsoChjVVWluF7QkxHRfYhTKSFbAVt8IhK7gZgDaO4'
client_secret = ('J7FfQ7CQdOxuKhQAf1eoGgBAE81Ns8Gu3EKaWFm3IO2JKh'
'AmmCWZuabe0O5Mp28s1')
qclient = QiitaClient(
"https://localhost:21174",
client_id,
client_secret,
server_cert=labman.db.settings.labman_settings.qiita_server_cert)
qclient.post("/apitest/reset/")
# The above call resets the qiita schema. Qiita does not create the
# labman structures, so create them here
path_builder = partial(join, dirname(__file__), 'support_files')
db_patch = path_builder('db_patch.sql')
db_patch_manual = path_builder('db_patch_manual.sql')
db_test = path_builder('populate_test_db.sql')
with TRN:
with open(db_patch, 'r') as f:
TRN.add(f.read())
with open(db_patch_manual, 'r') as f:
TRN.add(f.read())
with open(db_test, 'r') as f:
TRN.add(f.read())
TRN.execute()
def setUp(self):
# Register the URIs for the QiitaClient
httpretty.register_uri(
httpretty.POST,
"https://test_server.com/qiita_db/authenticate/",
body='{"access_token": "token", "token_type": "Bearer", '
'"expires_in": "3600"}')
self.qclient = QiitaClient('https://test_server.com', 'client_id',
'client_secret')
# Create a biom table
fd, self.biom_fp = mkstemp(suffix=".biom")
close(fd)
data = np.asarray([[0, 0, 1], [1, 3, 42]])
table = Table(data, ['O1', 'O2'], ['1.S1', '1.S2', '1.S3'])
with biom_open(self.biom_fp, 'w') as f:
table.to_hdf5(f, "Test")
self.out_dir = mkdtemp()
self.parameters = {
'template': 1,
'files': '{"BIOM": ["%s"]}' % self.biom_fp,
'artifact_type': 'BIOM'
}
self._clean_up_files = [self.biom_fp, self.out_dir]
def setUp(self):
# Registewr the URIs for the QiitaClient
httpretty.register_uri(
httpretty.POST,
"https://test_server.com/qiita_db/authenticate/",
body='{"access_token": "token", "token_type": "Bearer", '
'"expires_in": "3600"}')
self.qclient = QiitaClient('https://test_server.com', 'client_id',
'client_secret')
# creating files
self.out_dir = mkdtemp()
gz_file = join(self.out_dir, "file1.fastq.gz")
with GzipFile(gz_file, mode='w', mtime=1) as fh:
fh.write(READS)
fastq_file = join(self.out_dir, "file1.fastq")
with open(fastq_file, mode='w') as fh:
fh.write(READS)
self.filepaths = (
'[["%s", "raw_forward_seqs"], ["%s", "raw_barcodes"]]' % (
gz_file, fastq_file))
self.artifact_id = 4
self.parameters = {'input_data': self.artifact_id}
self._clean_up_files = [self.out_dir]
def setUp(self):
httpretty.register_uri(
httpretty.POST,
"https://test_server.com/qiita_db/authenticate/",
body='{"access_token": "token", "token_type": "Bearer", '
'"expires_in": "3600"}')
self.qclient = QiitaClient("https://test_server.com", 'client_id',
'client_secret')
def setUpClass(cls):
cls.client_id = '19ndkO3oMKsoChjVVWluF7QkxHRfYhTKSFbAVt8IhK7gZgDaO4'
cls.client_secret = ('J7FfQ7CQdOxuKhQAf1eoGgBAE81Ns8Gu3EKaWFm3IO2JKh'
'AmmCWZuabe0O5Mp28s1')
cls.server_cert = environ.get('QIITA_SERVER_CERT', None)
cls.qclient = QiitaClient("https://localhost:21174",
cls.client_id,
cls.client_secret,
server_cert=cls.server_cert)
cls.qclient.post('/apitest/reload_plugins/')
# Give enough time for the plugins to register
sleep(5)
def setUp(self):
# Register the URIs for the QiitaClient
httpretty.register_uri(
httpretty.POST,
"https://test_server.com/qiita_db/authenticate/",
body='{"access_token": "token", "token_type": "Bearer", '
'"expires_in": "3600"}')
self.qclient = QiitaClient('https://test_server.com', 'client_id',
'client_secret')
self._clean_up_files = []
def execute_job(url, job_id, output_dir):
# Set up the QiitaClient - Probably another function?
dflt_conf_fp = join(dirname(abspath(__file__)), 'support_files',
'config_file.cfg')
conf_fp = environ.get('QP_QIIME2_CONFIG_FP', dflt_conf_fp)
config = ConfigParser()
with open(conf_fp, 'U') as conf_file:
config.readfp(conf_file)
qclient = QiitaClient(url, config.get('main', 'CLIENT_ID'),
config.get('main', 'CLIENT_SECRET'),
server_cert=config.get('main', 'SERVER_CERT'))
# Request job information. If there is a problem retrieving the job
# information, the QiitaClient already raises an error
job_info = qclient.get_job_info(job_id)
# Starting the heartbeat
qclient.start_heartbeat(job_id)
try:
success, artifacts_info, error_msg = execute_task(qclient, job_id,
job_info)
except Exception:
exc_str = ''.join(traceback.format_exception(*sys.exc_info()))
error_msg = ("Error executing %s:\n%s" % (job_info['command'],
exc_str))
success = False
artifacts_info = None
# The job completed
qclient.complete_job(job_id, success, error_msg=error_msg,
artifacts_info=artifacts_info)
print(error_msg)
def execute_job(server_url, job_id, output_dir):
"""Starts the plugin and executes the assigned task
Parameters
----------
server_url : str
The url of the server
job_id : str
The job id
Raises
------
RuntimeError
If there is a problem gathering the job information
"""
# Set up the Qiita Client
dflt_conf_fp = join(dirname(abspath(__file__)), 'support_files',
'config_file.cfg')
conf_fp = environ.get('QP_BIOM_TYPE_CONFIG_FP', dflt_conf_fp)
config = ConfigParser()
with open(conf_fp, 'U') as conf_file:
config.readfp(conf_file)
qclient = QiitaClient(server_url,
config.get('main', 'CLIENT_ID'),
config.get('main', 'CLIENT_SECRET'),
server_cert=config.get('main', 'SERVER_CERT'))
# Request job information. If there is a problem retrieving the job
# information, the QiitaClient already raises an error
job_info = qclient.get_job_info(job_id)
# Starting the heartbeat
qclient.start_heartbeat(job_id)
# Execute the given task
task_name = job_info['command']
task = TASK_DICT[task_name]
if not exists(output_dir):
makedirs(output_dir)
try:
success, artifacts_info, error_msg = task(qclient, job_id,
job_info['parameters'],
output_dir)
except Exception:
exc_str = repr(traceback.format_exception(*sys.exc_info()))
error_msg = ("Error executing %s: \n%s" % (task_name, exc_str))
success = False
artifacts_info = None
# The job completed
qclient.complete_job(job_id,
success,
error_msg=error_msg,
artifacts_info=artifacts_info)
def get_qiita_client():
if config.debug:
class _mock:
def get(self, *args, **kwargs):
return {'categories': AG_DEBUG_OBSERVED_CATEGORIES}
def http_patch(self, *args, **kwargs):
return 'okay'
qclient = _mock()
else:
# interface for making HTTP requests against Qiita
qclient = QiitaClient(':'.join([config.qiita_host, config.qiita_port]),
config.qiita_client_id,
config.qiita_client_secret,
config.qiita_certificate)
# we are monkeypatching to use qclient's internal machinery
# and to fix the broken HTTP patch
qclient.http_patch = functools.partial(qclient._request_retry,
requests.patch)
return qclient
def __call__(self, server_url, job_id, output_dir):
"""Runs the plugin and executed the assigned task
Parameters
----------
server_url : str
The url of the server
job_id : str
The job id
output_dir : str
The output directory
Raises
------
RuntimeError
If there is a problem gathering the job information
"""
# Set up the Qiita Client
config = ConfigParser()
with open(self.conf_fp, 'U') as conf_file:
config.readfp(conf_file)
qclient = QiitaClient(server_url, config.get('oauth2', 'CLIENT_ID'),
config.get('oauth2', 'CLIENT_SECRET'),
server_cert=config.get('oauth2', 'SERVER_CERT'))
if job_id == 'register':
self._register(qclient)
else:
# Request job information. If there is a problem retrieving the job
# information, the QiitaClient already raises an error
job_info = qclient.get_job_info(job_id)
# Starting the heartbeat
qclient.start_heartbeat(job_id)
# Execute the given task
task_name = job_info['command']
task = self.task_dict[task_name]
if not exists(output_dir):
makedirs(output_dir)
try:
success, artifacts_info, error_msg = task(
qclient, job_id, job_info['parameters'], output_dir)
except Exception:
exc_str = repr(traceback.format_exception(*sys.exc_info()))
error_msg = ("Error executing %s:\n%s" % (task_name, exc_str))
success = False
artifacts_info = None
# The job completed
qclient.complete_job(job_id, success, error_msg=error_msg,
artifacts_info=artifacts_info)
def client_connect(url):
name = plugin_details['name']
version = plugin_details['version']
config = ConfigParser()
conf_dir = environ.get('QIITA_PLUGINS_DIR',
join(expanduser('~'), '.qiita_plugins'))
conf_fp = join(conf_dir, f'{name}_{version}.conf')
with open(conf_fp, 'U') as conf_file:
config.readfp(conf_file)
qclient = QiitaClient(url,
config.get('oauth2', 'CLIENT_ID'),
config.get('oauth2', 'CLIENT_SECRET'),
server_cert=config.get('oauth2', 'SERVER_CERT'))
return qclient
def execute_job(server_url, job_id, output_dir):
"""Starts the plugin and executes the assigned task
Parameters
----------
server_url : str
The url of the server
job_id : str
The job id
output_dir : str
The output directory
Raises
------
RuntimeError
If there is a problem gathering the job information
"""
# Set up the Qiita Client
try:
conf_fp = environ['QP_TARGET_GENE_TYPE_CONFIG_FP']
except KeyError:
conf_fp = join(dirname(abspath(__file__)), 'support_files',
'config_file.cfg')
config = ConfigParser()
with open(conf_fp, 'U') as conf_file:
config.readfp(conf_file)
qclient = QiitaClient(server_url, config.get('main', 'CLIENT_ID'),
config.get('main', 'CLIENT_SECRET'),
server_cert=config.get('main', 'SERVER_CERT'))
# Request job information. If there is a problem retrieving the job
# information, the QiitaClient already raises an error
job_info = qclient.get_job_info(job_id)
# Starting the heartbeat
qclient.start_heartbeat(job_id)
# Execute the given task
task_name = job_info['command']
task = TASK_DICT[task_name]
if not exists(output_dir):
makedirs(output_dir)
try:
success, artifacts_info, error_msg = task(
qclient, job_id, job_info['parameters'], output_dir)
except Exception:
exc_str = repr(traceback.format_exception(*sys.exc_info()))
error_msg = ("Error executing %s: \n%s" % (task_name, exc_str))
success = False
artifacts_info = None
# The job completed
qclient.complete_job(job_id, success, error_msg=error_msg,
artifacts_info=artifacts_info)
def __call__(self, server_url, job_id, output_dir):
"""Runs the plugin and executed the assigned task
Parameters
----------
server_url : str
The url of the server
job_id : str
The job id
output_dir : str
The output directory
Raises
------
RuntimeError
If there is a problem gathering the job information
"""
logger.debug('Entered BaseQiitaPlugin.__call__()')
# Set up the Qiita Client
config = ConfigParser()
with open(self.conf_fp, 'U') as conf_file:
config.readfp(conf_file)
qclient = QiitaClient(server_url, config.get('oauth2', 'CLIENT_ID'),
config.get('oauth2', 'CLIENT_SECRET'),
server_cert=config.get('oauth2', 'SERVER_CERT'))
if job_id == 'register':
self._register(qclient)
else:
# Request job information. If there is a problem retrieving the job
# information, the QiitaClient already raises an error
job_info = qclient.get_job_info(job_id)
# Starting the heartbeat
qclient.start_heartbeat(job_id)
# Execute the given task
task_name = job_info['command']
task = self.task_dict[task_name]
if not exists(output_dir):
makedirs(output_dir)
try:
success, artifacts_info, error_msg = task(
qclient, job_id, job_info['parameters'], output_dir)
except Exception:
exc_str = repr(traceback.format_exception(*sys.exc_info()))
error_msg = ("Error executing %s:\n%s" % (task_name, exc_str))
success = False
artifacts_info = None
# The job completed
qclient.complete_job(job_id, success, error_msg=error_msg,
artifacts_info=artifacts_info)
def execute_job(server_url, job_id, output_dir):
"""Starts the plugin and executes the assigned task
Parameters
----------
server_url : str
The url of the server
job_id : str
The job id
Raises
------
RuntimeError
If there is a problem gathering the job information
"""
# Set up the Qiita Client
dflt_conf_fp = join(dirname(abspath(__file__)), 'support_files',
'config_file.cfg')
conf_fp = environ.get('QP_BIOM_TYPE_CONFIG_FP', dflt_conf_fp)
config = ConfigParser()
with open(conf_fp, 'U') as conf_file:
config.readfp(conf_file)
qclient = QiitaClient(server_url, config.get('main', 'CLIENT_ID'),
config.get('main', 'CLIENT_SECRET'),
server_cert=config.get('main', 'SERVER_CERT'))
# Request job information
job_info = qclient.get_job_info(job_id)
# Check if we have received the job information so we can start it
if job_info and job_info['success']:
# Starting the heartbeat
qclient.start_heartbeat(job_id)
# Execute the given task
task_name = job_info['command']
task = TASK_DICT[task_name]
if not exists(output_dir):
makedirs(output_dir)
try:
payload = task(qclient, job_id, job_info['parameters'], output_dir)
except Exception:
exc_str = repr(traceback.format_exception(*sys.exc_info()))
error_msg = ("Error executing %s: \n%s" % (task_name, exc_str))
payload = format_payload(False, error_msg=error_msg)
# The job completed
qclient.complete_job(job_id, payload)
else:
raise RuntimeError("Can't get job (%s) information" % job_id)
def setUp(self):
# Registewr the URIs for the QiitaClient
httpretty.register_uri(
httpretty.POST,
"https://test_server.com/qiita_db/authenticate/",
body='{"access_token": "token", "token_type": "Bearer", '
'"expires_in": "3600"}')
self.qclient = QiitaClient('https://test_server.com', 'client_id',
'client_secret')
# creating files
self.out_dir = mkdtemp()
self.filepaths = (
'[["test_file", "raw_forward_seqs"], '
'["%s/test_data/101_seqs.demux", "preprocessed_demux"]]' %
dirname(__file__))
self.artifact_id = 4
self.parameters = {'input_data': self.artifact_id}
self._clean_up_files = [self.out_dir]
from microsetta_private_api.config_manager import SERVER_CONFIG
from qiita_client import QiitaClient, NotFoundError
if not SERVER_CONFIG['qiita_endpoint']:
class _mock:
def get(self, *args, **kwargs):
raise NotFoundError("No qiita client")
def post(self, *args, **kwargs):
raise NotFoundError("No qiita client")
def http_patch(self, *args, **kwargs):
raise NotFoundError("No qiita client")
qclient = _mock()
else:
qclient = QiitaClient(
SERVER_CONFIG["qiita_endpoint"],
SERVER_CONFIG["qiita_client_id"],
SERVER_CONFIG["qiita_client_secret"]
)