def create_new_core(self, dataset_id):
'''
Executes the Solr create_core command.
'''
logger.info('\n\nAttempt to create a Solr core.')
cmd = 'sudo -u solr /opt/solr/bin/solr create_core -c {d}'.format(
d=dataset_id)
run_shell_command(cmd)
def _download_file(self, url, output):
'''
Download a file located at `url` into a filepath given by the
`output` arg.
'''
download_cmd_template = 'wget {url} -O {output_file}'
try:
run_shell_command(
download_cmd_template.format(url=url, output_file=output))
except Exception as ex:
logger.info('Failed at downloading from {u}'.format(u=url))
raise ex
def check_if_container_running(container_id):
'''
Queries the status of a docker container to see if it is still running.
Returns True if running, False if exited.
'''
field = '.State.Status'
cmd = DOCKER_INSPECT_CMD.format(container_id=container_id, field=field)
logger.info('Inspect Docker container with: {cmd}'.format(cmd=cmd))
try:
stdout, stderr = run_shell_command(cmd)
except Exception as ex:
logger.error('Caught an exception when checking for running container.'
' This can be caused by a race condition if the timestamp on the'
' ExecutedOperation is not committed to the database before the second'
' request is issued. '
)
return False
stdout = stdout.decode('utf-8').strip()
if stdout == DOCKER_EXITED_FLAG:
return False
elif stdout == DOCKER_RUNNING_FLAG:
return True
else:
logger.info('Received a container status of: {status}'.format(
status=stdout
))
#TODO inform admins so we can track this case.
# returning True here (in this potential edge case) makes the container
# essentially permanant until we resolve its status.
return True
def check_image_exists(img_str):
logger.info('Check if {img} exists.'.format(img = img_str))
manifest_cmd = 'docker manifest inspect {img}'.format(img = img_str)
try:
stdout, stderr = run_shell_command(manifest_cmd)
logger.info('Successfully found Docker image')
return True
except Exception as ex:
logger.info('Docker image lookup failed.')
return False
def pull_image(remote_container_url):
'''
Provided with a fully qualifed docker image
url, pull the image to this machine.
'''
pull_cmd = 'docker pull {x}'.format(x=remote_container_url)
try:
stdout, stderr = run_shell_command(pull_cmd)
logger.info('Successfully pulled Docker image')
except Exception as ex:
logger.error('Docker pull failed.')
raise ex
def get_logs(container_id):
'''
Queries the logs from a given container
'''
log_cmd = 'docker logs {id}'.format(id=container_id)
logger.info('Query Docker logs with: {cmd}'.format(cmd=log_cmd))
try:
stdout, stderr = run_shell_command(log_cmd)
logger.info('Successfully queried container logs: {id}.'.format(id=container_id))
return stdout.decode('utf-8')
except Exception as ex:
logger.error('Query of container logs did not succeed.')
return ''
def get_timestamp_as_datetime(container_id, field):
cmd = DOCKER_INSPECT_CMD.format(container_id=container_id, field=field)
logger.info('Inspect Docker container with: {cmd}'.format(cmd=cmd))
stdout, stderr = run_shell_command(cmd)
# the timestamp by Docker is given like: b'"2020-09-28T17:51:52.393865325Z"\n'
# so we need to convert to a string (from bytes), and strip off the end-line
# and other stuff, like excessive microsends...
try:
time_str = stdout.decode('utf-8').strip()[:-2].split('.')[0]
t = datetime.datetime.strptime(time_str, '%Y-%m-%dT%H:%M:%S')
return t
except Exception as ex:
logger.error('Could not parse a timestamp from the Docker inspect command.'
' The timestamp string was: {s}. The raw string was: {r}'.format(
s=time_str,
r = stdout.decode('utf-8')
)
)
def check_container_exit_code(container_id):
'''
Queries the status of a docker container to see the exit code.
Note that running containers will give an exit code of zero, so this
should NOT be used to see if a container is still running.
'''
field = '.State.ExitCode'
cmd = DOCKER_INSPECT_CMD.format(container_id=container_id, field=field)
logger.info('Inspect Docker container with: {cmd}'.format(cmd=cmd))
stdout, stderr = run_shell_command(cmd)
logger.info('Results of inspect:\n\nSTDOUT: {stdout}\n\nSTDERR: {stderr}'.format(
stdout = stdout,
stderr = stderr
))
try:
exit_code = int(stdout)
return exit_code
except ValueError as ex:
logger.error('Received non-integer exit code from container: {id}'.format(
id=container_id
))
def index_example_file(self, dataset_id, filepath):
'''
Indexes a file into the core
'''
cmd = '/opt/solr/bin/post -c {d} {f}'.format(d=dataset_id, f=filepath)
run_shell_command(cmd)
def remove_container(container_id):
rm_cmd = 'docker rm {id}'.format(id=container_id)
logger.info('Remove Docker container with: {cmd}'.format(cmd=rm_cmd))
stdout, stderr = run_shell_command(rm_cmd)
logger.info('Successfully removed container: {id}.'.format(id=container_id))
def prepare(self):
'''
Handles prep of the dataset. Does NOT index!
'''
tmp_dir = self._create_tmp_dir()
ann_df = self._get_sample_annotations(tmp_dir)
pheno_df = self._get_phenotype_data(tmp_dir)
# Merge the sample-level table with the patient-level data
ann_df['subject_id'] = ann_df['SAMPID'].apply(
lambda x: '-'.join(x.split('-')[:2]))
# In the phenotypes file, sex is 2=F, 1=M
pheno_df['_SEX'] = pheno_df['SEX'].apply(lambda x: 'M'
if x == 1 else 'F')
merged_ann = pd.merge(ann_df,
pheno_df,
left_on='subject_id',
right_on='SUBJID')
# remap the column names and drop the others
merged_ann.rename(columns=self.COLUMN_MAPPING, inplace=True)
merged_ann = merged_ann[self.COLUMN_MAPPING.values()]
merged_ann = merged_ann.set_index('sample_id')
final_ann = pd.DataFrame()
counts_output_path = os.path.join(
self.ROOT_DIR,
self.COUNT_OUTPUT_FILE_TEMPLATE.format(tag=self.TAG,
date=self.date_str))
with pd.HDFStore(counts_output_path) as hdf_out:
for i, (tissue,
tissue_subdf) in enumerate(merged_ann.groupby('tissue')):
logger.info('Handling tissue {t}'.format(t=tissue))
try:
url = self.TISSUE_TO_FILE_MAP[tissue]
except KeyError as ex:
logger.info(
'No file exists in the map for {t}. Skipping.'.format(
t=tissue))
continue
output_file = '{d}/f{i}.gct.gz'.format(d=tmp_dir, i=i)
self._download_file(url, output_file)
run_shell_command('gunzip {f}'.format(f=output_file))
output_file = output_file[:-3]
# the GCT-format file has two header lines. The third line has the usual
# column headers
counts = pd.read_table(output_file,
sep='\t',
skiprows=2,
header=0,
index_col=1)
counts.drop(['Description'], axis=1, inplace=True)
counts.drop(['id'], axis=1, inplace=True)
# As of this writing, there are alternate ENSG Ids that are suffixed with _PAR_Y
# to denote features that are on the regions of chrY which are identical to those
# on chrX.
# https://www.gencodegenes.org/pages/faq.html (search "PAR_Y")
# We drop those here.
# It appears the mapping does not count to these regions anyway, since the rows are all
# zeros (while the canonical transcript is generally non-zero)
idx_par = pd.Series(
[x.endswith('_PAR_Y') for x in counts.index])
counts = counts.loc[~idx_par.values]
# Remove the version from the ENSG gene ID
counts.index = [x.split('.')[0] for x in counts.index]
samples_in_matrix = counts.columns
tissue_subdf = tissue_subdf.loc[samples_in_matrix]
final_ann = pd.concat([final_ann, tissue_subdf], axis=0)
group_id = RnaSeqMixin.create_python_compatible_id(
tissue) + '/ds'
hdf_out.put(group_id, counts)
final_ann.to_csv(os.path.join(
self.ROOT_DIR,
self.ANNOTATION_OUTPUT_FILE_TEMPLATE.format(tag=self.TAG,
date=self.date_str)),
sep=',',
index_label='sample_id')
def run(self, executed_op, op_data, validated_inputs):
logger.info('Running in local Docker mode.')
logger.info('Executed op type: %s' % type(executed_op))
logger.info('Executed op ID: %s' % str(executed_op.id))
logger.info('Op data: %s' % op_data)
logger.info(validated_inputs)
# the UUID identifying the execution of this operation:
execution_uuid = str(executed_op.id)
# get the operation dir so we can look at which converters and command to use:
op_dir = os.path.join(settings.OPERATION_LIBRARY_DIR,
str(op_data['id']))
# To avoid conflicts or corruption of user data, we run each operation in its
# own sandbox. We must first copy over their files to that sandbox dir.
execution_dir = os.path.join(settings.OPERATION_EXECUTION_DIR,
execution_uuid)
make_local_directory(execution_dir)
# convert the user inputs into args compatible with commandline usage:
# For instance, a differential gene expression requires one to specify
# the samples that are in each group-- to do this, the Operation requires
# two ObservationSet instances are submitted as arguments. The "translator"
# will take the ObservationSet data structures and turn them into something
# that the call with use- e.g. making a CSV list to submit as one of the args
# like:
# docker run <image> run_something.R -a sampleA,sampleB -b sampleC,sampleD
arg_dict = self._map_inputs(op_dir, validated_inputs, execution_dir)
logger.info('After mapping the user inputs, we have the'
' following structure: {d}'.format(d=arg_dict))
# Construct the command that will be run in the container:
entrypoint_file_path = os.path.join(op_dir, self.ENTRYPOINT_FILE)
if not os.path.exists(entrypoint_file_path):
logger.error(
'Could not find the required entrypoint file at {p}.'
' Something must have corrupted the operation directory.'.
format(p=entrypoint_file_path))
raise Exception('The repository must have been corrupted.'
' Failed to find the entrypoint file.'
' Check dir at: {d}'.format(d=op_dir))
entrypoint_cmd = self._get_entrypoint_command(entrypoint_file_path,
arg_dict)
image_str = get_image_name_and_tag(op_data['repo_name'],
op_data['git_hash'])
cmd = self.DOCKER_RUN_CMD.format(
container_name=execution_uuid,
execution_mount=settings.OPERATION_EXECUTION_DIR,
work_dir=settings.OPERATION_EXECUTION_DIR,
job_dir=execution_dir,
docker_image=image_str,
cmd=entrypoint_cmd)
try:
run_shell_command(cmd)
executed_op.job_id = execution_uuid
executed_op.save()
except Exception as ex:
logger.info('Failed when running shell command: {c}'.format(c=cmd))
logger.info('Exception was: {ex}'.format(ex=ex))
# if an exception is raised when issuing the Docker run
# command, then the job has failed. This error is likely
# not due to user error, but something with the issuing
# command or allocating appropriate Docker resources.
executed_op.job_failed = True
executed_op.execution_stop_datetime = datetime.datetime.now()
executed_op.status = ExecutedOperation.ADMIN_NOTIFIED
executed_op.save()
alert_admins(str(ex))
