def does_file_exist():
"""Adds localhost:8080/sftp/does_file_exist/
Check to see if a file exists
SFTP will throw an error if the directory doesn't exist
So we wrap this in a try/catch block
"""
request_data = json.loads(request.data.decode('utf-8'))
file = request_data.get('file')
dir = os.dirname(file)
ssh, sftp = initialize_ssh('gencore', 'dalma.abudhabi.nyu.edu')
results = {}
results['dir_exists'] = False
files = []
try:
files = sftp.listdir(dir)
results['dir_exists'] = True
except FileNotFoundError as e:
files = []
results['dir_exists'] = False
results['error'] = str(e)
if file in files:
results['file_exists'] = True
else:
results['file_exists'] = False
return jsonify({'results': results, 'context': {'endpoint': '/sftp/does_file_exist', 'args': request_data}})
def run_demultiplex_task(ds, **kwargs):
"""
Run the command that demultiplexes the run dir
:param ds:
:param kwargs:
:return:
"""
ssh, sftp = initialize_ssh('gencore', 'dalma.abudhabi.nyu.edu')
output = submit_demultiplex_job_to_slurm(ssh, sftp, kwargs['dag_run'])
slurm_job_id = parse_slurm_submission(output)
poll_slurm_job(ssh, slurm_job_id)
ssh.close()
def ensure_sample_sheet_exists_and_is_valid_csv(ds, **kwargs):
scratch_dir = kwargs['dag_run'].conf['scratch_dir']
sample_sheet = kwargs['dag_run'].conf['sample_sheet']
logger.info('Begin: Sample sheet check for {}'.format(scratch_dir))
ssh, sftp = initialize_ssh('gencore', 'dalma.abudhabi.nyu.edu')
logger.info('=============================================')
logger.info('Beginning Sample Sheet Job')
# This is already taken care of on the front end
# But we should run a check to make sure,
# incase of running directly through the REST API
run_dir_contents = ensure_run_dir_exists(sftp, scratch_dir)
csv_file = ensure_sample_sheet_exists_in_scratch_dir(sftp, sample_sheet)
ensure_valid_csv_file(sftp, scratch_dir, csv_file)
logger.info('Ending Sample Sheet Job')
logger.info('=============================================')
def get_qc_stable_workflows():
"""Adds localhost:8080/qc/get_qc_stable_workflows
This is simply a rest api that returns the workflows in /scratch/gencore/stable that match QC
"""
ssh, sftp = initialize_ssh('gencore', 'dalma.abudhabi.nyu.edu')
qc_workflows = []
dir = '/scratch/gencore/workflows/stable/'
for filename in sftp.listdir(dir):
if fnmatch.fnmatch(filename, "*QC*yml") or fnmatch.fnmatch(
filename, "*qc*yml"):
qc_workflows.append('{}{}'.format(dir, filename))
return jsonify({
'qc_workflows': qc_workflows,
'context': {
'endpoint': '/qc/get_qc_stable_workflows',
'args': {}
}
})