def does_file_exist(): """Adds localhost:8080/sftp/does_file_exist/ Check to see if a file exists SFTP will throw an error if the directory doesn't exist So we wrap this in a try/catch block """ request_data = json.loads(request.data.decode('utf-8')) file = request_data.get('file') dir = os.dirname(file) ssh, sftp = initialize_ssh('gencore', 'dalma.abudhabi.nyu.edu') results = {} results['dir_exists'] = False files = [] try: files = sftp.listdir(dir) results['dir_exists'] = True except FileNotFoundError as e: files = [] results['dir_exists'] = False results['error'] = str(e) if file in files: results['file_exists'] = True else: results['file_exists'] = False return jsonify({'results': results, 'context': {'endpoint': '/sftp/does_file_exist', 'args': request_data}})
def run_demultiplex_task(ds, **kwargs): """ Run the command that demultiplexes the run dir :param ds: :param kwargs: :return: """ ssh, sftp = initialize_ssh('gencore', 'dalma.abudhabi.nyu.edu') output = submit_demultiplex_job_to_slurm(ssh, sftp, kwargs['dag_run']) slurm_job_id = parse_slurm_submission(output) poll_slurm_job(ssh, slurm_job_id) ssh.close()
def ensure_sample_sheet_exists_and_is_valid_csv(ds, **kwargs): scratch_dir = kwargs['dag_run'].conf['scratch_dir'] sample_sheet = kwargs['dag_run'].conf['sample_sheet'] logger.info('Begin: Sample sheet check for {}'.format(scratch_dir)) ssh, sftp = initialize_ssh('gencore', 'dalma.abudhabi.nyu.edu') logger.info('=============================================') logger.info('Beginning Sample Sheet Job') # This is already taken care of on the front end # But we should run a check to make sure, # incase of running directly through the REST API run_dir_contents = ensure_run_dir_exists(sftp, scratch_dir) csv_file = ensure_sample_sheet_exists_in_scratch_dir(sftp, sample_sheet) ensure_valid_csv_file(sftp, scratch_dir, csv_file) logger.info('Ending Sample Sheet Job') logger.info('=============================================')
def get_qc_stable_workflows(): """Adds localhost:8080/qc/get_qc_stable_workflows This is simply a rest api that returns the workflows in /scratch/gencore/stable that match QC """ ssh, sftp = initialize_ssh('gencore', 'dalma.abudhabi.nyu.edu') qc_workflows = [] dir = '/scratch/gencore/workflows/stable/' for filename in sftp.listdir(dir): if fnmatch.fnmatch(filename, "*QC*yml") or fnmatch.fnmatch( filename, "*qc*yml"): qc_workflows.append('{}{}'.format(dir, filename)) return jsonify({ 'qc_workflows': qc_workflows, 'context': { 'endpoint': '/qc/get_qc_stable_workflows', 'args': {} } })