def download(ctx, destination, legacy, query): """Download files matching a given query""" results = query_hci(query, ctx["index"], ctx["password"]) if legacy: for item in results: itm = item["metadata"] samples = itm["samples_Fastq_paths"] for i in samples: obj = ctx["hcpm"].get_object(i) # Get object with json. if obj is not None: ctx["hcpm"].download_file( obj, f"{destination}/{os.path.basename(i)}") # Downloads file. else: log.error( f"File: '{s}' does not exist in bucket '{bucket}' on the HCP" ) elif not legacy: for item in results: itm = item["metadata"] samples = itm["samples_Fastq_paths"] string = "".join(samples).strip("[]").strip("{]}'") lst = string.replace('"', '').replace("\\", "").replace( "[", "").replace("]", "").replace(";", ",").split(",") for i in lst: if query in os.path.basename(i) or query in i: s = os.path.basename(i) log.info("downloading:", s.replace(".fastq.gz", ".fasterq").strip()) name = s.replace(".fastq.gz", ".fasterq").strip() # Replace suffix. obj = ctx["hcpm"].get_object(name) # Get object with json. if obj is not None: ctx["hcpm"].download_file( obj, f"{destination}/{os.path.basename(name)}" ) # Downloads file. else: log.error( f"File: '{name}' does not exist in bucket '{bucket}' on the HCP" )
def download(ctx, query, output, fast, silent): """Download files using a given query""" #Defaults output to input name if output == "": output = os.path.basename(query) #If output is folder. Default file name to input name elif output[-1] in ["/", "\\"]: output = os.path.join(output, os.path.basename(query)) if not fast: found_objs = ctx['hcpm'].search_objects(query) if len(found_objs) == 0: log.info( f"File: {query} does not exist on {ctx['hcpm'].bucket.name}") elif len(found_objs) > 1: for obj in found_objs: log.info(f"Found {len(found_obj)} files matching query") log.info(f"Download {obj}? [Y/N]") sys.stdout.write( f"[--] Do you wish to download {obj.key} on {ctx['hcpm'].bucket.name}? [Y/N]?\n" ) sys.stdout.flush() answer = sys.stdin.readline() if answer[0].lower() == "y": obj = ctx['hcpm'].get_object(query) # Get object with key. #Default output name to key if output == "": output = obj.key #If output is folder. Default file name to obj.key elif output[-1] in ["/", "\\"]: output = os.path.join(output, obj.key) if silent: ctx['hcpm'].download_file( obj, output, force=True, callback=False) # Downloads file. else: ctx['hcpm'].download_file( obj, output, force=True) # Downloads file. #log.info(f"Downloaded {obj.key}" elif len(found_objs) == 1: obj = ctx['hcpm'].get_object(query) # Get object with key. if silent: ctx['hcpm'].download_file(obj, output, force=True, callback=False) # Downloads file. else: ctx['hcpm'].download_file(obj, output, force=True) # Downloads file. elif fast: obj = ctx['hcpm'].get_object(query) # Get object with key. if silent: ctx['hcpm'].download_file(obj, output, force=True, callback=False) # Downloads file. else: ctx['hcpm'].download_file(obj, output, force=True) # Downloads file.
def search(ctx, query, mode): """List all file hits for a given query""" if query != "": found_objs = ctx['hcpm'].search_objects(query, mode=mode) if len(found_objs) > 0: for obj in found_objs: log.info(obj.key) else: log.info(f'No results found for: {query}') else: log.info( 'A query or file needs to be specified if you are using the "search" option' )
def search(ctx, query): """Displays file hits for a given query""" try: results = query_hci(query, ctx["index"], ctx["password"]) for item in results: itm = item["metadata"] meta = itm["HCI_displayName"] samples = itm["samples_Fastq_paths"] string = "".join(samples).strip("[]").strip("{]}'") lst = string.replace('"', '').replace("\\", "").replace( "[", "").replace("]", "").replace(";", ",").split(",") log.info(f"Metadata file: {meta}") for i in lst: if query in i or query in os.path.basename(i): log.info("check: ", i) name = i.replace(".fastq.gz", ".fasterq").strip() # Replace suffix. except: log.info(f"File(s) does not exists: {query}")
def delete(ctx, query, force): """Delete a file on the HCP""" objs = ctx['hcpm'].search_objects(query) # Get object with query if len(objs) < 1: log.info(f"File: {query} does not exist on {ctx['hcpm'].bucket.name}") else: hits = list() for obj in objs: hits.append(obj.key) log.info(f"Found {len(objs)} entries matching query '{query}':") log.info(f"{hits}") for obj in objs: if not force: sys.stdout.write(f"You are about to delete the file {obj.key} " \ f"on {ctx['hcpm'].bucket.name}, are you sure? [Y/N/Q]?\n") sys.stdout.flush() answer = sys.stdin.readline() if answer[0].lower() == "y": ctx['hcpm'].delete_object(obj) # Delete file. time.sleep(2) log.info(f"Deleted file {obj.key} ") elif answer[0].lower() == "q": break else: log.info(f"Skipped deleting {obj.key} ") elif force: ctx['hcpm'].delete_object(obj) # Delete file. time.sleep(2) log.info(f"Deleted file {obj.key} ")
def upload(ctx, input, output, tag, meta, silent, atypical): """Upload fastq files / fastq folder structure""" file_lst = [] #Defaults output to input name if output == "": output = os.path.basename(input) #If output is folder. Default file name to input name elif output[-1] in ["/", "\\"]: output = os.path.join(output, os.path.basename(input)) dstfld = Path(output) dstfld = dstfld.parent if dstfld.parts == (): dstfld = "" if os.path.isdir(input): #Recursively loop over all folders for root, dirs, files in os.walk(folder): for f in files: try: if not atypical: preproc.verify_fq_suffix(os.path.join(root, f)) preproc.verify_fq_content(os.path.join(root, f)) if meta != "": preproc.generate_tagmap(os.path.join(root, f), tag, meta) file_lst.append(os.path.join(root, f)) except Exception as e: log.warning(f"{f} is not a valid upload file: {e}") else: input = os.path.abspath(input) try: if not atypical: preproc.verify_fq_suffix(input) preproc.verify_fq_content(input) if meta != "": preproc.generate_tagmap(input, tag, meta) file_lst.append(input) except Exception as e: log.warning(f"{input} is not a valid upload file: {e}") sys.exit(-1) if file_lst == []: log.error( f"{input} could not be uploaded to NGPr. Try using an atypical upload" ) for file_pg in file_lst: if silent: ctx['hcpm'].upload_file(file_pg, output, callback=False) else: ctx['hcpm'].upload_file(file_pg, output) #time.sleep(2) log.info(f"Uploaded: {file_pg}") if meta != "": meta_fn = Path(meta).name # Uploads associated json files. if silent: ctx['hcpm'].upload_file(meta, os.path.join(dstfld, meta_fn), callback=False) else: ctx['hcpm'].upload_file(meta, os.path.join(dstfld, meta_fn))