def handle(self, *args, **options): if options['file'] is None: logger.error("You must specify a file.") return "1" if 's3://' in options["file"]: bucket, key = parse_s3_url(options["file"]) s3 = boto3.resource('s3') try: filepath = "/tmp/input_" + str(uuid.uuid4()) + ".txt" s3.Bucket(bucket).download_file(key, filepath) except botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == "404": logger.error("The remote file does not exist.") raise else: raise else: filepath = options["file"] with open(filepath) as accession_file: for i, accession in enumerate(accession_file): if i < options["offset"]: continue accession = accession.strip() try: queue_surveyor_for_accession(accession) except Exception as e: logger.exception(e)
def handle(self, *args, **options): if options["file"] is None: logger.error("You must specify a file.") return "1" if "s3://" in options["file"]: bucket, key = parse_s3_url(options["file"]) s3 = boto3.resource("s3") try: filepath = "/tmp/input_" + str(uuid.uuid4()) + ".txt" s3.Bucket(bucket).download_file(key, filepath) except botocore.exceptions.ClientError as e: if e.response["Error"]["Code"] == "404": logger.error("The remote file does not exist.") raise else: raise else: filepath = options["file"] with open(filepath) as accession_file: for i, accession in enumerate(accession_file): if i < options["offset"]: continue accession = accession.strip() try: logger.info(f"Queuing surveyor job for {accession}.") queue_surveyor_for_accession(accession) # Sleep for 30 seconds so all surveyor jobs don't # start at the exact same time and overload the # database or ENA. time.sleep(30) except Exception as e: logger.exception(e)
def handle(self, *args, **options): if options["accession"] is None and options['file'] is None: logger.error("You must specify an experiment accession or file.") sys.exit(1) if not options["force"]: print( '-------------------------------------------------------------------------------' ) print( 'This will delete all objects in the database related to these accessions.' ' Are you sure you want to do this?') answer = input( 'You must type "yes", all other input will be ignored: ') if answer != "yes": print('Not unsurveying because confirmation was denied.') sys.exit(1) accessions = [] if options["file"]: if 's3://' in options["file"]: bucket, key = parse_s3_url(options["file"]) s3 = boto3.resource('s3') try: filepath = "/tmp/input_" + str(uuid.uuid4()) + ".txt" s3.Bucket(bucket).download_file(key, filepath) except botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == "404": logger.error("The remote file does not exist.") raise else: raise else: filepath = options["file"] with open(filepath) as file: for accession in file: accessions.append(accession.strip()) else: accessions.append(options['accession']) for accession in accessions: logger.info("Purging Experiment with accession: %s", accession) try: purge_experiment(accession) except Exception as e: logger.exception( "Exception caught while purging experiment with accession: %s", accession)
def handle(self, *args, **options): if options["file"] is None and options[ "accession"] is None and options["job_id"] is None: logger.error("You must specify an accession or file or job ID.") return "1" if options["file"]: if "s3://" in options["file"]: bucket, key = parse_s3_url(options["file"]) s3 = boto3.resource("s3") try: filepath = "/tmp/input_" + str(uuid.uuid4()) + ".txt" s3.Bucket(bucket).download_file(key, filepath) except botocore.exceptions.ClientError as e: if e.response["Error"]["Code"] == "404": logger.error("The remote file does not exist.") raise else: raise else: filepath = options["file"] with open(filepath) as accession_file: for i, accession in enumerate(accession_file): if i < options["offset"]: continue accession = accession.strip() try: run_surveyor_for_accession(accession) except Exception as e: logger.exception(e) if options["accession"]: accession = options["accession"] try: run_surveyor_for_accession(accession) except Exception as e: logger.exception(e) if options["job_id"]: job_id = options["job_id"] try: survey_job = SurveyJob.objects.get(id=job_id) surveyor.run_job(survey_job) except Exception as e: logger.exception(e)
def handle(self, *args, **options): okay = True if options["file"] is None: logger.error("You must specify a file to import metadata from") okay = False if options["source_name"] is None: logger.error("You must specify a source name") okay = False if options["methods_url"] is None: logger.error("You must specify a methods url") okay = False if not okay: sys.exit(1) if "s3://" in options["file"]: bucket, key = parse_s3_url(options["file"]) s3 = boto3.resource("s3") try: filepath = "/tmp/keyword_" + str(uuid.uuid4()) + ".json" s3.Bucket(bucket).download_file(key, filepath) except botocore.exceptions.ClientError as e: if e.response["Error"]["Code"] == "404": logger.error("The remote file does not exist.") raise else: raise else: filepath = options["file"] with open(filepath) as file: keywords = json.load(file) if type(keywords) != dict: logger.error( "The provided keywords file is not a dict with accession code keys" ) sys.exit(1) source, _ = Contribution.objects.get_or_create( source_name=options["source_name"], methods_url=options["methods_url"]) import_keywords(keywords, source)