def zip_revision_number(self, fid): """ Look at previously supplied files and determine the next revision number """ revision = None bucket_name = self.publish_bucket prefix = self.published_zip_folder + '/' # Connect to S3 and bucket s3_conn = S3Connection(self.settings.aws_access_key_id, self.settings.aws_secret_access_key) bucket = s3_conn.lookup(bucket_name) s3_key_names = s3lib.get_s3_key_names_from_bucket( bucket = bucket, prefix = prefix) s3_key_name = s3lib.latest_pmc_zip_revision(fid, s3_key_names) if s3_key_name: # Found an existing PMC zip file, look for a revision number revision_match = re.match(ur'.*r(.*)\.zip$', s3_key_name) if revision_match is None: # There is a zip but no revision number, use 1 revision = 1 else: # Use the latest revision plus 1 revision = int(revision_match.group(1)) + 1 return revision
def download_pmc_zip_from_s3(self, doi_id, workflow): """ Simple download of PMC zip file from the live bucket """ bucket_name = self.pmc_zip_bucket prefix = self.pmc_zip_folder # Connect to S3 and bucket s3_conn = S3Connection(self.settings.aws_access_key_id, self.settings.aws_secret_access_key) bucket = s3_conn.lookup(bucket_name) s3_key_names = s3lib.get_s3_key_names_from_bucket(bucket=bucket, prefix=prefix) s3_key_name = s3lib.latest_pmc_zip_revision(doi_id, s3_key_names) if s3_key_name: # Download s3_key = bucket.get_key(s3_key_name) filename = s3_key_name.split("/")[-1] filename_plus_path = (self.get_tmp_dir() + os.sep + self.INPUT_DIR + os.sep + filename) mode = "wb" f = open(filename_plus_path, mode) s3_key.get_contents_to_file(f) f.close() return True else: return False
def download_pmc_zip_from_s3(self, doi_id, workflow): """ Simple download of PMC zip file from the live bucket """ bucket_name = self.pmc_zip_bucket prefix = self.pmc_zip_folder # Connect to S3 and bucket s3_conn = S3Connection(self.settings.aws_access_key_id, self.settings.aws_secret_access_key) bucket = s3_conn.lookup(bucket_name) s3_key_names = s3lib.get_s3_key_names_from_bucket( bucket = bucket, prefix = prefix) s3_key_name = s3lib.latest_pmc_zip_revision(doi_id, s3_key_names) if s3_key_name: # Download s3_key = bucket.get_key(s3_key_name) filename = s3_key_name.split("/")[-1] filename_plus_path = (self.get_tmp_dir() + os.sep + self.INPUT_DIR + os.sep + filename) mode = "wb" f = open(filename_plus_path, mode) s3_key.get_contents_to_file(f) f.close() return True else: return False
def zip_revision_number(self, fid): """ Look at previously supplied files and determine the next revision number """ revision = None bucket_name = self.publish_bucket prefix = self.published_zip_folder + '/' # Connect to S3 and bucket s3_conn = S3Connection(self.settings.aws_access_key_id, self.settings.aws_secret_access_key) bucket = s3_conn.lookup(bucket_name) s3_key_names = s3lib.get_s3_key_names_from_bucket( bucket=bucket, prefix=prefix) s3_key_name = s3lib.latest_pmc_zip_revision(fid, s3_key_names) if s3_key_name: # Found an existing PMC zip file, look for a revision number revision_match = re.match(ur'.*r(.*)\.zip$', s3_key_name) if revision_match is None: # There is a zip but no revision number, use 1 revision = 1 else: # Use the latest revision plus 1 revision = int(revision_match.group(1)) + 1 return revision
def does_source_zip_exist_from_s3(self, doi_id): """ """ bucket_name = self.pmc_zip_bucket prefix = self.pmc_zip_folder # Connect to S3 and bucket s3_conn = S3Connection(self.settings.aws_access_key_id, self.settings.aws_secret_access_key) bucket = s3_conn.lookup(bucket_name) s3_key_names = s3lib.get_s3_key_names_from_bucket(bucket=bucket, prefix=prefix) s3_key_name = s3lib.latest_pmc_zip_revision(doi_id, s3_key_names) if s3_key_name: return True else: return False
def does_source_zip_exist_from_s3(self, doi_id): """ """ bucket_name = self.pmc_zip_bucket prefix = self.pmc_zip_folder # Connect to S3 and bucket s3_conn = S3Connection(self.settings.aws_access_key_id, self.settings.aws_secret_access_key) bucket = s3_conn.lookup(bucket_name) s3_key_names = s3lib.get_s3_key_names_from_bucket( bucket=bucket, prefix=prefix) s3_key_name = s3lib.latest_pmc_zip_revision(doi_id, s3_key_names) if s3_key_name: return True else: return False
def test_latest_pmc_zip_revision(self, doi_id, s3_key_names, expected_s3_key_name): self.assertEqual(s3lib.latest_pmc_zip_revision(doi_id, s3_key_names), expected_s3_key_name)