def zip_revision_number(self, fid):
"""
Look at previously supplied files and determine the
next revision number
"""
revision = None
bucket_name = self.publish_bucket
prefix = self.published_zip_folder + '/'
# Connect to S3 and bucket
s3_conn = S3Connection(self.settings.aws_access_key_id, self.settings.aws_secret_access_key)
bucket = s3_conn.lookup(bucket_name)
s3_key_names = s3lib.get_s3_key_names_from_bucket(
bucket = bucket,
prefix = prefix)
s3_key_name = s3lib.latest_pmc_zip_revision(fid, s3_key_names)
if s3_key_name:
# Found an existing PMC zip file, look for a revision number
revision_match = re.match(ur'.*r(.*)\.zip$', s3_key_name)
if revision_match is None:
# There is a zip but no revision number, use 1
revision = 1
else:
# Use the latest revision plus 1
revision = int(revision_match.group(1)) + 1
return revision
def download_pmc_zip_from_s3(self, doi_id, workflow):
"""
Simple download of PMC zip file from the live bucket
"""
bucket_name = self.pmc_zip_bucket
prefix = self.pmc_zip_folder
# Connect to S3 and bucket
s3_conn = S3Connection(self.settings.aws_access_key_id,
self.settings.aws_secret_access_key)
bucket = s3_conn.lookup(bucket_name)
s3_key_names = s3lib.get_s3_key_names_from_bucket(bucket=bucket,
prefix=prefix)
s3_key_name = s3lib.latest_pmc_zip_revision(doi_id, s3_key_names)
if s3_key_name:
# Download
s3_key = bucket.get_key(s3_key_name)
filename = s3_key_name.split("/")[-1]
filename_plus_path = (self.get_tmp_dir() + os.sep +
self.INPUT_DIR + os.sep + filename)
mode = "wb"
f = open(filename_plus_path, mode)
s3_key.get_contents_to_file(f)
f.close()
return True
else:
return False
def download_pmc_zip_from_s3(self, doi_id, workflow):
"""
Simple download of PMC zip file from the live bucket
"""
bucket_name = self.pmc_zip_bucket
prefix = self.pmc_zip_folder
# Connect to S3 and bucket
s3_conn = S3Connection(self.settings.aws_access_key_id, self.settings.aws_secret_access_key)
bucket = s3_conn.lookup(bucket_name)
s3_key_names = s3lib.get_s3_key_names_from_bucket(
bucket = bucket,
prefix = prefix)
s3_key_name = s3lib.latest_pmc_zip_revision(doi_id, s3_key_names)
if s3_key_name:
# Download
s3_key = bucket.get_key(s3_key_name)
filename = s3_key_name.split("/")[-1]
filename_plus_path = (self.get_tmp_dir() + os.sep +
self.INPUT_DIR + os.sep + filename)
mode = "wb"
f = open(filename_plus_path, mode)
s3_key.get_contents_to_file(f)
f.close()
return True
else:
return False
def zip_revision_number(self, fid):
"""
Look at previously supplied files and determine the
next revision number
"""
revision = None
bucket_name = self.publish_bucket
prefix = self.published_zip_folder + '/'
# Connect to S3 and bucket
s3_conn = S3Connection(self.settings.aws_access_key_id, self.settings.aws_secret_access_key)
bucket = s3_conn.lookup(bucket_name)
s3_key_names = s3lib.get_s3_key_names_from_bucket(
bucket=bucket,
prefix=prefix)
s3_key_name = s3lib.latest_pmc_zip_revision(fid, s3_key_names)
if s3_key_name:
# Found an existing PMC zip file, look for a revision number
revision_match = re.match(ur'.*r(.*)\.zip$', s3_key_name)
if revision_match is None:
# There is a zip but no revision number, use 1
revision = 1
else:
# Use the latest revision plus 1
revision = int(revision_match.group(1)) + 1
return revision
def does_source_zip_exist_from_s3(self, doi_id):
"""
"""
bucket_name = self.pmc_zip_bucket
prefix = self.pmc_zip_folder
# Connect to S3 and bucket
s3_conn = S3Connection(self.settings.aws_access_key_id,
self.settings.aws_secret_access_key)
bucket = s3_conn.lookup(bucket_name)
s3_key_names = s3lib.get_s3_key_names_from_bucket(bucket=bucket,
prefix=prefix)
s3_key_name = s3lib.latest_pmc_zip_revision(doi_id, s3_key_names)
if s3_key_name:
return True
else:
return False
def does_source_zip_exist_from_s3(self, doi_id):
"""
"""
bucket_name = self.pmc_zip_bucket
prefix = self.pmc_zip_folder
# Connect to S3 and bucket
s3_conn = S3Connection(self.settings.aws_access_key_id,
self.settings.aws_secret_access_key)
bucket = s3_conn.lookup(bucket_name)
s3_key_names = s3lib.get_s3_key_names_from_bucket(
bucket=bucket,
prefix=prefix)
s3_key_name = s3lib.latest_pmc_zip_revision(doi_id, s3_key_names)
if s3_key_name:
return True
else:
return False
def test_latest_pmc_zip_revision(self, doi_id, s3_key_names, expected_s3_key_name):
self.assertEqual(s3lib.latest_pmc_zip_revision(doi_id, s3_key_names), expected_s3_key_name)
