def test_exists_true(self):
""" Check for existence of object that exists """
uri = self.s3path + '/arealkey'
s3.upload(self.payload, uri)
self.assertTrue(s3.exists(uri))
s3.delete(uri)
def setUpClass(cls):
""" Put some input files up on S3 """
# create the bucket first
cls.s3 = s3.get_client()
cls.s3.create_bucket(Bucket=cls.bucket)
fouts = cls.create_files(cls.input_files)
for f in fouts:
s3.upload(f, os.path.join(cls.s3path, os.path.basename(f)))
def test_download(self):
""" Download file from S3 """
# first upload something
uri = self.s3path + '/file.txt'
s3.upload(self.payload, uri)
fout = s3.download(uri, path=self.path)
self.assertEqual(fout, os.path.join(self.path, 'file.txt'))
s3.delete(uri)
os.remove(fout)
def add_ancillary_file(self, local_file):
""" Add ancillary file"""
config = self.config
md5_sum = self._get_md5_sum(local_file)
md5_file = f'{local_file}.md5'
filename = os.path.split(md5_file)[1]
collection_string = (f'{config["collection"]["name"]}__' +
f'{config["collection"]["version"]}')
md5_key = (f's3://{config["buckets"]["internal"]["name"]}/staging/' +
f'{config["stack"]}/{collection_string}/{filename}')
self._write_md5sum_file(md5_file, md5_sum)
upload(md5_file, md5_key)
return md5_key
def test_upload(self):
""" Upload file to S3 then delete """
filename = os.path.basename(__file__)
s3_uri = os.path.join(self.s3path, filename)
uri = s3.upload(__file__, s3_uri)
self.assertEqual(uri, s3_uri)
s3.delete(s3_uri)
def upload_file_to_s3(self, filename, uri):
""" Upload a local file to s3 if collection payload provided """
try:
return s3.upload(filename, uri, extra={})
except Exception as e:
self.logger.error(
"{self.dmrpp_version}: Error uploading file %s: %s" %
(os.path.basename(os.path.basename(filename)), str(e)))
def upload_file(self, filename):
""" Upload a local file to s3 if collection payload provided """
info = self.get_publish_info(filename)
if info is None:
return filename
try:
return s3.upload(filename, info['s3'], extra={}) if info.get('s3', False) else None
except Exception as e:
self.logger.error("Error uploading file %s: %s" % (os.path.basename(os.path.basename(filename)), str(e)))
def upload_file(self, filename):
""" Upload a local file to s3 if collection payload provided """
warnings.warn(
'upload_file method is deprecated and will be removed in the next release. '
+ 'use upload functions in s3 module instead', DeprecationWarning)
info = self.get_publish_info(filename)
if info is None:
return filename
try:
return upload(filename, info['s3'], extra={}) if info.get(
's3', False) else None
except Exception as e:
self.logger.error(
"Error uploading file %s: %s" %
(os.path.basename(os.path.basename(filename)), str(e)))
def upload_files(files, bucket, prefix):
"""uploads list of local files to a given bucket and prefix
Arguments:
files: list of file paths
bucket: name of the bucket to upload the data to
prefix: the prefix key the appears before the filename
Returns:
returns a list of s3 uris e.g. s3://example-bucket/my/prefix/filename.txt
"""
list_of_uris = []
for f in files:
s3_uri = os.path.join('s3://', bucket, prefix, os.path.basename(f))
list_of_uris.append(upload(f, s3_uri))
return list_of_uris
def upload_file(self, filename):
""" Upload a local file to s3 if collection payload provided """
warnings.warn(
'upload_file method is deprecated and will be removed in the next release. ' +
'use upload functions in s3 module instead',
DeprecationWarning
)
info = self.get_publish_info(filename)
if info is None:
return filename
try:
uri = None
if info.get('s3', None) is not None:
bucket = self.buckets.get(info.get('bucket', 'public'), None)
bucketType = None
if bucket is not None:
bucketType = bucket['type']
extra = {'ACL': 'public-read'} if bucketType == 'public' else {}
uri = upload(filename, info['s3'], extra=extra)
return uri
except Exception as e:
self.logger.error("Error uploading file %s: %s" % (os.path.basename(os.path.basename(filename)), str(e)))
def _setUpClass(cls):
""" Put some input files up on S3 """
fouts = cls.create_files(cls.input_files)
for f in fouts:
s3.upload(f, cls.s3path)
os.remove(f)