def uploadToS3():
key = input("Enter the AWSKey:")
secretkey = input("Enter the Secret Key:")
awsKey = key #
awsSecret = secretkey #
bucname = input("Enter the bucket name:")
conn = S3Connection(awsKey, awsSecret)
#print(conn)
print("inside Upload function..")
#Connecting to a bucket
bucket_name = bucname #"luigibuckets"
bucket = conn.get_bucket(bucket_name)
print(bucket)
#Setting the keys
k = Key(bucket)
print(k)
k.key = "XGB_File_400trees_TrainDatafinal.csv"
k.set_contents_from_filename("XGB_File_400trees_TrainDatafinal.csv")
k2 = Key(bucket)
k2.key = "documents_meta.csv"
k2.set_contents_from_filename("documents_meta.csv")
k3 = Key(bucket)
k3.key = "xgb_results.csv"
k3.set_contents_from_filename("xgb_results.csv")
k4 = Key(bucket)
k4.key = "xgboost.pkl"
k4.set_contents_from_filename("xgboost.pkl")
print('Upload Completed..')
def delete_img_aws(instance, **kwargs):
conn = S3Connection(settings.AWS_ACCESS_KEY_ID,
settings.AWS_SECRET_ACCESS_KEY)
b = Bucket(conn, settings.AWS_STORAGE_BUCKET_NAME)
img_k = Key(b)
img_thumb_k = Key(b)
img_k.key = instance.image.name
img_thumb_k.key = instance.image_thumb.name
b.delete_key(img_k)
b.delete_key(img_thumb_k)
def test_s3(self):
title = "Dino Test"
link = "http://example.com"
description = "My dino test"
base_url = "http://example.com/items/"
aws_key = os.environ['MYDINOSAUR_AWS_ACCESS_KEY']
aws_secret = os.environ['MYDINOSAUR_AWS_SECRET_KEY']
s3_bucket = os.environ['MYDINOSAUR_S3_BUCKET']
dino = mydinosaur.MyS3Dinosaur(':memory:',
title=title,
link=link,
description=description,
base_url=base_url,
aws_access_key=aws_key,
aws_secret_key=aws_secret,
s3_bucket=s3_bucket)
dino.update("hello there")
# make sure that stuff got uploaded
conn = S3Connection(aws_key, aws_secret)
bucket = conn.get_bucket(s3_bucket)
k = Key(bucket)
k.key = 'rss.xml'
feed = feedparser.parse(k.get_contents_as_string())
self.assertEqual(len(feed.entries), 1)
k = Key(bucket)
k.key = '1.html'
html = k.get_contents_as_string()
self.assertIn('<!-- MyDinosaur default html template -->', html)
# now, update with media!
filehandle = StringIO("Hello there.\n")
dino.update_with_media('this is a test',
filehandle,
media_type="text/plain",
ext="txt")
k = Key(bucket)
k.key = 'rss.xml'
feed = feedparser.parse(k.get_contents_as_string())
self.assertEqual(len(feed.entries), 2)
# ensure that media was uploaded
media_url = feed.entries[0].enclosures[0].url
key_name = urlparse(media_url).path.split('/')[-1]
k = Key(bucket)
k.key = key_name
contents = k.get_contents_as_string()
self.assertEqual(contents, "Hello there.\n")
def test_upload_and_download_with_encryption(tmpdir):
from toil_lib.urls import s3am_upload
from toil_lib.urls import download_url
from boto.s3.connection import S3Connection, Bucket, Key
work_dir = str(tmpdir)
# Create temporary encryption key
key_path = os.path.join(work_dir, 'foo.key')
subprocess.check_call([
'dd', 'if=/dev/urandom', 'bs=1', 'count=32', 'of={}'.format(key_path)
])
# Create test file
upload_fpath = os.path.join(work_dir, 'upload_file')
with open(upload_fpath, 'wb') as fout:
fout.write(os.urandom(1024))
# Upload file
random_key = os.path.join('test/', str(uuid4()), 'upload_file')
s3_url = os.path.join('s3://cgl-driver-projects/', random_key)
try:
s3_dir = os.path.split(s3_url)[0]
s3am_upload(fpath=upload_fpath, s3_dir=s3_dir, s3_key_path=key_path)
# Download the file
download_url(url=s3_url,
name='download_file',
work_dir=work_dir,
s3_key_path=key_path)
download_fpath = os.path.join(work_dir, 'download_file')
assert os.path.exists(download_fpath)
assert filecmp.cmp(upload_fpath, download_fpath)
finally:
# Delete the Key. Key deletion never fails so we don't need to catch any exceptions
with closing(S3Connection()) as conn:
b = Bucket(conn, 'cgl-driver-projects')
k = Key(b)
k.key = random_key
k.delete()
def upload_to_s3(fp, name):
conn = _get_s3_connection()
bucket = conn.create_bucket('muxlist')
k = Key(bucket)
k.key = name
k.set_contents_from_file(fp)
return 'http://muxlist.s3.amazonaws.com/%s' % name
def download_file_s3(aws_path, aws_config, local_folder=DATA_PATH):
""" Download a file from an S3 bucket and save it in the local folder. """
# remove the prefix and extract the S3 bucket, folder, and file name
m = re.match(S3_PREFIX, aws_path)
split = aws_path[len(m.group()):].split('/')
s3_bucket = split.pop(0)
s3_folder = '/'.join(split[:-1])
keyname = split[-1]
# create the local folder if necessary
if local_folder is not None:
ensure_directory(local_folder)
path = os.path.join(local_folder, keyname)
else:
path = keyname
if os.path.isfile(path):
print 'file %s already exists!' % path
return path
conn = S3Connection(aws_config.access_key, aws_config.secret_key)
bucket = conn.get_bucket(s3_bucket)
if s3_folder:
aws_keyname = os.path.join(s3_folder, keyname)
else:
aws_keyname = keyname
print 'downloading data from S3...'
s3key = Key(bucket)
s3key.key = aws_keyname
s3key.get_contents_to_filename(path)
return path
def delete_product_image(product):
S3_BUCKET = settings.S3_BUCKET
AWS_ACCESS_KEY_ID = settings.AWS_ACCESS_KEY_ID
AWS_SECRET_ACCESS_KEY = settings.AWS_SECRET_ACCESS_KEY
conn = S3Connection(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
url_list1 = product.image_url.split("/")
filename1 = url_list1[len(url_list1) - 1]
url_list2 = product.thumbnail.url.split("/")
filename2 = url_list2[len(url_list2) - 1]
url_list3 = product.watermark.url.split("/")
filename3 = url_list3[len(url_list3) - 1]
b = Bucket(conn, S3_BUCKET)
k = Key(b)
k.key = 'products/' + filename1
b.delete_key(k)
k.key = 'products/thumbnails/' + filename2
b.delete_key(k)
k.key = 'products/watermarked/' + filename3
b.delete_key(k)
def delete_from_s3(image_name):
"""Delete image from S3 bucket"""
conn = S3Connection(aws_access_key_id, aws_secret_access_key)
bucket = Bucket(conn, "shopifyimagerepository")
k = Key(bucket)
k.key = image_name
bucket.delete_key(k)
def remove_profile_image(user):
S3_BUCKET = settings.S3_BUCKET
AWS_ACCESS_KEY_ID = settings.AWS_ACCESS_KEY_ID
AWS_SECRET_ACCESS_KEY = settings.AWS_SECRET_ACCESS_KEY
conn = S3Connection(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
url_list1 = user.profile_image_url.split("/")
filename1 = url_list1[len(url_list1) - 1]
url_list2 = user.profile_image_crop.url.split("/")
filename2 = url_list2[len(url_list2) - 1]
b = Bucket(conn, S3_BUCKET)
k = Key(b)
k.key = 'profile_images/' + filename1
b.delete_key(k)
k.key = 'profile_images/crop/' + filename2
b.delete_key(k)
user.profile_image_url = None
user.profile_image_crop = None
user.save()
def delete_from_S3(filename):
conn = S3Connection(settings.AWS_ACCESS_KEY_ID,
settings.AWS_SECRET_ACCESS_KEY)
bucket = Bucket(conn, settings.AWS_STORAGE_BUCKET_NAME)
k = Key(bucket)
k.key = settings.MEDIA_URL + filename
bucket.delete_key(k)
def delete_file(self, data, suffix=''):
""" Handle file deletion requests. For this, we use the Amazon Python SDK,
boto.
"""
from .models import FileAndUrl
boto.set_stream_logger('boto')
S3 = S3Connection(settings.AWS_ACCESS_KEY_ID,
settings.AWS_SECRET_ACCESS_KEY)
if boto:
file_id = data.get('file_id', None)
bucket_name = self.s3_bucket
aws_bucket = S3.get_bucket(bucket_name, validate=False)
fileuploader = FileAndUrl()
log.info(u"fileuploader.get_file_path(file_id)%s",
fileuploader.get_file_path(file_id))
#Delete for S3
file_key = Key(aws_bucket, fileuploader.get_file_path(file_id))
file_key.delete()
#Delete from db
fileuploader.delete_record(file_id)
return
else:
return
def add_bucket(self, bucket_name, access, zonename, create_date):
try:
bucket_count = len(self.conn.get_all_buckets())
# print 'bucket_count is %s ' %bucket_count
if bucket_count < self.bucket_limit:
self.conn.create_bucket(bucket_name)
b = self.conn.get_bucket(bucket_name)
try:
# k=b.new_key('create_info')
# k.set_contents_from_string("{'bucket_name':'%s','zonename':'%s','access':'%s','create_date':'%s'}" %(bucket_name,zonename,access,create_date))
k1 = Key(b)
k1.key = 'create_info'
#k1.set_metadata('Bucket_Name',bucket_name),注意，在设置元数据的时候key名不能带有下划线，该示例在创建的时候会报错403
k1.set_metadata('BucketName', bucket_name)
k1.set_metadata('ZoneName', zonename)
k1.set_metadata('Access', access)
k1.set_metadata('CreateDate', create_date)
k1.set_contents_from_string('')
except Exception as e:
print r'\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\', e
return True
else:
return False
except Exception as e:
return False
def download_file_s3(keyname,
aws_key,
aws_secret,
s3_bucket,
s3_folder=None,
local_folder=None):
""" Download a file from an S3 bucket and save it at keyname. """
if local_folder is not None:
ensure_directory(local_folder)
path = os.path.join(local_folder, keyname)
else:
path = keyname
if os.path.isfile(path):
print 'file %s already exists!' % path
return path
conn = S3Connection(aws_key, aws_secret)
bucket = conn.get_bucket(s3_bucket)
if s3_folder:
aws_keyname = os.path.join(s3_folder, keyname)
else:
aws_keyname = keyname
print 'downloading data from S3...'
s3key = Key(bucket)
s3key.key = aws_keyname
s3key.get_contents_to_filename(path)
return path
def handle_DELETE(request):
""" Handle file deletion requests. For this, we use the Amazon Python SDK,
boto.
"""
try:
boto.set_stream_logger('boto')
S3 = S3Connection(aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY)
except ImportError:
print("Could not import boto, the Amazon SDK for Python.")
print("Deleting files will not work.")
print("Install boto with")
print("$ pip install boto")
return make_response(500)
bucket_name = request.POST.get('bucket')
key_name = request.POST.get('key')
try:
aws_bucket = S3.get_bucket(bucket_name, validate=False)
aws_key = Key(aws_bucket, key_name)
aws_key.delete()
return make_response(200)
except Exception as err:
print(err)
return make_response(500)
def clean_s3(self):
if self.file_to_clean is not None:
from boto.s3.connection import S3Connection, Bucket, Key
conn = S3Connection(cconfig.S3_ACCESS_KEY, cconfig.S3_SECRET_KEY)
bucket = conn.get_bucket(cconfig.S3_BUCKET)
k = Key(bucket)
k.key = self.file_to_clean[1:]
bucket.delete_key(k)
def getGroupsTrackerKey(self):
groups_tracker_key_name = self.getGroupsTrackerKeyName()
bucket = getHDISBucket()
groups_tracker_key = bucket.get_key(groups_tracker_key_name)
if not groups_tracker_key:
groups_tracker_key = Key(bucket)
groups_tracker_key.key = groups_tracker_key_name
return groups_tracker_key
def uploadStrToS3(self, destDir, filename, contents):
'''Uploads a string to an S3 file.'''
print 'Uploading %s to Amazon S3 bucket %s' % (filename, self.bucket_name)
k2 = Key(self.bucket)
k2.key = os.path.join(destDir, filename)
k2.set_contents_from_string(contents, reduced_redundancy=True)
print # This newline is needed to get the path of the compiled binary printed on a newline.
def save_product_image_to_s3(self):
if any([key == self.slug for key in BUCKET_LIST]):
return
k = Key(bucket)
k.key = self.slug
file_object = urllib2.urlopen(self.img)
fp = StringIO.StringIO(file_object.read())
k.set_contents_from_file(fp)
def uploadFileToS3(self, filename):
'''Uploads file to S3.'''
destDir = '' # Root folder of the S3 bucket
destpath = os.path.join(destDir, os.path.basename(filename))
print 'Uploading %s to Amazon S3 bucket %s' % (filename, self.bucket_name)
k = Key(self.bucket)
k.key = destpath
k.set_contents_from_filename(filename, reduced_redundancy=True)
def uploadStrToS3(self, destDir, filename, contents): # pylint: disable=invalid-name,missing-param-doc
# pylint: disable=missing-type-doc
"""Upload a string to an S3 file."""
print(f"Uploading {filename} to Amazon S3 bucket {self.bucket_name}")
k2 = Key(self.bucket) # pylint: disable=invalid-name
k2.key = os.path.join(destDir, filename)
k2.set_contents_from_string(contents, reduced_redundancy=True)
print() # This newline is needed to get the path of the compiled binary printed on a newline.
def transfer_files(self):
from boto.s3.connection import S3Connection
from boto.s3.connection import Key
conn = S3Connection(self.extra_args['aws_access_key'],
self.extra_args['aws_secret_key'])
bucket = conn.get_bucket(self.extra_args['s3_bucket'])
for fname in self.files:
key = Key(bucket)
key.key = os.path.basename(fname)
key.set_contents_from_filename(fname)
def uploadFileToS3(self, filename): # pylint: disable=invalid-name,missing-param-doc,missing-type-doc
"""Upload file to S3."""
# Root folder of the S3 bucket
destDir = "" # pylint: disable=invalid-name
destpath = os.path.join(destDir, os.path.basename(filename))
print(f"Uploading {filename} to Amazon S3 bucket {self.bucket_name}")
k = Key(self.bucket)
k.key = destpath
k.set_contents_from_filename(filename, reduced_redundancy=True)
def delete_image_from_s3(file_name):
try:
conn = S3Connection(AWS_ACCESS_KEY, AWS_SECRET_KEY)
logging.info("success s3 connection")
bucket = Bucket(conn, BUCKET)
k = Key(bucket=bucket, name=file_name)
k.delete()
logging.info("success delete image from s3")
except Exception as e:
logging.debug(e)
def uploadFileToS3(self, filename):
'''Uploads file to S3.'''
destDir = '' # Root folder of the S3 bucket
destpath = os.path.join(destDir, os.path.basename(filename))
print 'Uploading %s to Amazon S3 bucket %s' % (filename, self.bucket_name)
k = Key(self.bucket)
k.key = destpath
k.set_contents_from_filename(filename, reduced_redundancy=True)
print # This newline is needed to get the path of the compiled binary printed on a newline.
def add_latest(self, archive_path):
print(" Adding backup from", archive_path)
print(" * Uploading to", self.latestPath)
pbar = S3Progress(archive_path)
Key(self.bucket, self.latestPath).set_contents_from_filename(archive_path, cb=pbar.cb_function, num_cb=pbar.num_cb)
pbar.final()
#self.bucket.get_key(self.latestPath).set_contents_from_filename(archive_path)
print(" * Copying to", self.currentPath)
self.bucket.copy_key(self.currentPath, self.bucket.name, self.latestPath)
def post(self, request):
key = Key(self._bucket)
id = request.POST['id']
attachment = AIAttachment.objects.get(id=id)
key.key = attachment.attachment
self._bucket.delete_key(key)
attachment.delete()
return JsonResponse({'ok': True})
def store_image(self,
callback,
image_id,
request,
body=None,
filename=None,
**kwargs):
bucket = self._get_bucket()
image = Key(bucket, image_id)
image.set_contents_from_file(body)
callback(image.generate_url(HOUR))
def upload(self, local_filepath, aws_filepath):
"""
Uploads `local_filepath` to `aws_filepath`.
Returns the published URL for the file.
"""
logging.info('Publishing %s to %s' % (local_filepath, aws_filepath))
key = Key(bucket=self.bucket, name=aws_filepath)
key.key = aws_filepath
key.set_contents_from_filename(local_filepath)
key.set_acl('public-read')
def s3_delete_image(data):
try:
from boto.s3.connection import S3Connection, Bucket, Key
conn = S3Connection(data['S3_KEY'], data['S3_SECRET'])
b = Bucket(conn, data['S3_BUCKET'])
k = Key(b)
k.key = data['S3_UPLOAD_DIRECTORY'] + '/' + data['destinationFileName']
b.delete_key(k)
except Exception as e:
return e
def delete_file_from_s3(filename):
conn = S3Connection(
settings.AWS_ACCESS_KEY_ID,
settings.AWS_SECRET_ACCESS_KEY,
)
b = Bucket(
conn,
settings.AWS_STORAGE_BUCKET_NAME,
)
k = Key(b)
k.key = filename
b.delete_key(k)
