def S3Upload(upload_name, fileObj, bucket_name=None):
print 'check upload args'
if not bucket_name:
raise ValueError('No Bucket Name')
print 'conn'
conn = S3.AWSAuthConnection(config.AWS_ACCESS_KEY_ID,
config.AWS_SECRET_ACCESS_KEY)
content_type = mimetypes.guess_type(upload_name)[0]
if not content_type:
content_type = 'text/plain'
print 'conn put'
st = conn.put(bucket_name, upload_name, S3.S3Object(fileObj), {
'x-amz-acl': 'public-read',
'Content-Type': content_type
})
print 'end conn put'
resp = st.http_response
print 'resp', resp, resp.status
if 200 != resp.status:
print 'upload failed'
print resp.msg
return False
print 'upload successed'
return True
def connect_s3(self):
if tornado.options.options.shunt_s3:
logging.info('skipping s3 connection --shunt-s3')
return
aws_access_key_id = _utf8(tornado.options.options.aws_key)
aws_secret_access_key = _utf8(tornado.options.options.aws_secret)
self.conn = S3.AWSAuthConnection(aws_access_key_id,
aws_secret_access_key)
def upload(filename):
conn = S3.AWSAuthConnection(config.S3_ACCESS_KEY, config.S3_SECRET)
result = conn.check_bucket_exists(config.S3_BUCKET)
if result.status != 200:
result = conn.create_located_bucket(config.S3_BUCKET, S3.Location.DEFAULT)
assert 200 == conn.put(config.S3_BUCKET, os.path.basename(filename), read_file(filename)).http_response.status
print "File %s successfully backed up to S3 (with same filename)." % filename
def upload_s3(fname, mimetype, uname=''):
if not uname:
uname = os.path.basename(fname)
filedata = open(fname, 'rb').read()
conn = S3.AWSAuthConnection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY)
conn.put(settings.BUCKET_NAME, uname, S3.S3Object(filedata),
{'x-amz-acl': 'public-read', 'Content-Type': mimetype})
def list_files():
conn = S3.AWSAuthConnection(config.S3_ACCESS_KEY, config.S3_SECRET)
result = conn.check_bucket_exists(config.S3_BUCKET)
if result.status != 200:
result = conn.create_located_bucket(config.S3_BUCKET,
S3.Location.DEFAULT)
result = conn.list_bucket(config.S3_BUCKET)
assert 200 == result.http_response.status
print "Size\t\tKey"
for entry in result.entries:
print "%s\t%s" % (entry.size, entry.key)
def retrieve(filename):
conn = S3.AWSAuthConnection(config.S3_ACCESS_KEY, config.S3_SECRET)
assert 200 == conn.check_bucket_exists(config.S3_BUCKET).status
result = conn.get(config.S3_BUCKET, filename)
assert 200 == result.http_response.status
f = open(filename, "w")
f.write(result.object.data)
f.close()
print "File %s successfully retrieved (with same filename)." % filename
def __init__(self, access_key_id, secret_access_key, bucket,
key_length=8, publish_domain=None, http=None,
generate_key=generate_random_word):
if publish_domain is None:
publish_domain = '%s.%s' % (bucket, S3.DEFAULT_HOST)
if http is None:
http = httplib2.Http()
self.conn = S3.AWSAuthConnection(access_key_id, secret_access_key)
self.bucket = bucket
self.key_length = key_length
self.publish_domain = publish_domain
self.http = http
self.generate_key = generate_key
if self.conn.check_bucket_exists(bucket).status == 404:
self.conn.create_located_bucket(bucket, S3.Location.DEFAULT)
def update_s3():
conn = S3.AWSAuthConnection(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
for line in sys.stdin:
filename = os.path.normpath(line[:-1])
if filename == '.' or not os.path.isfile(filename):
continue # Skip this, because it's not a file.
print "Uploading %s" % filename
filedata = open(filename, 'rb').read()
content_type = mimetypes.guess_type(filename)[0]
if not content_type:
content_type = 'text/plain'
conn.put(BUCKET_NAME, filename, S3.S3Object(filedata), {
'x-amz-acl': 'public-read',
'Content-Type': content_type
})
def list_s3(request):
"""
List Amazon S3 bucket contents
"""
if S3 is not None:
conn = S3.AWSAuthConnection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY)
generator = S3.QueryStringAuthGenerator(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY, calling_format=S3.CallingFormat.VANITY)
generator.set_expires_in(300)
bucket_entries = conn.list_bucket(settings.AWS_BUCKET_NAME).entries
entries = []
for entry in bucket_entries:
entry.s3url = generator.get(settings.AWS_BUCKET_NAME, entry.key)
entries.append(entry)
return direct_to_template(request, 'export/list_s3.html', {'object_list': entries, 's3support': True})
else:
return direct_to_template(request, 'export/list_s3.html', {'object_list': [], 's3support': False})
def __init__(self, upload_to='', stored_file_implementation=StoredFile):
# try to work around bug S3 code which uses bad names of days
# http://code.google.com/p/boto/issues/detail?id=140
# but workaround doesn't work :(
#import locale
# locale.setlocale(locale.LC_TIME, 'en_US.utf8')
# print 'create S3 storage'
import settings
import S3
self.upload_to = upload_to
conn = S3.AWSAuthConnection(settings.AWS_ACCESS_KEY_ID,
settings.AWS_SECRET_ACCESS_KEY)
# _generator = S3.QueryStringAuthGenerator( settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY )
if (conn.check_bucket_exists(settings.AWS_BUCKET_NAME).status == 200):
pass
else:
conn.create_located_bucket(settings.AWS_BUCKET_NAME,
settings.AWS_LOCATION).message
def publish(filepath, s3bucket, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY,
version):
filename = filepath.split("/")[-1]
s3key = "/".join([p['release.type'], p['project.key'], filename])
print "Reading in content from %s" % filepath
filedata = open(filepath, "rb").read()
filehash = _sha(filedata).hexdigest()
print "Preparing to upload %s to %s/%s" % (filename, s3bucket, s3key)
content_type = mimetypes.guess_type(filename)[0]
if content_type is None:
content_type = 'text/plain'
print "File appears to be %s" % content_type
print "Connecting to S3..."
conn = S3.AWSAuthConnection(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
print "Checking if bucket %s exists..." % s3bucket
check = conn.check_bucket_exists(s3bucket)
if (check.status == 200):
print "Uploading %s to %s/%s" % (filename, s3bucket, s3key)
print conn.put(
s3bucket, s3key, S3.S3Object(filedata), {
'Content-Type': content_type,
'x-amz-acl': 'public-read',
'x-amz-meta-project.name': 'Spring Python',
'x-amz-meta-release.type': p['release.type'],
'x-amz-meta-bundle.version': version,
'x-amz-meta-package.file.name': filename
}).message
print "Uploading SHA1 digest to %s/%s" % (s3bucket, s3key + '.sha1')
print conn.put(s3bucket, s3key + '.sha1',
S3.S3Object(filehash + ' ' + filename + "\n"), {
'Content-Type': content_type,
'x-amz-acl': 'public-read'
}).message
else:
print "Error code %s: Unable to publish" % check.status
def save(self):
conn = S3.AWSAuthConnection(secrets.AWS_ID, secrets.AWS_SECRET_KEY)
uploaded_filename = self.cleaned_data['filename'].name
filename = '/%s/%s' % (self.project, uploaded_filename)
content = self.cleaned_data['filename'].read()
try:
old_file = self.project.projectfile_set.get(
filename=uploaded_filename)
versions = old_file.projectfileversion_set.all().count()
split_f = filename.rsplit('.', 1)
name_no_ext = ''.join(split_f[:-1])
filename = '%s-%s.%s' % (name_no_ext, versions + 1, split_f[-1])
response = conn.put(defaults.bucket, filename, content)
saved_file = old_file
saved_file_revision = ProjectFileVersion(file=saved_file,
revision_name=filename,
user=self.user,
size=len(content))
saved_file_revision.save()
saved_file.current_revision = saved_file_revision
saved_file.total_size += saved_file_revision.size
saved_file.save()
except ProjectFile.DoesNotExist, e:
split_f = filename.rsplit('.', 1)
name_no_ext = ''.join(split_f[:-1])
filename = '%s-%s.%s' % (name_no_ext, 1, split_f[-1])
response = conn.put(defaults.bucket, filename, content)
saved_file = ProjectFile(project=self.project,
filename=uploaded_filename,
total_size=0)
saved_file.save()
saved_file_revision = ProjectFileVersion(file=saved_file,
revision_name=filename,
user=self.user,
size=len(content))
saved_file_revision.save()
saved_file.current_revision = saved_file_revision
saved_file.total_size = saved_file_revision.size
saved_file.save()
def push_media_to_s3(subpath, content_type):
"""
Upload a subpath of the media directory to S3.
"""
if not settings.USE_S3:
return
import S3
conn = S3.AWSAuthConnection(settings.S3_ACCESS_KEY, settings.S3_SECRET_KEY)
localPath = os.path.join(settings.MEDIA_ROOT, subpath)
obj = S3.S3Object(file(localPath).read())
tries = 5
while True:
try:
conn.put(settings.S3_BUCKET, settings.S3_PATH + subpath, obj, {
'Content-Type': content_type,
'x-amz-acl': 'public-read'
})
except:
tries -= 1
if not tries:
raise
else:
return
def files(request, project_name):
"""Files for a project. Shows the files uploaded for a project.
Actions available:
Add files: Owner Participant
"""
project = get_project(request, project_name)
gen = S3.QueryStringAuthGenerator(secrets.AWS_ID, secrets.AWS_SECRET_KEY)
addfileform = bforms.AddFileForm(project=project, user=request.user)
if request.method == 'POST':
if request.POST.has_key('Addfile'):
addfileform = bforms.AddFileForm(project, request.user,
request.POST, request.FILES)
if addfileform.is_valid():
addfileform.save()
return HttpResponseRedirect('.')
if request.POST.has_key('fileid'):
fileid = int(request.POST['fileid'])
file = ProjectFile.objects.get(project=project, id=fileid)
conn = S3.AWSAuthConnection(secrets.AWS_ID, secrets.AWS_SECRET_KEY)
for revision in file.projectfileversion_set.all():
conn.delete(defaults.bucket, revision.revision_name)
file.delete()
payload = locals()
return render(request, 'project/files.html', payload)
def export_to_s3(request):
"""
Dump the database and upload the dump to Amazon S3
"""
if request.method == 'POST':
if settings.DATABASE_ENGINE == 'mysql':
cmd = MYSQLDUMP_CMD % (settings.DATABASE_HOST, settings.DATABASE_USER, settings.DATABASE_PASSWORD, settings.DATABASE_NAME)
elif settings.DATABASE_ENGINE == 'sqlite3':
cmd = SQLITE3DUMP_CMD % settings.DATABASE_NAME
else:
raise ImproperlyConfigured, "Sorry, django-export only supports mysql and sqlite3 database backends."
stdin, stdout = os.popen2(cmd)
stdin.close()
file_name = 'dump_%s.sql.bz2' % time.strftime('%Y%m%d-%H%M')
conn = S3.AWSAuthConnection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY)
res = conn.put(settings.AWS_BUCKET_NAME, file_name, S3.S3Object(stdout.read()), {'Content-Type': 'application/x-bzip2',})
if res.http_response.status == 200:
request.user.message_set.create(message="%s" % _(u"%(filename)s saved on Amazon S3") % {'filename': file_name})
else:
request.user.message_set.create(message="%s" % _(u"Upload failed with %(status)s") % {'status': res.http_response.status})
stdout.close()
return HttpResponseRedirect('/admin/')
return direct_to_template(request, 'export/export.html', {'what': _(u'Export Database to S3'), 's3support': (S3 is not None), 's3': True})
def conn(self):
try:
return self._conn
except AttributeError:
self._conn = S3.AWSAuthConnection(opts.key, opts.secret)
return self._conn
from urllib2 import URLError
import os
import re
import random
import secret as s
import fileinput
import S3
import time
import threading
import sys
import pdb
import json
import mapreduce
from claim_from_body import claims_from_html2
conn = S3.AWSAuthConnection(s.amazon_access, s.amazon_secret)
num_threads = 200
urls = []
downloaded = {}
def download_urls():
threads = []
for num in range(0, num_threads):
t = DownloadThread()
t.start()
threads.append(t)
for t in threads:
t.join()
import S3 AWS_ACCESS_KEY_ID = 'AKIAI7R3ZZAEQ2GBDHUA' AWS_SECRET_ACCESS_KEY = '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' # for subdomains (bucket.s3.amazonaws.com), # the bucket name must be lowercase since DNS is case-insensitive # BUCKET_NAME = "%s-test-bucket" % AWS_ACCESS_KEY_ID.lower(); BUCKET_NAME = "spajic" conn = S3.AWSAuthConnection(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) response = conn.list_bucket(BUCKET_NAME) print response.entries[0].key print response.entries[1].key print response.entries[2].key text = 'this is a test' key = 'example.txt' response = conn.put(BUCKET_NAME, key, text)
import S3
conn = S3.AWSAuthConnection('18K4VFS99J8R21G8TBR2', 'EVf+5YzNyDMMXU12VFUf3N+LrTTSU/iWsf81Sp/M')
conn.delete('s3map', 'lib/s3map-1.0.jar')
def setUp(self):
self.conn = S3.AWSAuthConnection(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
