def save_gtfs_file(self, contents, user, gtfs_crawler, comments, filename): assert '@' in user # if setting is prod if tornado.options.options.shunt_s3: if filename.startswith("queue/"): filename = filename[len("queue/"):] if not os.path.exists("/tmp/gtfs_s3/queue"): os.makedirs("/tmp/gtfs_s3/queue") filename = os.path.join("/tmp/gtfs_s3/queue", filename) logging.info('writing %s' % filename) f = open(filename, 'wb') f.write(contents) f.close() f = open(filename + '.meta', 'wb') f.write( json.dumps( dict(user=user, gtfs_crawler=gtfs_crawler, comments=comments))) f.close() else: obj = S3.S3Object(contents) obj.metadata['user'] = user obj.metadata['gtfs_crawler'] = gtfs_crawler obj.metadata['comments'] = comments logging.info('putting %r' % filename) self.conn.put("gtfs", filename, obj)
def S3Upload(upload_name, fileObj, bucket_name=None): print 'check upload args' if not bucket_name: raise ValueError('No Bucket Name') print 'conn' conn = S3.AWSAuthConnection(config.AWS_ACCESS_KEY_ID, config.AWS_SECRET_ACCESS_KEY) content_type = mimetypes.guess_type(upload_name)[0] if not content_type: content_type = 'text/plain' print 'conn put' st = conn.put(bucket_name, upload_name, S3.S3Object(fileObj), { 'x-amz-acl': 'public-read', 'Content-Type': content_type }) print 'end conn put' resp = st.http_response print 'resp', resp, resp.status if 200 != resp.status: print 'upload failed' print resp.msg return False print 'upload successed' return True
def upload_s3(fname, mimetype, uname=''): if not uname: uname = os.path.basename(fname) filedata = open(fname, 'rb').read() conn = S3.AWSAuthConnection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY) conn.put(settings.BUCKET_NAME, uname, S3.S3Object(filedata), {'x-amz-acl': 'public-read', 'Content-Type': mimetype})
def publish(filepath, s3bucket, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, version): filename = filepath.split("/")[-1] s3key = "/".join([p['release.type'], p['project.key'], filename]) print "Reading in content from %s" % filepath filedata = open(filepath, "rb").read() filehash = _sha(filedata).hexdigest() print "Preparing to upload %s to %s/%s" % (filename, s3bucket, s3key) content_type = mimetypes.guess_type(filename)[0] if content_type is None: content_type = 'text/plain' print "File appears to be %s" % content_type print "Connecting to S3..." conn = S3.AWSAuthConnection(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) print "Checking if bucket %s exists..." % s3bucket check = conn.check_bucket_exists(s3bucket) if (check.status == 200): print "Uploading %s to %s/%s" % (filename, s3bucket, s3key) print conn.put( s3bucket, s3key, S3.S3Object(filedata), { 'Content-Type': content_type, 'x-amz-acl': 'public-read', 'x-amz-meta-project.name': 'Spring Python', 'x-amz-meta-release.type': p['release.type'], 'x-amz-meta-bundle.version': version, 'x-amz-meta-package.file.name': filename }).message print "Uploading SHA1 digest to %s/%s" % (s3bucket, s3key + '.sha1') print conn.put(s3bucket, s3key + '.sha1', S3.S3Object(filehash + ' ' + filename + "\n"), { 'Content-Type': content_type, 'x-amz-acl': 'public-read' }).message else: print "Error code %s: Unable to publish" % check.status
def create(self, content, mimetype, metadata): key = self._generate_valid_key() obj = S3.S3Object(content, metadata) self.conn.put(self.bucket, key, obj, { 'x-amz-storage-class': 'REDUCED_REDUNDANCY', 'Content-Type': mimetype, }) aclxml = self.conn.get_acl(self.bucket, key).body acl = parseString(aclxml) acl.getElementsByTagName('AccessControlList')[0].appendChild(_pub_read_grant) self.conn.put_acl(self.bucket, key, acl.toxml()) return self._published_url(key)
def update_s3(): conn = S3.AWSAuthConnection(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) for line in sys.stdin: filename = os.path.normpath(line[:-1]) if filename == '.' or not os.path.isfile(filename): continue # Skip this, because it's not a file. print "Uploading %s" % filename filedata = open(filename, 'rb').read() content_type = mimetypes.guess_type(filename)[0] if not content_type: content_type = 'text/plain' conn.put(BUCKET_NAME, filename, S3.S3Object(filedata), { 'x-amz-acl': 'public-read', 'Content-Type': content_type })
def push_media_to_s3(subpath, content_type): """ Upload a subpath of the media directory to S3. """ if not settings.USE_S3: return import S3 conn = S3.AWSAuthConnection(settings.S3_ACCESS_KEY, settings.S3_SECRET_KEY) localPath = os.path.join(settings.MEDIA_ROOT, subpath) obj = S3.S3Object(file(localPath).read()) tries = 5 while True: try: conn.put(settings.S3_BUCKET, settings.S3_PATH + subpath, obj, { 'Content-Type': content_type, 'x-amz-acl': 'public-read' }) except: tries -= 1 if not tries: raise else: return
def export_to_s3(request): """ Dump the database and upload the dump to Amazon S3 """ if request.method == 'POST': if settings.DATABASE_ENGINE == 'mysql': cmd = MYSQLDUMP_CMD % (settings.DATABASE_HOST, settings.DATABASE_USER, settings.DATABASE_PASSWORD, settings.DATABASE_NAME) elif settings.DATABASE_ENGINE == 'sqlite3': cmd = SQLITE3DUMP_CMD % settings.DATABASE_NAME else: raise ImproperlyConfigured, "Sorry, django-export only supports mysql and sqlite3 database backends." stdin, stdout = os.popen2(cmd) stdin.close() file_name = 'dump_%s.sql.bz2' % time.strftime('%Y%m%d-%H%M') conn = S3.AWSAuthConnection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY) res = conn.put(settings.AWS_BUCKET_NAME, file_name, S3.S3Object(stdout.read()), {'Content-Type': 'application/x-bzip2',}) if res.http_response.status == 200: request.user.message_set.create(message="%s" % _(u"%(filename)s saved on Amazon S3") % {'filename': file_name}) else: request.user.message_set.create(message="%s" % _(u"Upload failed with %(status)s") % {'status': res.http_response.status}) stdout.close() return HttpResponseRedirect('/admin/') return direct_to_template(request, 'export/export.html', {'what': _(u'Export Database to S3'), 's3support': (S3 is not None), 's3': True})
print '----- creating bucket -----' print conn.create_located_bucket(BUCKET_NAME, S3.Location.DEFAULT).message # to create an EU bucket #print conn.create_located_bucket(BUCKET_NAME, S3.Location.EU).message print '----- bucket location -----' print conn.get_bucket_location(BUCKET_NAME).location print '----- listing bucket -----' print map(lambda x: x.key, conn.list_bucket(BUCKET_NAME).entries) print '----- putting object (with content type) -----' print conn.put( BUCKET_NAME, KEY_NAME, S3.S3Object('this is a test'), { 'Content-Type': 'text/plain' }).message print '----- listing bucket -----' print map(lambda x: x.key, conn.list_bucket(BUCKET_NAME).entries) print '----- getting object -----' print conn.get(BUCKET_NAME, KEY_NAME).object.data print '----- query string auth example -----' print "\nTry this url out in your browser (it will only be valid for 60 seconds).\n" generator.set_expires_in(60); url = generator.get(BUCKET_NAME, KEY_NAME) print url print '\npress enter> ', sys.stdin.readline()
def run_tests(self, calling_format, location): self.conn.calling_format = calling_format response = self.conn.create_located_bucket(BUCKET_NAME, location) self.assertEquals(response.http_response.status, 200, 'create bucket') response = self.conn.list_bucket(BUCKET_NAME) self.assertEquals(response.http_response.status, 200, 'list bucket') self.assertEquals(len(response.entries), 0, 'bucket is empty') text = 'this is a test' key = 'example.txt' response = self.conn.put(BUCKET_NAME, key, text) self.assertEquals(response.http_response.status, 200, 'put with a string argument') response = \ self.conn.put( BUCKET_NAME, key, S3.S3Object(text, {'title': 'title'}), {'Content-Type': 'text/plain'}) self.assertEquals(response.http_response.status, 200, 'put with complex argument and headers') response = self.conn.get(BUCKET_NAME, key) self.assertEquals(response.http_response.status, 200, 'get object') self.assertEquals(response.object.data, text, 'got right data') self.assertEquals(response.object.metadata, { 'title': 'title' }, 'metadata is correct') self.assertEquals(int(response.http_response.getheader('Content-Length')), len(text), 'got content-length header') title_with_spaces = " \t title with leading and trailing spaces " response = \ self.conn.put( BUCKET_NAME, key, S3.S3Object(text, {'title': title_with_spaces}), {'Content-Type': 'text/plain'}) self.assertEquals(response.http_response.status, 200, 'put with headers with spaces') response = self.conn.get(BUCKET_NAME, key) self.assertEquals(response.http_response.status, 200, 'get object') self.assertEquals( response.object.metadata, { 'title': title_with_spaces.strip() }, 'metadata with spaces is correct') # delimited list tests inner_key = 'test/inner.txt' last_key = 'z-last-key.txt' response = self.conn.put(BUCKET_NAME, inner_key, text) self.assertEquals(response.http_response.status, 200, 'put inner key') response = self.conn.put(BUCKET_NAME, last_key, text) self.assertEquals(response.http_response.status, 200, 'put last key') response = self.do_delimited_list(BUCKET_NAME, False, {'delimiter': '/'}, 2, 1, 'root list') response = self.do_delimited_list(BUCKET_NAME, True, {'max-keys': 1, 'delimiter': '/'}, 1, 0, 'root list with max keys of 1', 'example.txt') response = self.do_delimited_list(BUCKET_NAME, True, {'max-keys': 2, 'delimiter': '/'}, 1, 1, 'root list with max keys of 2, page 1', 'test/') marker = response.next_marker response = self.do_delimited_list(BUCKET_NAME, False, {'marker': marker, 'max-keys': 2, 'delimiter': '/'}, 1, 0, 'root list with max keys of 2, page 2') response = self.do_delimited_list(BUCKET_NAME, False, {'prefix': 'test/', 'delimiter': '/'}, 1, 0, 'test/ list') response = self.conn.delete(BUCKET_NAME, inner_key) self.assertEquals(response.http_response.status, 204, 'delete %s' % inner_key) response = self.conn.delete(BUCKET_NAME, last_key) self.assertEquals(response.http_response.status, 204, 'delete %s' % last_key) weird_key = '&=//%# ++++' response = self.conn.put(BUCKET_NAME, weird_key, text) self.assertEquals(response.http_response.status, 200, 'put weird key') response = self.conn.get(BUCKET_NAME, weird_key) self.assertEquals(response.http_response.status, 200, 'get weird key') response = self.conn.get_acl(BUCKET_NAME, key) self.assertEquals(response.http_response.status, 200, 'get acl') acl = response.object.data response = self.conn.put_acl(BUCKET_NAME, key, acl) self.assertEquals(response.http_response.status, 200, 'put acl') response = self.conn.get_bucket_acl(BUCKET_NAME) self.assertEquals(response.http_response.status, 200, 'get bucket acl') bucket_acl = response.object.data response = self.conn.put_bucket_acl(BUCKET_NAME, bucket_acl) self.assertEquals(response.http_response.status, 200, 'put bucket acl') response = self.conn.get_bucket_acl(BUCKET_NAME) self.assertEquals(response.http_response.status, 200, 'get bucket logging') bucket_logging = response.object.data response = self.conn.put_bucket_acl(BUCKET_NAME, bucket_logging) self.assertEquals(response.http_response.status, 200, 'put bucket logging') response = self.conn.list_bucket(BUCKET_NAME) self.assertEquals(response.http_response.status, 200, 'list bucket') entries = response.entries self.assertEquals(len(entries), 2, 'got back right number of keys') # depends on weird_key < key self.assertEquals(entries[0].key, weird_key, 'first key is right') self.assertEquals(entries[1].key, key, 'second key is right') response = self.conn.list_bucket(BUCKET_NAME, {'max-keys': 1}) self.assertEquals(response.http_response.status, 200, 'list bucket with args') self.assertEquals(len(response.entries), 1, 'got back right number of keys') for entry in entries: response = self.conn.delete(BUCKET_NAME, entry.key) self.assertEquals(response.http_response.status, 204, 'delete %s' % entry.key) response = self.conn.list_all_my_buckets() self.assertEquals(response.http_response.status, 200, 'list all my buckets') buckets = response.entries response = self.conn.delete_bucket(BUCKET_NAME) self.assertEquals(response.http_response.status, 204, 'delete bucket') response = self.conn.list_all_my_buckets() self.assertEquals(response.http_response.status, 200, 'list all my buckets again') self.assertEquals(len(response.entries), len(buckets) - 1, 'bucket count is correct')
print resp.message print resp.http_response print resp.http_response.status # to create an EU bucket #print conn.create_located_bucket(BUCKET_NAME, S3.Location.EU).message print '----- bucket location -----' print conn.get_bucket_location(BUCKET_NAME).location print '----- listing bucket -----' print map(lambda x: x.key, conn.list_bucket(BUCKET_NAME).entries) sys.exit(0) print '----- putting object (with content type) -----' print conn.put(BUCKET_NAME, KEY_NAME, S3.S3Object('this is a test'), { 'Content-Type': 'text/plain' }).message print '----- listing bucket -----' print map(lambda x: x.key, conn.list_bucket(BUCKET_NAME).entries) print '----- getting object -----' print conn.get(BUCKET_NAME, KEY_NAME).object.data print '----- query string auth example -----' print "\nTry this url out in your browser (it will only be valid for 60 seconds).\n" generator.set_expires_in(60) url = generator.get(BUCKET_NAME, KEY_NAME) print url print '\npress enter> ',