def save_job_results(geocoder, job_id): """ Download and save to S3 results for completed jobs. """ logging.info('Saving results for %s to S3' % job_id) finished_folder = 'geocode_finished_jobs' pending_folder = 'geocode_pending_jobs' connection = boto.connect_s3() bucket = connection.get_bucket(GEO_BUCKET) old_key = bucket.get_key('%s/%s' % (pending_folder, job_id)) new_name = old_key.get_contents_as_string() new_key = Key(bucket) new_key.key = '%s/%s' % (finished_folder, new_name) results = geocoder.get_job_results(job_id) result_string = StringIO.StringIO() writer = DictWriter(result_string, fieldnames=results[0].keys()) writer.writeheader() writer.writerows(results) result_string.seek(0) email_address = old_key.get_metadata('email') if email_address: new_key.set_metadata('email', email_address) send_email_notification( email_address, geocoder.get_job_statuses(job_id=job_id), new_name, 'finished') new_key.set_contents_from_string(result_string.getvalue()) new_key.make_public() old_key.delete()
def addDataToS3(self, name, data, contentType): num_retries = 0 max_retries = 5 while True: try: conn = S3Connection(keys.aws.AWS_ACCESS_KEY_ID, keys.aws.AWS_SECRET_KEY) bucket = conn.lookup(self.bucket_name) key = Key(bucket, name) key.set_metadata('Content-Type', contentType) # for some reason, if we use set_contents_from_file here, an empty file is created key.set_contents_from_string(data.getvalue(), policy='public-read') #key.set_contents_from_file(data, policy='public-read') key.close() return "%s/%s" % (self.base_url, name) except Exception as e: logs.warning('S3 Exception: %s' % e) num_retries += 1 if num_retries > max_retries: msg = "Unable to connect to S3 after %d retries (%s)" % \ (max_retries, self.__class__.__name__) logs.warning(msg) raise Exception(msg) logs.info("Retrying (%s)" % (num_retries)) time.sleep(0.5) finally: try: if not key.closed: key.close() except Exception: logs.warning("Error closing key")
def dump(self, key, value, public=False): """ Dump file to S3. Optionally make public """ assert isinstance(key, basestring), u'Key must be a string' k = Key(self.bucket) k.key = key try: k.set_metadata(u'Content-Type', u'application/json') k.set_contents_from_string(json.dumps(value, sort_keys=True, indent=4, separators=(u',', u': '))) # set file permissions if public: k.set_acl(u'public-read') except Exception as e: print e return False else: # now update the cache if self._keys is not None: self._keys.add(key) return True
def uploadPublicImageFromString(self, s3KeyName, imageString): key = Key(self._bucket) key.key = s3KeyName # todo: check content-type key.set_metadata("Content-Type", "image/jpeg") key.set_contents_from_string(imageString) key.set_acl("public-read")
def _save_file_to_bucket(conn, bucket_name, remote_filename, local_file, **kwargs): """ Save the local_file to bucket_name as remote_filename. Also, any additional arguments passed as key-value pairs, are stored as file's metadata on S3.""" # print "Establishing handle with bucket '%s'..." % bucket_name b = _get_bucket(conn, bucket_name) if b is not None: # print "Establishing handle with key object '%s'..." % remote_filename k = Key( b, remote_filename ) print("Attempting to save file '%s' to bucket '%s'..." % (remote_filename, bucket_name)) try: # Store some metadata (key-value pairs) about the contents of the file being uploaded # Note that the metadata must be set *before* writing the file k.set_metadata('date_uploaded', str(datetime.utcnow())) for args_key in kwargs: print("Adding metadata to file '%s': %s=%s" % (remote_filename, args_key, kwargs[args_key])) k.set_metadata(args_key, kwargs[args_key]) print("Saving file '%s'" % local_file) k.set_contents_from_filename(local_file) print("Successfully added file '%s' to bucket '%s'." % (remote_filename, bucket_name)) make_public = True if make_public: k.make_public() except S3ResponseError as e: print("Failed to save file local file '%s' to bucket '%s' as file '%s': %s" % ( local_file, bucket_name, remote_filename, e )) return False return True else: return False
def store_file_from_disk(self, key, filepath, metadata=None, multipart=None): size = os.path.getsize(filepath) key = self.format_key_for_backend(key) if not multipart or size <= self.multipart_chunk_size: s3key = Key(self.bucket) s3key.key = key if metadata: for k, v in metadata.items(): s3key.set_metadata(k, v) s3key.set_contents_from_filename(filepath, replace=True) else: start_of_multipart_upload = time.monotonic() chunks = math.ceil(size / self.multipart_chunk_size) self.log.debug("Starting to upload multipart file: %r, size: %r, chunks: %d", key, size, chunks) mp = self.bucket.initiate_multipart_upload(key, metadata=metadata) with open(filepath, "rb") as fp: part_num = 0 while fp.tell() < size: part_num += 1 start_time = time.monotonic() self._store_multipart_upload(mp, fp, part_num, filepath) self.log.info("Upload of part: %r/%r of %r, part size: %r took: %.2fs", part_num, chunks, filepath, self.multipart_chunk_size, time.monotonic() - start_time) if len(mp.get_all_parts()) == chunks: self.log.info("Multipart upload of %r, size: %r, took: %.2fs, now completing multipart", filepath, size, time.monotonic() - start_of_multipart_upload) mp.complete_upload() else: err = "Multipart upload of {!r} does not match expected chunk list".format(key) self.log.error(err) mp.cancel_upload() raise StorageError(err)
def upload_to_s3(self, file, key, content_type=None): try: size = os.fstat(file.fileno()).st_size except: # Not all file objects implement fileno(), # so we fall back on this file.seek(0, os.SEEK_END) size = file.tell() conn = boto.connect_s3(self.aws_access_key_id, self.aws_secret_access_key) bucket = conn.get_bucket(self.bucketname, validate=False) k = Key(bucket) k.key = key k.delete() if content_type: k.set_metadata('Content-Type', content_type) sent = k.set_contents_from_file(file, policy='public-read') # Rewind for later use file.seek(0) if sent == size: url = "https://s3-ap-northeast-1.amazonaws.com/" url = url + self.bucketname url = url + "/" + key print (url) return url return ""
def save(key, path): key = 'uploads/' + key k = Key(getBucket(), key) k.set_contents_from_filename(path) mime = mimetypes.guess_type(key) k.set_metadata("Content-Type", mime) k.set_acl('public-read')
def upload_archive(self, filename, key, auto_create_bucket=True): """ Uploads an application archive version to s3 """ try: bucket = self.s3.get_bucket(self.aws.bucket) if (( self.aws.region != 'us-east-1' and self.aws.region != 'eu-west-1') and bucket.get_location() != self.aws.region) or ( self.aws.region == 'us-east-1' and bucket.get_location() != '') or ( self.aws.region == 'eu-west-1' and bucket.get_location() != 'eu-west-1'): raise Exception("Existing bucket doesn't match region") except S3ResponseError: bucket = self.s3.create_bucket(self.aws.bucket, location=self.aws.region) def __report_upload_progress(sent, total): if not sent: sent = 0 if not total: total = 0 out("Uploaded " + str(sent) + " bytes of " + str(total) \ + " (" + str(int(float(max(1, sent)) / float(total) * 100)) + "%)") # upload the new version k = Key(bucket) k.key = self.aws.bucket_path + key k.set_metadata('time', str(time())) k.set_contents_from_filename(filename, cb=__report_upload_progress, num_cb=10)
def _save_to_s3(self, data): assert len(data) > 0, "Need data to save!" # TODO: store AWS credentials in a better way. assert 'AWS_ACCESS_KEY_ID' in config, "Need AWS key!" assert 'AWS_SECRET_ACCESS_KEY' in config, "Need AWS secret!" assert 'MESSAGE_STORE_BUCKET_NAME' in config, \ "Need bucket name to store message data!" # Boto pools connections at the class level conn = S3Connection(config.get('AWS_ACCESS_KEY_ID'), config.get('AWS_SECRET_ACCESS_KEY')) bucket = conn.get_bucket(config.get('MESSAGE_STORE_BUCKET_NAME')) # See if it alreays exists and has the same hash data_obj = bucket.get_key(self.data_sha256) if data_obj: assert data_obj.get_metadata('data_sha256') == self.data_sha256, \ "Block hash doesn't match what we previously stored on s3!" # log.info("Block already exists on S3.") return data_obj = Key(bucket) # if metadata: # assert type(metadata) is dict # for k, v in metadata.iteritems(): # data_obj.set_metadata(k, v) data_obj.set_metadata('data_sha256', self.data_sha256) # data_obj.content_type = self.content_type # Experimental data_obj.key = self.data_sha256 # log.info("Writing data to S3 with hash {0}".format(self.data_sha256)) # def progress(done, total): # log.info("%.2f%% done" % (done/total * 100) ) # data_obj.set_contents_from_string(data, cb=progress) data_obj.set_contents_from_string(data)
def _addImageToS3(bucket, name, data): num_retries = 0 max_retries = 5 while True: try: key = Key(bucket, name) key.set_metadata("Content-Type", "image/jpeg") key.set_contents_from_string(data.getvalue(), policy="public-read") key.close() return key except Exception as e: logs.warning("S3 Exception: %s" % e) num_retries += 1 if num_retries > max_retries: msg = "Unable to connect to S3 after %d retries (%s)" % (max_retries, self.__class__.__name__) logs.warning(msg) raise Exception(msg) logs.info("Retrying (%s)" % (num_retries)) time.sleep(0.5) finally: try: if not key.closed: key.close() except Exception: logs.warning("Error closing key")
def addPhoto(photo, setTitle): url = flickr.photos_getSizes(photo_id = photo.attrib['id']) realUrl = None for url in url.find('sizes').findall('size'): if url.attrib['label'] == "Original": realUrl = url.attrib['source'] if realUrl: keyId = setTitle + "/" + photo.attrib['id'] + ".jpg" dataKeyId = keyId + ".metadata" # Upload photo if bucket.get_key(keyId) is None: print "%s not found on S3; uploading" % keyId f, h = urllib.urlretrieve(realUrl, reporthook = makeFlickrCallback()) key = Key(bucket) key.key = keyId print "Uploading %s to %s/%s" % (photo.attrib['title'], bucket.name, key.key) key.set_metadata('flickrInfo', key.key + ".metadata") key.set_metadata('inFlickrSet', set.attrib['id']) key.set_contents_from_filename(f, cb = makeBotoCallback()) os.unlink(f) # Upload metadata if bucket.get_key(dataKeyId) is None: print "%s not found on S3, setting metadata" % dataKeyId photoInfo = flickr.photos_getInfo(photo_id = photo.attrib['id'], format = "rest") key = Key(bucket) key.key = dataKeyId key.set_contents_from_string(photoInfo)
def _write_files(app, static_url_loc, static_folder, files, bucket, ex_keys=None, hashes=None): """ Writes all the files inside a static folder to S3. """ new_hashes = [] for file_path in files: asset_loc = _path_to_relative_url(file_path) key_name = _static_folder_path(static_url_loc, static_folder, asset_loc) msg = "Uploading %s to %s as %s" % (file_path, bucket, key_name) logger.debug(msg) exclude = False if app.config.get('S3_ONLY_MODIFIED', False): file_hash = hash_file(file_path) new_hashes.append((key_name, file_hash)) if hashes and hashes.get(key_name, None) == file_hash: exclude = True if ex_keys and key_name in ex_keys or exclude: logger.debug("%s excluded from upload" % key_name) else: k = Key(bucket=bucket, name=key_name) # Set custom headers for header, value in app.config['S3_HEADERS'].iteritems(): k.set_metadata(header, value) k.set_contents_from_filename(file_path) k.make_public() return new_hashes
def upload_to_s3(filename, key, bucket='dev.blibb'): c = boto.connect_s3() b = c.create_bucket(bucket) k = Key(b) k.key = key k.set_metadata('info_test', 'Imported from flask') k.set_contents_from_filename(app.config['UPLOAD_FOLDER'] + filename)
def get_key(self, url, create=False): import boto import boto.s3.connection from boto.s3.key import Key parsed = urlparse.urlparse(url) scheme = parsed.scheme self.hostname = parsed.netloc.partition(':')[0] self.port = int(parsed.netloc.partition(':')[2]) path = parsed.path.strip("/") self.__conn = boto.connect_s3( aws_access_key_id = self.access_key, aws_secret_access_key = self.secret_key, host = self.hostname, port = self.port, is_secure = self.is_secure, #False, # uncommmnt if you are not using ssl calling_format = boto.s3.connection.OrdinaryCallingFormat(), ) pos = path.index("/") bucket_name = path[:pos] key_name = path[pos+1:] if create: bucket = self.__conn.create_bucket(bucket_name) key = Key(bucket, key_name) key.set_metadata('mode',33188) else: bucket = self.__conn.get_bucket(bucket_name) key = bucket.get_key(key_name) return key
def put_backup_in_s3(self, type, filename): """Puts a backup file into s3""" k = Key(self.s3bucket) k.key = _get_backup_path(self.appid, type) _log.info("Putting %s into s3" % k.key) k.set_metadata("backup_date", script_start_time.strftime("%s")) k.set_contents_from_filename(filename)
def upload_geoaudio(): # API point for uploading geo-tagged sound clips. connection = mongo_conn() db = connection[os.environ['MONGODB_NAME']] audio = request.json['audio'] # Save the audio to S3 conn = S3Connection(os.environ['AWS_S3_KEY'], os.environ['AWS_S3_SECRET']) bucket = conn.create_bucket(os.environ['AWS_S3_BUCKET']) k = Key(bucket) k.key = str(uuid.uuid4()) k.set_metadata("Content-Type", 'audio/aac') k.set_contents_from_string(audio) # Get the geolocation data geoaudioData = { "loc": [float(request.json['longitude']), float(request.json['latitude'])], "s3_key": k.key } # Get the collection geoaudio = db.geoaudio # Insert it into the db geoaudio.insert(geoaudioData) # Return success return jsonify(result="Success")
def index(pin): s3_conn = S3Connection(AWS_KEY, AWS_SECRET) bucket = s3_conn.get_bucket('property-image-cache') s3_key = Key(bucket) s3_key.key = '{0}.jpg'.format(pin) if s3_key.exists(): output = BytesIO() s3_key.get_contents_to_file(output) else: image_viewer = 'http://www.cookcountyassessor.com/PropertyImage.aspx?pin={0}' image_url = image_viewer.format(pin) image = requests.get(image_url) print(image.headers) if 'image/jpeg' in image.headers['Content-Type']: output = BytesIO(image.content) s3_key.set_metadata('Content-Type', 'image/jpg') s3_key.set_contents_from_file(output) s3_key.set_acl('public-read') else: sentry.captureMessage('Could not find image for PIN %s' % pin) abort(404) output.seek(0) response = make_response(output.read()) response.headers['Content-Type'] = 'image/jpg' return response
def _upload_to_s3(file_to_upload, path, name): ''' Upload file to S3 using provided keyname. Returns: public_url: URL to access uploaded file ''' if settings.S3_HOST is None: conn = S3Connection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY) else: conn = S3Connection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY, host=settings.S3_HOST, calling_format=OrdinaryCallingFormat()) bucketname = settings.S3_BUCKET bucket = conn.create_bucket(bucketname) prefix = getattr(settings, 'S3_PATH_PREFIX') path = '{0}/{1}'.format(prefix, path) k = Key(bucket) k.key = '{path}/{name}'.format(path=path, name=name) public_url = k.generate_url(60*60*24*365) # URL timeout in seconds. k.set_metadata('filename', file_to_upload.name) k.set_contents_from_file(file_to_upload) return public_url
def output_s3(fmt, output, opts, version): try: from boto.s3.connection import S3Connection from boto.s3.key import Key except ImportError: print >> sys.stderr, "Unable to publish to S3: Boto not installed." return # Verify the S3 configuration bucket_name = opts.output_s3 access_key = opts.aws_access_key or os.environ.get("AWS_ACCESS_KEY_ID") secret_key = opts.aws_secret_key or os.environ.get("AWS_SECRET_ACCESS_KEY") if not access_key or not secret_key: print >> sys.stderr, "We need an AWS access key and AWS secret key" return conn = S3Connection(access_key, secret_key) bucket = conn.get_bucket(bucket_name) k = Key(bucket) if version == 2: file_name = "regions." else: file_name = "regions-v" + version + "." k.key = file_name + fmt # Set a content type content_types = {"json": "application/json", "xml": "text/xml"} if fmt in content_types: k.set_metadata("Content-Type", content_types[fmt]) print "Writing %s/%s" % (bucket_name, k.key) k.set_contents_from_string(output)
def save_file_in_s3(filename, subdirectory, env, gzipped_file_path): if env == 'staging': BUCKET_NAME = STAGE_BUCKET_NAME else: BUCKET_NAME = PROD_BUCKET_NAME if subdirectory != '': remote_filepath = WP_PATH + subdirectory + '/' else: remote_filepath = WP_PATH + '/' print 'uploading -- %s -- to -- %s --' % (filename, BUCKET_NAME + ' : ' + remote_filepath) #set headers # css -> content-type: text/css, content-encoding: gzip # js -> content-type: application/javascript, content-encoding: gzip conn = S3Connection(ACCESS_KEY, SECRET) bucket = conn.get_bucket(BUCKET_NAME) k = Key(bucket) k.key = remote_filepath + filename k.set_metadata('Content-Encoding', 'gzip') k.set_contents_from_filename(gzipped_file_path) k.make_public() print '**** Deleting ' + gzipped_file_path + '****' os.remove(gzipped_file_path)
def s3_create_default_thumbs_for(item): """Create copies of the default thumbs for the given item. This copies the default files (all named with an id of 'new') to use the given item's id. This means there could be lots of duplicate copies of the default thumbs, but at least we can always use the same url when rendering. :param item: A 2-tuple with a subdir name and an ID. If given a ORM mapped class with _thumb_dir and id attributes, the info can be extracted automatically. :type item: ``tuple`` or mapped class instance """ # fallback to normal thumb handling if no S3 engine is enabled. storage = get_s3_storage() if not storage: return None bucket = storage.connect_to_bucket() image_dir, item_id = normalize_thumb_item(item) for key in config['thumb_sizes'][image_dir].iterkeys(): src_file = os.path.join(config['cache.dir'], 'images', thumb_path((image_dir, 'new'), key)) dst_file = thumb_path(item, key) key = Key(bucket) key.key = dst_file key.set_metadata('is_default_thumb', '1') key.set_contents_from_filename(src_file, {'Content-Type': 'image/jpeg'}) key.set_acl('public-read') return True
def s3_upload(filename, label=None, content_disposition=None): '''Upload a file to the Amazon S3 bucket configured in local settings. (Should be configured to auto-expire after 24 hours.) :param filename: full path to file to be uploaded :param label: optional file name to use for the download; if not specified, will be generated from filename basename :param content_disposition: optional content-disposition header value, e.g. to prompt download of a type that could be displayed in the browser (e.g. xml) ''' s3_conn = S3Connection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY) s3_bucket = s3_conn.get_bucket(settings.AWS_S3_BUCKET) key = Key(s3_bucket) # use base filename as label, if no label is specified if label is None: label = basename(filename) key.key = label if content_disposition is not None: key.set_metadata('Content-Disposition', content_disposition) # NOTE: if zip file exports get very large (e.g. when including # images or deep zoom), this will need to be converted to a # multi-part upload # for now, log file size so we can check on errors filesize = os.stat(filename).st_size logger.info('File %s to upload to S3 is %s (%sb)', filename, filesizeformat(filesize), filesize) key.set_contents_from_filename(filename) # make the file publicly readable key.set_acl('public-read') return 'https://s3.amazonaws.com/%s/%s' % \ (settings.AWS_S3_BUCKET, label)
def _save_to_s3(self, data): assert len(data) > 0, 'Need data to save!' access_key = config.get_required('AWS_ACCESS_KEY_ID') secret_key = config.get_required('AWS_SECRET_ACCESS_KEY') bucket_name = config.get_required('MESSAGE_STORE_BUCKET_NAME') # Boto pools connections at the class level conn = S3Connection(access_key, secret_key) bucket = conn.get_bucket(bucket_name, validate=False) # See if it already exists and has the same hash data_obj = bucket.get_key(self.data_sha256) if data_obj: assert data_obj.get_metadata('data_sha256') == self.data_sha256, \ "Block hash doesn't match what we previously stored on s3!" return data_obj = Key(bucket) # if metadata: # assert type(metadata) is dict # for k, v in metadata.iteritems(): # data_obj.set_metadata(k, v) data_obj.set_metadata('data_sha256', self.data_sha256) # data_obj.content_type = self.content_type # Experimental data_obj.key = self.data_sha256 # log.info("Writing data to S3 with hash {0}".format(self.data_sha256)) # def progress(done, total): # log.info("%.2f%% done" % (done/total * 100) ) # data_obj.set_contents_from_string(data, cb=progress) data_obj.set_contents_from_string(data)
def store_file_from_memory(self, key, memstring, metadata=None): s3key = Key(self.bucket) s3key.key = self.format_key_for_backend(key) if metadata: for k, v in self.sanitize_metadata(metadata).items(): s3key.set_metadata(k, v) s3key.set_contents_from_string(memstring, replace=True)
def sync(self, source, target_name): upload_path = self.compress_data(source, target_name) if self.gpg_binary and self.encrypt_command: upload_path = self.encrypt_file(upload_path) print upload_path now = self.now() now_str = now.strftime('%Y-%m-%dT%H:%M:%S') name_parts = target_name.split('.') if len(name_parts) > 1: new_name = name_parts[:-1] new_name.append(now_str) new_name.append(name_parts[-1]) if self.compress: new_name.append('bz2') else: new_name = name_parts new_name.append(now_str) if self.compress: new_name.append('bz2') target_name = u'.'.join(new_name) bucket = self.get_bucket() key = Key(bucket) key.key = os.path.join(self.path, target_name) logger.info('Uploading to %s' % key.key) key.set_metadata('created', now_str) key.set_contents_from_filename(upload_path) key.set_acl('private')
def add(): image = request.POST.get("photo") name = request.POST.get("name") common_name = request.POST.get("common_name") plant = Plant.retrieve(request.db, name) if plant: return "This species is already in the DB" if image is not None: mime = mimetypes.guess_type(image.filename)[0] conn = S3Connection("AKIAIMXIHJX3TFDQFVCA", "Lf7xWpeOB9mnY1zfFzl7WNtxtNhmCZ4ZXOI8Kvrr") bucket = conn.get_bucket("db_leaves") key = Key(bucket) key.key = name key.set_metadata("Content-Type", mime) key.set_contents_from_string(image.value) key.set_acl("public-read") descriptors, ax, bx, ay, by = EFD(Threshold(image.file).process(), 50, 100).fourier_coefficients() return Plant( { "name": name, "common_name": common_name, "wiki": request.POST.get("wiki"), "photo": "https://s3.amazonaws.com/db_leaves/%s" % quote(name), "descriptors": descriptors, } ).save(request.db) return []
def upload_to_s3(file_path, path, name): """ Upload file to S3 using provided keyname. Returns: public_url: URL to access uploaded file """ try: conn = S3Connection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY) bucketname = settings.S3_BUCKETNAME try: bucket = conn.create_bucket(bucketname.lower()) except Exception: bucket = conn.get_bucket(bucketname.lower()) prefix = getattr(settings, "S3_PATH_PREFIX") path = u"{0}/{1}".format(prefix, path) key = u"{path}/{name}".format(path=removeNonAscii(path), name=removeNonAscii(name)) k = Key(bucket) k.key = key k.set_contents_from_filename(file_path) k.set_acl("public-read") k.set_metadata("filename", removeNonAscii(name)) public_url = k.generate_url(60 * 60 * 24 * 365) # URL timeout in seconds. return True, public_url except Exception: error = "Could not connect to S3." log.exception(error) return False, error
def download_jobs(geocoder): """ Download and submit jobs from S3. """ logging.info('Downloading jobs') awaiting_folder = 'geocode_awaiting_submission' pending_folder = 'geocode_pending_jobs' connection = boto.connect_s3() bucket = connection.get_bucket(GEO_BUCKET) files = bucket.list('%s' % awaiting_folder) for f in files: try: name = f.name.replace('%s/' % awaiting_folder, '') fkey = bucket.get_key(f.name) email_address = fkey.get_metadata('email') if name: logging.info('Uploading %s to Bing' % name) job_id = geocoder.upload_address_batch(fkey.get_contents_as_string()) if job_id: logging.info('Moving batch with old id %s to new id %s in %s' % ( name, job_id, pending_folder)) new_key = Key(bucket) new_key.key = '%s/%s' % (pending_folder, job_id) if email_address: logging.info('Setting metadata to %s' % email_address) new_key.set_metadata('email', email_address) send_email_notification(email_address, {}, name, 'pending') new_key.set_contents_from_string(name) old_key = Key(bucket) old_key.key = '%s/%s' % (awaiting_folder, name) old_key.delete() else: send_email_notification(email_address, {}, name, 'error') except Exception, e: logging.warning('Error uploading %s to Bing: %s' % (name, e))
def upload(request, bucket, key, data): if not key or key.endswith('/'): raise ValueError('key required for upload') perms = request.session['perms'] (can_read, can_write) = has_permission(request.user, perms, bucket, key) if not can_write: return HttpResponseForbidden('%s does not have access to %s/%s' % (request.user.email, bucket, key or '')) b = request.s3.get_bucket(bucket) k = Key(bucket=b, name=key) if k.exists() and 'force' not in request.GET: raise HttpResponseForbidden('write failed: file exists') headers = { "x-amz-acl": "public-read", "Content-Length": len(data), } ct = mimetypes.guess_type(key)[0] if ct is not None: headers["Content-Type"] = ct k.set_contents_from_string(data, headers=headers) k.set_metadata('uploaded-by', request.user.email) if request.is_ajax(): return HttpResponse('{}') else: return redirect('/' + '/'.join(path.split('/')[:-1]))
def store_in_s3(filename, content): conn = S3Connection( config_value("S3_API_KEY"), config_value("S3_API_SECRET"),) bucket = conn.create_bucket( config_value('S3_BUCKET') ) k = Key(bucket) # create key on this bucket k.key = filename mime = mimetypes.guess_type(filename)[0] k.set_metadata('Content-Type', mime) k.set_contents_from_filename(content) k.set_acl('public-read')
def test_normal_object_metadata(self): key_name = str(uuid.uuid4()) bucket = self.conn.create_bucket(self.bucket_name) key = Key(bucket, key_name) for k, v in self.metadata.items(): key.set_metadata(k, v) key.set_contents_from_string("test_normal_object_metadata") self.assert_metadata(bucket, key_name) self.change_metadata(bucket, key_name) self.assert_updated_metadata(bucket, key_name)
def upload_to_s3(aws_access_key_id, aws_secret_access_key, file, bucket, key, sub_directory, callback=None, md5=None, reduced_redundancy=False, content_type=None): """ Uploads the given file to the AWS S3 bucket and key specified. callback is a function of the form: def callback(complete, total) The callback should accept two integer parameters, the first representing the number of bytes that have been successfully transmitted to S3 and the second representing the size of the to be transmitted object. Returns boolean indicating success/failure of upload. """ try: size = os.fstat(file.fileno()).st_size except: # Not all file objects implement fileno(), # so we fall back on this file.seek(0, os.SEEK_END) size = file.tell() conn = boto.connect_s3(aws_access_key_id, aws_secret_access_key) bucket = conn.get_bucket(bucket, validate=True) k = Key(bucket) if sub_directory != "": k.key = os.path.join(sub_directory, key) else: k.key = key if content_type: k.set_metadata('Content-Type', content_type) sent = k.set_contents_from_file(file, cb=callback, md5=md5, reduced_redundancy=reduced_redundancy, rewind=True) # Rewind for later use file.seek(0) if sent == size: return True return False
def save_s3(bucket, filename, contents, systemfile, content_type=None, acl='public-read', meta=None, encode=None): from boto.dynamodb2.table import Item key = Key(bucket, filename) print 'new s3 key:', 'http://s3.amazonaws.com/' + bucket.name + ( key.name if key.name.startswith('/') else '/' + key.name) if isinstance(meta, Item): meta = meta._data if isinstance(meta, dict): trim_meta = fixed.check_entity_size(meta) trim_meta = dict([(k, value) for (k, value) in trim_meta.items() if value is not None and value]) trim_meta = json.loads(json.dumps(trim_meta, cls=fixed.SetEncoder)) print 'meta key length:', len(trim_meta.keys()) key.metadata = trim_meta if content_type is not None: print 'set content type:', content_type key.content_type = content_type elif systemfile and systemfile.endswith('js.map'): print 'js map!' key.content_type = 'application/json' elif systemfile: gt = mimetypes.guess_type(systemfile) key.set_metadata('Content-Type', gt[0]) if encode is not None and encode == 'gzip': key.set_metadata('Content-Encoding', 'gzip') gzmem = StringIO.StringIO() gzip_file = gzip.GzipFile(fileobj=gzmem, mode='w') if contents is not None: gzip_file.write(contents) elif systemfile is not None: with open(systemfile, 'rb') as outfile: gzip_file.write(outfile.read()) gzip_file.close() key.set_contents_from_string(gzmem.getvalue()) print 'gzip!' elif contents is not None: print 'from string' key.set_contents_from_string(contents) elif systemfile is not None: io = StringIO.StringIO(open(systemfile, 'r').read()).getvalue() print 'from disk:', systemfile, 'io:', len(io) key.set_contents_from_string(io) if acl is not None: print 'save acl:', acl key.set_acl(acl) print 'save complete:', key.name return key
def save_to_s3(self, image_str): expires = datetime.datetime.now() + datetime.timedelta(days=60) expires = expires.strftime("%a, %d %b %Y %H:%M:%S GMT") k = Key(settings.S3_ICONS_BUCKET) k.key = self.feed.s3_icons_key k.set_metadata('Content-Type', 'image/png') k.set_metadata('Expires', expires) k.set_contents_from_string(image_str.decode('base64')) k.set_acl('public-read') self.feed.s3_icon = True
def upload(filename): initalize_connection() bucket = conn.get_bucket("ninja-kpom-mailgun") k = Key(bucket) k.key = "{}.csv".format(uuid.uuid4()) k.set_contents_from_filename(filename) k.set_metadata('Content-Type', "application/csv") k.set_acl('public-read') return k.generate_url(expires_in=0, query_auth=False)
def s3_set_contents_from_file(path, file, private=1): ''' `file` is a file descriptor ''' bucket = s3.get_bucket(PRIVATE_BUCKET_NAME) if private else s3.get_bucket( PUBLIC_BUCKET_NAME) k = Key(bucket) k.key = path k.set_metadata('Content-Type', file.content_type) file.seek(0) return k.set_contents_from_file(file)
def test_delete(self): """ delete() should remove package from storage """ key = Key(self.bucket) name, version, path = 'mypkg', '1.2', 'path/to/file.tar.gz' key.key = path key.set_metadata('name', name) key.set_metadata('version', version) key.set_contents_from_string('foobar') self.storage.delete(key.key) new_key = self.bucket.get_key(key.key) self.assertIsNone(new_key)
def getDownloadableMediaMessageBody(self, message): # save as temp file #filename = "%s/%s%s"%(tempfile.gettempdir(),message.getId(),message.getExtension()) #with open(filename, 'wb') as f: # f.write(message.getMediaContent()) k = Key(self.b) k.key = "%s%s" % (message.getId(), message.getExtension()) k.set_contents_from_string(str(message.getMediaContent())) k.set_metadata('Content-Type', message.getMimeType()) k.set_acl('public-read') return k.generate_url(expires_in=0, query_auth=False)
def storeInS3(recipe_id, filename, content): conn = S3Connection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY) b = conn.create_bucket(settings.S3BUCKET) mime = mimetypes.guess_type(filename)[0] k = Key(b) k.key = 'recipe_id_' + str(recipe_id) + '_' + filename k.set_metadata("Content_Type", mime) content.seek(0) k.set_contents_from_file(content) k.set_acl("public-read")
def upload_chunk(i): chunk = df[i:(i + chunk_size)] k = Key(bucket) k.key = 'data-{}-{}-{}.csv.gz'.format(name, i, i + chunk_size) k.set_metadata('parent', 'db.py') out = StringIO() with gzip.GzipFile(fileobj=out, mode="w") as f: f.write(chunk.to_csv(index=False, encoding='utf-8')) k.set_contents_from_string(out.getvalue()) sys.stdout.write(".") return i
def upload_to_s3(upload_file, directory, callback=None, md5=None, reduced_redundancy=False, content_type=None): """ Uploads the given file to the AWS S3 bucket and key specified. callback is a function of the form: def callback(complete, total) The callback should accept two integer parameters, the first representing the number of bytes that have been successfully transmitted to S3 and the second representing the size of the to be transmitted object. Returns boolean indicating success/failure of upload. Borrowed from http://stackabuse.com/example-upload-a-file-to-aws-s3/. """ file = open(upload_file, 'r+') key = os.path.join(directory, os.path.split(file.name)[-1]) try: size = os.fstat(file.fileno()).st_size except: # Not all file objects implement fileno(), # so we fall back on this file.seek(0, os.SEEK_END) size = file.tell() conn = boto.connect_s3(SETTINGS['access_id'], SETTINGS['secret_key']) bucket = conn.get_bucket(SETTINGS['s3_bucket'], validate=True) k = Key(bucket) k.key = key if content_type: k.set_metadata('Content-Type', content_type) sent = k.set_contents_from_file(file, cb=callback, md5=md5, reduced_redundancy=reduced_redundancy, rewind=True) k.set_acl('public-read') # Rewind for later use file.seek(0) if sent == size: return True return False
def put(self, filedata, content_type, remote_path, force=False): now = datetime.datetime.utcnow() then = now + datetime.timedelta(self.expiration_days) expires = then.strftime("%a, %d %b %Y %H:%M:%S GMT") if self.aws_prefix: remote_path = "%s/%s" % (self.aws_prefix, remote_path) (hexdigest, b64digest) = mediasync.checksum(filedata) raw_b64digest = b64digest # store raw b64digest to add as file metadata # create initial set of headers headers = { "x-amz-acl": "public-read", "Content-Type": content_type, "Expires": expires, "Cache-Control": 'max-age=%d' % (self.expiration_days * 24 * 3600), } key = self._bucket.get_key(remote_path) if key is None: key = Key(self._bucket, remote_path) key_meta = key.get_metadata('mediasync-checksum') or '' s3_checksum = key_meta.replace(' ', '+') if force or s3_checksum != raw_b64digest: key.set_metadata('mediasync-checksum', raw_b64digest) key.set_contents_from_string(filedata, headers=headers, md5=(hexdigest, b64digest)) # check to see if file should be gzipped based on content_type # also check to see if filesize is greater than 1kb if content_type in TYPES_TO_COMPRESS: key = Key(self._bucket, "%s.gz" % remote_path) filedata = mediasync.compress(filedata) (hexdigest, b64digest) = mediasync.checksum( filedata) # update checksum with compressed data headers[ "Content-Disposition"] = 'inline; filename="%sgz"' % remote_path.split( '/')[-1] headers["Content-Encoding"] = 'gzip' key.set_metadata('mediasync-checksum', raw_b64digest) key.set_contents_from_string(filedata, headers=headers, md5=(hexdigest, b64digest)) return True
def test_list(self): """ Can construct a package from a S3 Key """ key = Key(self.bucket) name, version, path = 'mypkg', '1.2', 'path/to/file.tar.gz' key.key = path key.set_metadata('name', name) key.set_metadata('version', version) key.set_contents_from_string('foobar') package = list(self.storage.list(Package))[0] self.assertEquals(package.name, name) self.assertEquals(package.version, version) self.assertEquals(package.path, path)
def moni_s3_upload(key, data, bucket_path): from boto.s3.key import Key from boto.s3.connection import S3Connection conn = S3Connection(app_settings.S3_ACCESS_KEY, app_settings.S3_SECRET_KEY) bucket = conn.get_bucket(bucket_path) k = Key(bucket) k.key = key k.set_contents_from_file(data, rewind=True) k.set_acl('public-read') k.set_metadata('Content-Type', 'text/csv') return k.generate_url(expires_in=0, query_auth=False)
def upload_chunk(i): conn = S3Connection(AWS_ACCESS_KEY, AWS_SECRET_KEY) chunk = df[i:(i + chunk_size)] k = Key(bucket) k.key = 'data-%d-%d.csv.gz' % (i, i + chunk_size) k.set_metadata('parent', 'db.py') out = StringIO() with gzip.GzipFile(fileobj=out, mode="w") as f: f.write(chunk.to_csv(index=False, encoding='utf-8')) k.set_contents_from_string(out.getvalue()) sys.stderr.write(".") return i
def save_png(self, keyString, valueString): key = Key(self.bucket) key.key = keyString # good until one year from now, and public key.set_metadata('Cache-Control', self.cache_control) key.set_metadata('Content-Type', 'image/png') key.set_contents_from_string(valueString) key.make_public() # a non-expiring URL return key.generate_url(expires_in=0, query_auth=False)
def upload_s3_keys(s3conn, source, bucket_name, prefix, meta): start = time.time() bucket = s3conn.get_bucket(bucket_name, validate=False) k = Key(bucket) k.key = prefix for m in meta: k.set_metadata(m, meta[m]) k.set_contents_from_filename(source) k.set_metadata('time', "foo") return time.time() - start
def save_schema(self, schema): account = Account.objects.get(visualization__query=self.query) conn = S3Connection( aws_access_key_id=account.aws_access_key_id, aws_secret_access_key=account.aws_secret_access_key) bucket = conn.get_bucket('lx-pilot') key = Key(bucket) key.key = self.schema_key() key.set_metadata('Content-Type', 'application/json') key.set_metadata('Content-Encoding', 'gzip') key.set_contents_from_string( gzip.compress(bytes(json.dumps(schema), 'utf-8')))
def push_file_to_s3(bucket, local_file, metadata=None): if not CloudHelper.ENABLED['s3']: return None key_name = os.path.basename(local_file) k = Key(CloudHelper.connect_s3_bucket(bucket)) k.key = key_name if metadata is not None: for meta_name, meta_value in metadata.iteritems(): k.set_metadata(meta_name, meta_value) k.set_contents_from_filename(local_file) return k
def upload(self, local, remote, ignoreMissing=False, force=False, hash=None): # maybe upload and download should use trailing slash to indicate directory should be uploaded instead of just a file assert not remote.startswith("/") # assert not local.startswith("/") remote_path = os.path.normpath(self.remote_path + "/" + remote) local_path = os.path.normpath(os.path.join(self.local_dir, local)) # cope when case where local was passed as an abs path # local = os.path.relpath(local, self.local_dir) # assert not local.startswith("."), "local={}, local_dir={}".format(local, self.local_dir) # local_path = local uploaded_url = None if os.path.exists(local_path): if os.path.isfile(local_path): # if it's a file, upload it uploaded_url = "s3://" + self.bucket.name + "/" + remote_path if self.bucket.get_key(remote_path) is None or force: key = Key(self.bucket) key.name = remote_path log.info("Uploading file %s to %s", local, uploaded_url) key.set_contents_from_filename(local_path) if hash is None: hash = calc_hash(local_path) key.set_metadata("sha256", hash) else: # upload everything in the dir assert hash is None for fn in os.listdir(local_path): full_fn = os.path.join(local_path, fn) if os.path.isfile(full_fn): r = os.path.join(remote_path, fn) if self.bucket.get_key(r) is None or force: k = Key(self.bucket) k.key = r log.info("Uploading dir %s (%s to %s)", local_path, fn, fn) k.set_contents_from_filename(full_fn) hash = calc_hash(local_path) k.set_metadata("sha256", hash) else: log.info( "Uploading dir %s (%s to %s), skiping existing file", local_path, fn, fn) elif not ignoreMissing: raise Exception("Could not find {}".format(local)) return uploaded_url
def test_write_over_key_with_meta(self): """ test that metadata does not persist when a key is written over """ key_name = "test-key" test_string = os.urandom(1024) test_string_1 = os.urandom(1024) meta_key = "meta_key" meta_value = "pork" # create the bucket bucket = self._s3_connection.create_unique_bucket() self.assertTrue(bucket is not None) # create an empty key write_key = Key(bucket) # set the name write_key.name = key_name # self.assertFalse(write_key.exists()) # set some metadata write_key.set_metadata(meta_key, meta_value) # upload some data write_key.set_contents_from_string(test_string) self.assertTrue(write_key.exists()) # create another key to write over the first key write_key1 = Key(bucket, key_name) # upload some data write_key1.set_contents_from_string(test_string_1) self.assertTrue(write_key.exists()) # create another key with the same name read_key = Key(bucket, key_name) # read back the data returned_string = read_key.get_contents_as_string() self.assertEqual(returned_string, test_string_1) # get the metadata returned_meta_value = read_key.get_metadata(meta_key) self.assertEqual(returned_meta_value, None) # delete the key read_key.delete() self.assertFalse(write_key.exists()) # delete the bucket self._s3_connection.delete_bucket(bucket.name)
def upload_file(gallery, bucket, f): logger.debug("Uploading file %s" % (f)) key = Key(bucket) key.key = f cache_metadata = generate_cache_metadata(gallery, f) if cache_metadata: key.set_metadata('Cache-Control', cache_metadata) key.set_contents_from_filename( os.path.join(gallery.settings['destination'], f), policy=gallery.settings['upload_s3_options']['policy'])
def _upload_s3(datafile, key_id, access_key, bucket_name, key): with open(datafile) as f: md5 = compute_md5(f) conn = boto.connect_s3(key_id, access_key) bucket = conn.get_bucket(bucket_name, validate=False) k = Key(bucket) k.key = key k.set_metadata("Content-Type", "application/json") k.set_contents_from_filename(datafile, md5=md5, replace=True) return "s3://%s/%s" % (bucket_name, k.key)
def uploadEpubS3(self, book_file, file_dir): conn = S3Connection(s3access, s3secret) bucket = conn.get_bucket(S3BUCKET) k = Key(bucket) key_name = 'epubs/' + book_file.filename k.key = key_name k.set_metadata('Content-Type', 'application/epub+zip') k.set_contents_from_string(book_file.read()) return key_name
def test_copy_key_replace_metadata(): conn = boto.connect_s3('the_key', 'the_secret') bucket = conn.create_bucket("foobar") key = Key(bucket) key.key = "the-key" key.set_metadata('md', 'Metadatastring') key.set_contents_from_string("some value") bucket.copy_key('new-key', 'foobar', 'the-key', metadata={'momd': 'Mometadatastring'}) bucket.get_key("new-key").get_metadata('md').should.be.none bucket.get_key("new-key").get_metadata('momd').should.equal('Mometadatastring')
def post(self, key, data): # the new object to persist obj = Key(self.bucket) # get the previous version, if any previous = self.get(key, None) if previous: obj.set_metadata('previous_version_id', previous.version_id) # persist the new object obj.key = key obj.set_contents_from_string(json.dumps(data))
def upload(self, name, version, filename, data): key = Key(self.bucket) if self.prepend_hash: m = md5() m.update(name) m.update(version) prefix = m.digest().encode('hex')[:4] filename = prefix + '/' + filename key.key = self.bucket_prefix + filename key.set_metadata('name', name) key.set_metadata('version', version) key.set_contents_from_file(data) return key.key
def uploadMetadata(setting, nodeid, metadatapath, dest): '''Upload metadata to cloud.''' #S3connection: conn = connectCloud(setting, nodeid) bucketname = setting.nodeInfo[nodeid].bucketname #Upload metadata to bucket: bucket = conn.get_bucket(bucketname) k = Key(bucket) k.key = dest fobj = open(metadatapath, 'rb') k.set_metadata('0', base64.urlsafe_b64encode(fobj.read())) fobj.close() return True
def set_metadata(): """Take a list of files to be uploaded to s3 and gzip CSS, JS, and HTML, setting metadata for all files including an 'expires' header defined at the beginning of the file. HTML expires after 1 hour.""" s3_list = s3_filename() conn = S3Connection(AWS_KEY, AWS_SECRET_KEY, calling_format=OrdinaryCallingFormat()) mybucket = conn.get_bucket(AWS_BUCKET) expires = time.time() + STATIC_EXPIRES expires_header = time.strftime("%a, %d-%b-%Y %T GMT", time.gmtime(expires)) for filename in s3_list: k = Key(mybucket) ext = os.path.splitext(filename)[1] if ext == '': ext = '.html' if ext == '.html': # deletes '.html' from s3 key so no ext on url local_name = os.path.splitext(filename)[0] if local_name == 'index': local_name = '/index.html' if local_name[0] != '/': # if file within child dir k.key = AWS_DIRECTORY + '/' + local_name else: # if file in top level dir k.key = AWS_DIRECTORY + local_name k.set_metadata('Expires', time.time() + HTML_EXPIRES) else: k.key = AWS_DIRECTORY + '/' + filename # strip leading 0 k.set_metadata('Expires', expires_header) if ext == '.css' or ext == '.js' or ext == '.html': build_file = PUSH_FROM + filename f_in = open(build_file, 'rb') with gzip.open(build_file + '.gz', 'w+') as f: f.writelines(f_in) f_in.close() f = build_file + '.gz' k.set_metadata('Content-Encoding', 'gzip') else: f = PUSH_FROM + filename k.set_metadata('Content-Type', content_types[ext]) etag_hash = hashlib.sha1(f + str(time.time())).hexdigest() k.set_metadata('ETag', etag_hash) k.set_contents_from_filename(f) k.make_public() print '\nPage successfully updated' print "On " + datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')