def ned_available(polyfilename): polyfile = open(polyfilename, 'r') tiffset = set() for line in polyfile: if re.match("^[ \t]*[0-9.\-]+[ \t]+[0-9.\-]+[ \t]*$", line) is not None: arr = re.split("[ \t]+",line) line = re.sub("^[ \t]+", '', line[:-1]) arr = re.split("[ \t]+",line) x = math.floor(float(arr[0])) y = math.ceil(float(arr[1])) nsdir = 'n' if y > 0 else 's' ewdir = 'e' if x > 0 else 'w' tiff_file = "%s%02d%s%03d.tiff" % (nsdir, abs(y), ewdir, abs(x)) tiffset.add(tiff_file) polyfile.close connection = connect_s3(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_KEY) bucket = connection.get_bucket('ned13') all_exist = True for tiff_file in tiffset: key = Key(bucket) key.key = tiff_file print "%s exists: %s" % (tiff_file, key.exists()) all_exist = all_exist and key.exists() return all_exist
def upload2s3(): conn = S3Connection(AWS_ACCESS_KEY, AWS_SECRET_KEY) now = datetime.now() sevendaysbefore = now - timedelta(days=7) try: print 'createing bucket' bucket = conn.create_bucket('mongodbdump') print 'get key' k = Key(bucket) k.key = sevendaysbefore.date().isoformat() if k.exists(): print 'delete key', k.key k.delete() k.key = now.date().isoformat() if k.exists(): print 'delete key', k.key k.delete() options = mock.Mock() options.concurrency = 20 options.reduced_redundancy = False options.bucket = "mongodbdump" options.path = "." options.files = [ DUMP_FILE ] upload(options) except Exception, e: traceback.print_exc()
def test_key_with_strings(self): """ test simple key 'from_string' and 'as_string' functions """ key_name = "test-key" test_string = os.urandom(1024) # create the bucket bucket = self._s3_connection.create_unique_bucket() self.assertTrue(bucket is not None) # create an empty key write_key = Key(bucket) # set the name write_key.name = key_name # self.assertFalse(write_key.exists()) # upload some data write_key.set_contents_from_string(test_string) self.assertTrue(write_key.exists()) # create another key with the same name read_key = Key(bucket, key_name) # read back the data returned_string = read_key.get_contents_as_string() self.assertEqual(returned_string, test_string, (len(returned_string), len(test_string))) # delete the key read_key.delete() self.assertFalse(write_key.exists()) # delete the bucket self._s3_connection.delete_bucket(bucket.name)
def rename_package(self, package_old_name, package_new_name): package_old_key = Key(self.bucket, package_old_name) package_new_key = Key(self.bucket, package_new_name) if package_old_key.exists() and (not package_new_key.exists()): package_old_key.copy(self.bucket, package_new_key) if package_new_key.exists(): package_old_key.delete() return
def test_key_with_files_and_callback(self): """ test simple key 'from_file' and 'to_file' functions """ def _archive_callback(bytes_sent, total_bytes): print("archived {0} out of {1}".format(bytes_sent, total_bytes)) def _retrieve_callback(bytes_sent, total_bytes): print("retrieved {0} out of {1}".format(bytes_sent, total_bytes)) log = logging.getLogger("test_key_with_files") key_name = "A" * 1024 test_file_path = os.path.join(test_dir_path, "test_key_with_files-orignal") test_file_size = 1024 ** 2 buffer_size = 1024 log.debug("writing {0} bytes to {1}".format(test_file_size, test_file_path)) bytes_written = 0 with open(test_file_path, "wb") as output_file: while bytes_written < test_file_size: output_file.write(os.urandom(buffer_size)) bytes_written += buffer_size # create the bucket bucket = self._s3_connection.create_unique_bucket() self.assertTrue(bucket is not None) # create an empty key write_key = Key(bucket) # set the name write_key.name = key_name self.assertFalse(write_key.exists()) # upload some data with open(test_file_path, "rb") as archive_file: write_key.set_contents_from_file(archive_file, cb=_archive_callback) self.assertTrue(write_key.exists()) # create another key with the same name read_key = Key(bucket, key_name) # read back the data retrieve_file_path = os.path.join(test_dir_path, "test_key_with_files-orignal") # 2011-08-08 dougfort boto aborts if you don't tell it the size read_key.size = test_file_size with open(retrieve_file_path, "wb") as retrieve_file: read_key.get_contents_to_file(retrieve_file, cb=_retrieve_callback) self.assertTrue(filecmp.cmp(test_file_path, retrieve_file_path, shallow=False)) # delete the key read_key.delete() self.assertFalse(write_key.exists()) # delete the bucket self._s3_connection.delete_bucket(bucket.name)
def test_write_over_key_with_meta(self): """ test that metadata does not persist when a key is written over """ key_name = "test-key" test_string = os.urandom(1024) test_string_1 = os.urandom(1024) meta_key = "meta_key" meta_value = "pork" # create the bucket bucket = self._s3_connection.create_unique_bucket() self.assertTrue(bucket is not None) # create an empty key write_key = Key(bucket) # set the name write_key.name = key_name # self.assertFalse(write_key.exists()) # set some metadata write_key.set_metadata(meta_key, meta_value) # upload some data write_key.set_contents_from_string(test_string) self.assertTrue(write_key.exists()) # create another key to write over the first key write_key1 = Key(bucket, key_name) # upload some data write_key1.set_contents_from_string(test_string_1) self.assertTrue(write_key.exists()) # create another key with the same name read_key = Key(bucket, key_name) # read back the data returned_string = read_key.get_contents_as_string() self.assertEqual(returned_string, test_string_1) # get the metadata returned_meta_value = read_key.get_metadata(meta_key) self.assertEqual(returned_meta_value, None) # delete the key read_key.delete() self.assertFalse(write_key.exists()) # delete the bucket self._s3_connection.delete_bucket(bucket.name)
def test_key_with_files(self): """ test simple key 'from_file' and 'to_file' functions """ log = logging.getLogger("test_key_with_files") key_name = "A" * 1024 test_file_path = os.path.join(test_dir_path, "test_key_with_files-orignal") test_file_size = 1024**2 buffer_size = 1024 log.debug("writing {0} bytes to {1}".format(test_file_size, test_file_path)) bytes_written = 0 with open(test_file_path, "wb") as output_file: while bytes_written < test_file_size: output_file.write(os.urandom(buffer_size)) bytes_written += buffer_size # create the bucket bucket = self._s3_connection.create_unique_bucket() self.assertTrue(bucket is not None) # create an empty key write_key = Key(bucket) # set the name write_key.name = key_name self.assertFalse(write_key.exists()) # upload some data with open(test_file_path, "rb") as archive_file: write_key.set_contents_from_file(archive_file) self.assertTrue(write_key.exists()) # create another key with the same name read_key = Key(bucket, key_name) # read back the data retrieve_file_path = os.path.join(test_dir_path, "test_key_with_files-orignal") with open(retrieve_file_path, "wb") as retrieve_file: read_key.get_contents_to_file(retrieve_file) self.assertTrue( filecmp.cmp(test_file_path, retrieve_file_path, shallow=False)) # delete the key read_key.delete() self.assertFalse(write_key.exists()) # delete the bucket self._s3_connection.delete_bucket(bucket.name)
def s3_delete(aws_access_key_id, aws_secret_access_key, bucket, key): try: conn = boto.connect_s3(aws_access_key_id, aws_secret_access_key, is_secure=False, calling_format=OrdinaryCallingFormat()) bucket = conn.get_bucket(bucket, validate=True) k = Key(bucket=bucket, name=key) if k.exists(): k.delete() if k.exists(): return False else: return True except Exception as e: info(e) return False
def updateStatusFile(completedFile): dst_s3conn = boto.connect_s3(DST_AWS_ACCESS_KEY, DST_AWS_SECRET_KEY) dst_bucket = dst_s3conn.get_bucket(DST_PATH[:DST_PATH.index('/')]) status_file_path = "%s/%s" % (completedFile.rsplit('/', 1)[0],PROCESSING_STATUS_FILE) status_file_key = Key(dst_bucket, status_file_path) theCompletedFile = completedFile.rsplit('/', 1)[1].strip() try: if not status_file_key.exists(): if len(completionListVerify) > 0: print("Seeking Verification for %s, but this directory has no status file... rerun scheduler to delete / restart processing please. I'll end now on this directory" % completedFile) return print ("WARN: failed to retrieve file \"%s\", starting new key." % status_file_path) status_file_key.set_contents_from_string(theCompletedFile) else: status_file_text = bytes(status_file_key.get_contents_as_string()).decode(encoding='UTF-8') #everything has been done if we hvae a completionList object, so validate this if len(status_file_text) < 3: new_status_file_text = theCompletedFile else: new_status_file_text = "%s\n%s" % (status_file_text, theCompletedFile) status_file_key.set_contents_from_string(new_status_file_text) print("Updated Status file with latest data, file %s" % theCompletedFile) except: print("Exception trying to update Status file with dir \"%s\", trying again" % completedFile) updateStatusFile(completedFile) #validate status file contents status_file_text = bytes(status_file_key.get_contents_as_string()).decode(encoding='UTF-8') if theCompletedFile in status_file_text: return else: print("The file \"%s\" didn't get outputted to the status file, trying again") updateStatusFile(completedFile) dst_s3conn.close()
def list(self, index=None, limit=10): """Returns a list of entries starting at the specific index""" dbres = self.table.scan(max_results=limit, exclusive_start_key=index) res = [] for dbr in dbres: if ('thumbnail' in dbr): pass else: # thumbnail is not present in DB logging.info("thumbnail does not exist") rawname = dbr['resource'] t_video_name = rawname + '-tx.mp4' thumb_name = rawname + '-00003.png' obj = Key(self.bucket) obj.key = thumb_name # if thumbnails exists in self.bucket update thumbnail and resource if (obj.exists()): # delete the other three thumbnails and source for video: self.bucket.delete_key(rawname+'-00001.png') self.bucket.delete_key(rawname+'-00002.png') self.bucket.delete_key(rawname+'-00004.png') self.bucket.delete_key(rawname) # update resource and thumbnail dbr['resource'] = t_video_name dbr['thumbnail'] = thumb_name self.update(dbr) res.append(dbr) return res
def get_from_s3(self, doc_id): """ Returns a response with the XML of the parsed text :param doc_id: the id of the document in Solr :type doc_id: str :return: json response :rtype: dict """ try: bucket = get_s3_bucket() key = Key(bucket) key.key = 'xml/%s/%s.xml' % tuple(doc_id.split('_')) if key.exists(): response = { 'status': 200, doc_id: key.get_contents_as_string() } else: response = {'status': 500, 'message': 'Key does not exist'} return response except socket.error: # probably need to refresh our connection global S3_BUCKET S3_BUCKET = None return self.get_from_s3(doc_id)
def get_from_s3(self, doc_id): """ Returns a response with the XML of the parsed text :param doc_id: the id of the document in Solr :type doc_id: str :return: json response :rtype: dict """ try: bucket = get_s3_bucket() key = Key(bucket) key.key = 'xml/%s/%s.xml' % tuple(doc_id.split('_')) if key.exists(): response = {'status': 200, doc_id: key.get_contents_as_string()} else: response = {'status': 500, 'message': 'Key does not exist'} return response except socket.error: # probably need to refresh our connection global S3_BUCKET S3_BUCKET = None return self.get_from_s3(doc_id)
def render_resource(key): key = Key(bucket=app.bucket, name=key) if not key.exists(): abort(404) name = key.name.strip('/').split('/')[-1] key.open() key.name = None resp = send_file(key, mimetype=key.content_type, attachment_filename=name, as_attachment=True) adname = name.encode('utf8') if isinstance(name, unicode) else name advalue = adler32(adname) & 0xffffffff resp.content_length = key.size resp.last_modified = time.strptime(key.last_modified, '%a, %d %b %Y %H:%M:%S %Z') resp.set_etag('flask-%s-%s-%s' % (key.last_modified, key.size, advalue)) return resp
def download_from_s3(aws_access_key_id, aws_secret_access_key, bucket, fname, key, dry_run=False, host='s3.amazonaws.com'): """Download file from bucket """ switch_validation = False if host is not None and not isinstance( host, boto.s3.connection.NoHostProvided): if 'eu-central' in host: switch_validation = True os.environ['S3_USE_SIGV4'] = 'True' com = boto.connect_s3(aws_access_key_id, aws_secret_access_key, host=host) bucket = com.get_bucket(bucket, validate=False) my_key = Key(bucket) my_key.key = key out = False if my_key.exists(): if not dry_run: s3fid = bucket.get_key(key) s3fid.get_contents_to_filename(fname) out = True else: return True else: print('could not get %s : it does not exist' % key) out = False if switch_validation: del os.environ['S3_USE_SIGV4'] return out
def GET(self): login = check_login(self.s3conn, self.mc, cherrypy.url()) b = self.s3conn.create_bucket('openid.amp.fm') k = Key(b) k.key = '%s/trusted' % login trusted = None if k.exists(): trusted = k.get_contents_as_string() if trusted: trusted = trusted.split('\r\n') body = """<html> <head /> <body> <p>You can revoke the following trusted parties:</p> <form name="revoke_trust" action="%s/revoke" method="post">""" % BASE_SECURE_URL pos = 0 for trust in trusted: if trust: body = body + """<input type="checkbox" name="trustee%d" value="%s">%s</input><br />""" % (pos, quote(trust), trust) pos += 1 return body + """<br /><input type="submit" value="Revoke" /></form></body></html>""" return "You have no trustees to revoke"
def POST(self, login, password, confirm_password): cherrypy.response.headers['content-type'] = 'application/xml' if login in _reserved_login: error_body = file(os.path.join(current_dir, 'signupPOSTerror.xml')).read() return error_body % "This login cannot be used" if password != confirm_password: error_body = file(os.path.join(current_dir, 'signupPOSTerror.xml')).read() return error_body % "Passwords do not match" # Here we suppose that the best acceptable charset for the client # is also the one it used to encode the data sent charset = extract_best_charset() login = quote(login.decode(charset).encode('utf-8')) oid = login b = self.s3conn.create_bucket('openid.amp.fm') k = Key(b) k.key = oid if k.exists() == True: error_body = file(os.path.join(current_dir, 'signupPOSTerror.xml')).read() return error_body % "This login is already used" k.set_contents_from_string(md5.new(password).hexdigest()) #return "Your identifier is %s" % urljoin(BASE_URL, oid) success_body = file(os.path.join(current_dir, 'signupPOSTsuccess.xml')).read() return success_body % urljoin(BASE_URL, oid)
def delete_packetage(self, package_name): package_key = Key(self.bucket, package_name) if package_key.exists(): package_key.delete() else: raise ValueError('package:%s are not exist' % package_name) return
def download_from_s3(aws_access_key_id, aws_secret_access_key, bucket, fname, key, dry_run=False, host='s3.amazonaws.com'): """Download file from bucket """ switch_validation = False if host is not None and not isinstance(host, boto.s3.connection.NoHostProvided): if 'eu-central' in host: switch_validation = True os.environ['S3_USE_SIGV4'] = 'True' com = boto.connect_s3(aws_access_key_id, aws_secret_access_key, host=host) bucket = com.get_bucket(bucket, validate=False) my_key = Key(bucket) my_key.key = key out = False if my_key.exists(): if not dry_run: s3fid = bucket.get_key(key) s3fid.get_contents_to_filename(fname) out = True else: return True else: print('could not get %s : it does not exist' % key) out = False if switch_validation: del os.environ['S3_USE_SIGV4'] return out
def download_from_s3(bucket, key, target, mock=False, overwrite=False, verbose=0): """Download file from bucket """ my_key = Key(bucket) my_key.key = key if my_key.exists(): s3fid = bucket.get_key(key) if not mock: if not os.path.exists(target) or overwrite: if verbose: print('Downloading %s from %s' % (target, key)) s3fid.get_contents_to_filename(target) name, ext = os.path.splitext(target) if ext == '.gz': try: _ = nibabel.load(target).get_shape() if verbose: print('Nifti consistency checked.') except: raise ConnectionError('Corrupted download') else: if verbose: print('Skipping %s as it already exists' % target) else: if verbose: print('Mock download %s from %s' % (target, key)) else: raise FileNotFoundError('File does not exist on S3')
def move_local_file_into_s3_dir(local_file, s3_path, make_public=True, retry_sleep_time=default_retry_sleep_time, make_protected=False, basename=None, headers={}, unlink_source=True): conn = get_connection() bucket = conn.get_bucket(settings.S3_BUCKET) key_name = s3_path k = Key(bucket, name=key_name) if k.exists(): log.warning("** already exists: %r, deleting: %r" % (key_name, local_file)) return def set_contents(): k.set_contents_from_filename(local_file, headers=headers) run_in_retry_loop(set_contents, retry_sleep_time=retry_sleep_time) if unlink_source: os.unlink(local_file) if make_public: assert not make_protected, 'cannot mix make_public and make_protected' run_in_retry_loop(k.make_public, retry_sleep_time=retry_sleep_time) elif make_protected: mp = lambda: bucket.set_canned_acl('authenticated-read', key_name) run_in_retry_loop(mp, retry_sleep_time=retry_sleep_time) log.info('uploaded %s' % key_name)
def uploadFile( filename, dest=None, bucket=None, overwrite=False ): "Upload a file to a bucket" if not bucket: bucket = 'onos' if not dest: key = basename( filename ) else: key = dest + basename( filename ) #FIXME add the / print '* Uploading', filename, 'to bucket', bucket, 'as', key stdout.flush() start = time() def callback( transmitted, size ): "Progress callback for set_contents_from_filename" elapsed = time() - start percent = 100.0 * transmitted / size kbps = .001 * transmitted / elapsed print ( '\r%d bytes transmitted of %d (%.2f%%),' ' %.2f KB/sec ' % ( transmitted, size, percent, kbps ) ), stdout.flush() conn = S3Connection() bucket = conn.get_bucket( bucket ) k = Key( bucket ) k.key = key if overwrite or not k.exists(): k.set_contents_from_filename( filename, cb=callback, num_cb=100 ) print elapsed = time() - start print "* elapsed time: %.2f seconds" % elapsed else: print 'file', basename( filename ), 'already exists in', bucket.name
def push(article_id, img_name, conn, bucket_name): img_path = article_id + "/" + img_name post_path = "posts/" + img_path if not conn.lookup(bucket_name): # Create the bucket and connect to it if it doesn't exist. bucket = conn.create_bucket(bucket_name, location=boto.s3.connection.Location.DEFAULT) else: # Connect to the bucket. bucket = conn.get_bucket(bucket_name) k = Key(bucket) # Give the key the same name as the image. k.key = img_path local_hash = hash_check(post_path) # If the image path exists, check if the image has been modified. if k.exists(): # Find local md5. local_hash = hash_check(post_path) # Access cloudfront md5. cloudfront_hash = bucket.get_key(img_path).etag[1:-1] if local_hash != cloudfront_hash: print 'Updating ' + img_path + ' in Amazon S3 bucket ' + bucket_name k.set_contents_from_filename(post_path) else: # If the image doesn't exist, add it. print 'Uploading ' + img_path + ' to Amazon S3 bucket ' + bucket_name k.set_contents_from_filename(post_path)
def add_key_user_grant(s3_conn, bucket_name, key_name, permission, canonical_ids): """ Boto wrapper that provides a quick way to add a canonical user grant to a key. :type permission: string :param permission: Name of the bucket where the key resides :type permission: string :param permission: Name of the key to add the permission to :type permission: string :param permission: The permission being granted. Should be one of: (READ, WRITE, READ_ACP, WRITE_ACP, FULL_CONTROL). :type user_id: list of strings :param canonical_ids: A list of strings with canonical user ids associated with the AWS account your are granting the permission to. """ b = get_bucket(s3_conn, bucket_name) if b: try: k = Key(b, key_name) if k.exists(): for c_id in canonical_ids: log.debug("Adding '%s' permission for key '%s' for user '%s'" % ( permission, key_name, c_id)) k.add_user_grant(permission, c_id) return True except S3ResponseError as e: log.error("Could not add permission '%s' for bucket '%s': %s" % ( permission, bucket_name, e)) return False
def index(pin): s3_conn = S3Connection(AWS_KEY, AWS_SECRET) bucket = s3_conn.get_bucket('property-image-cache') s3_key = Key(bucket) s3_key.key = '{0}.jpg'.format(pin) if s3_key.exists(): output = BytesIO() s3_key.get_contents_to_file(output) else: image_viewer = 'http://www.cookcountyassessor.com/PropertyImage.aspx?pin={0}' image_url = image_viewer.format(pin) image = requests.get(image_url) print(image.headers) if 'image/jpeg' in image.headers['Content-Type']: output = BytesIO(image.content) s3_key.set_metadata('Content-Type', 'image/jpg') s3_key.set_contents_from_file(output) s3_key.set_acl('public-read') else: sentry.captureMessage('Could not find image for PIN %s' % pin) abort(404) output.seek(0) response = make_response(output.read()) response.headers['Content-Type'] = 'image/jpg' return response
def uploadFile(filename, dest=None, bucket=None, overwrite=False): "Upload a file to a bucket" if not bucket: bucket = 'onos' if not dest: key = basename(filename) else: key = dest + basename(filename) #FIXME add the / print '* Uploading', filename, 'to bucket', bucket, 'as', key stdout.flush() start = time() def callback(transmitted, size): "Progress callback for set_contents_from_filename" elapsed = time() - start percent = 100.0 * transmitted / size kbps = .001 * transmitted / elapsed print('\r%d bytes transmitted of %d (%.2f%%),' ' %.2f KB/sec ' % (transmitted, size, percent, kbps)), stdout.flush() conn = S3Connection() bucket = conn.get_bucket(bucket) k = Key(bucket) k.key = key if overwrite or not k.exists(): k.set_contents_from_filename(filename, cb=callback, num_cb=100) print elapsed = time() - start print "* elapsed time: %.2f seconds" % elapsed else: print 'file', basename(filename), 'already exists in', bucket.name
def upload_compiled_js_to_s3(local_path, s3_path): with file(local_path, 'rb') as handle: raw_filedata = handle.read() filedata = gzip_string(raw_filedata) headers = { 'Cache-Control': 'max-age=315360000, public', 'Expires': 'Thu, 31 Dec 2037 23:55:55 GMT', 'Content-Encoding': 'gzip', 'Content-Type': 'text/javascript', } conn = S3Connection(*aws) bucket = conn.get_bucket(Config['compress_bucket']) key = Key(bucket) key.key = s3_path try: if key.exists(): print "Skipping", s3_path, " already exists." else: print "Uploading %s (%s kb)" % (s3_path, len(filedata) // 1024) key.set_contents_from_string(filedata, headers=headers) except BotoServerError, bse: print bse
def get_file_from_location(self, path, file, location): """ Perform a file seek, and return the file contents from that position. Uses the KeyFile to perform low-level file operations. :param path: :param file: :param location: :return: """ bconn = self.connect_bucket() if not bconn: raise Exception("No connection to default bucket ({}).".format(self._bucketName)) k = Key(bconn) k.key = self.buildPath(path,file) data = None if k.exists(): kf = KeyFile(k) kf.seek(location) size = k.size data = kf.read(size-location) return data
def fileupload(request): file = request.FILES['files'] key = request.POST.get('key') response = {} filename = FileView.urlify(file.name.split('.')[0]) conn = boto.connect_s3(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY) bucket = conn.get_bucket(settings.AWS_STORAGE_BUCKET_NAME) contentType = file.content_type key_name = key + '/' + file.name k = Key(bucket) k.key = key_name print(key_name) if not k.exists(): key = bucket.new_key(key_name) key.set_contents_from_string(file.read()) key.set_metadata('Content-Type', contentType) key.set_acl('public-read') key.make_public() response['success'] = True; response['msg'] = "Successfully Uploaded"; else: response['success'] = False; response['msg'] = "File name already exists"; return HttpResponse(json.dumps(response), content_type="application/json")
class GetObject(object): def __init__(self, id=None, bucket=None): self.connection = Storage(bucket) self.storage = self.connection.instance self.kobj = Key(self.storage) self.kobj.key = id def get(self, path=DEFAULT_SAVE_PATH): if self.kobj.exists(): self.filepath = os.path.join(path, self.kobj.key) self.kobj.get_contents_to_filename(self.filepath, cb=self.progress) else: raise StoringException('Key <%s> invalid.' % self.kobj.key) def progress(self, part, complete): if part == complete: return True else: return False def delete(self): return self.kobj.delete() @property def path(self): return self.filepath
def upload(request, bucket, key, data): if not key or key.endswith('/'): raise ValueError('key required for upload') perms = request.session['perms'] (can_read, can_write) = has_permission(request.user, perms, bucket, key) if not can_write: return HttpResponseForbidden('%s does not have access to %s/%s' % (request.user.email, bucket, key or '')) b = request.s3.get_bucket(bucket) k = Key(bucket=b, name=key) if k.exists() and 'force' not in request.GET: raise HttpResponseForbidden('write failed: file exists') headers = { "x-amz-acl": "public-read", "Content-Length": len(data), } ct = mimetypes.guess_type(key)[0] if ct is not None: headers["Content-Type"] = ct k.set_contents_from_string(data, headers=headers) k.set_metadata('uploaded-by', request.user.email) if request.is_ajax(): return HttpResponse('{}') else: return redirect('/' + '/'.join(path.split('/')[:-1]))
def upload(self, schedule): from boto.s3.connection import S3Connection from boto.s3.key import Key local_file_path = schedule['args_helpers']['file'] remote_file_path = os.path.join(schedule['aws_s3_storage_key']%schedule['args_helpers'], os.path.basename(local_file_path)) connection = S3Connection(self.access_key, self.secret_key) s3_uri = "%s://%s:%s%s"%(connection.protocol, connection.host, connection.port, connection.get_path()) Logger.log("\n* Uploading %s to aws s3 bucket: '%s', key: '%s', Uri: '%s'."%(local_file_path, schedule['aws_s3_bucket_name'], remote_file_path, s3_uri)) bucket = connection.get_bucket(schedule['aws_s3_bucket_name']) remote_file = Key(bucket) remote_file.key = remote_file_path remote_file.set_contents_from_filename(filename = local_file_path, reduced_redundancy = True) remote_file_exists = remote_file.exists() if remote_file_exists: Logger.log("\nYep, backup uploaded to S3.\n") else: Logger.log("\nOops!, backup not uploaded to S3.\n") connection.close() return remote_file_exists
def if_file_exist(bucketname, filename, aws_access_key="", aws_secret_key=""): bucket = get_bucket(bucketname, aws_access_key, aws_secret_key) k = Key(bucket, filename) if k.exists(): return True else: return False
def _get_file_from_bucket(s3_conn, bucket_name, remote_filename, local_filename): local_filename = os.path.join(LOCAL_PATH, local_filename) try: # log.debug("Establishing handle with bucket '%s'" % bucket_name) b = s3_conn.get_bucket(bucket_name) # log.debug("Establishing handle with file object '%s'" % remote_filename) k = Key(b, remote_filename) log.debug("Attempting to retrieve file '%s' from bucket '%s'" % (remote_filename, bucket_name)) if k.exists(): k.get_contents_to_filename(local_filename) log.info( "Successfully retrieved file '%s' from bucket '%s' to '%s'." % (remote_filename, bucket_name, local_filename)) return True else: log.error("File '%s' in bucket '%s' not found." % (remote_filename, bucket_name)) return False except S3ResponseError, e: log.error("Failed to get file '%s' from bucket '%s': %s" % (remote_filename, bucket_name, e)) return False
def is_file_existing(folder_path, name): import os b, c = get_s3_bucket_and_conn() k = Key(b) k.key = '%s/%s' % (folder_path, os.path.basename(name)) return k.exists()
def put(source_url, bucket, dest_key, mime_type, acl, compress, jsonp, overwrite=False): k = Key(bucket) k.key = dest_key headers = {"Content-Type": mime_type} if k.exists() and not overwrite: logging.info("Skipping %s - already exists") return False try: logging.info("Downloading from %s" % source_url) stream = urllib.urlopen(source_url) contents = stream.read() logging.info("Uploading to %s" % dest_key) string_to_store = "%s(%s);" % (prefix, contents) if jsonp else contents if compress: headers["Content-Encoding"] = "gzip" string_to_store = compress_string(string_to_store) k.set_contents_from_string(string_to_store, headers=headers, cb=s3_progress, num_cb=1000) k.set_acl(acl) except: logging.info("There was an error uploading to %s" % dest_key) logging.info("Finished uploading to %s" % dest_key)
def __init__( self, host, port, access_key, secret_key, bucket, calling_format, secure, root_path="/", ): try: self.__storage_type = "S3boto" root_path = str(root_path) assert len(root_path) > 0, "No root path provided." assert root_path[0] == "/", "Root path should start with /." root_path = str(Path(root_path).resolve()) if root_path[-1] != "/": root_path = root_path + "/" self.__root_path_full = root_path self.__cd_full = root_path self.__cd = "/" self.__host = str(host) self.__port = int(port) self.__access_key = str(access_key) self.__secret_key = str(secret_key) self.__bucket = str(bucket) self.__calling_format = str(calling_format) if type(secure) == bool: self.__secure = secure else: self.__secure = secure == "True" self.__connection = S3Connection( host=self.__host, port=self.__port, aws_access_key_id=self.__access_key, aws_secret_access_key=self.__secret_key, calling_format=self.__calling_format, is_secure=self.__secure) assert self.__connection.lookup(self.__bucket) is not None, \ "The bucket specified doesn't exists!" self.__connection_bucket = self.__connection\ .get_bucket(self.__bucket) if len(self.__root_path_full) > 0: k = Key(self.__connection_bucket) k.key = self.__rm_lead_slash(self.__root_path_full) assert k.exists(), "Root folder not found!" self.__initialized = True logger.debug("Storage DISK initialized.") except Exception as e: self.__initialized = False logger.error("Initialization failed. " + str(e)) raise ValueError("init failed!")
def upload_packetage(self, package_path): package_name = os.path.basename(package_path) package_key = Key(self.bucket, package_name) if package_key.exists(): package_key.delete() else: packege_key.set_contents_from_filename(package_path) return
def get(self, key): """ Retrieve a given object from this bucket. """ key = Key(self._bucket, key) if key.exists(): return AWSBucketObject(self._provider, key) return None
def chunk_exists(self, chunkhash): chunkhash = hexlify(chunkhash) k = Key(self.b) k.key = 'data/' + chunkhash try: return k.exists() except socket.gaierror: raise TryAgain
def execute(self, context, obj): connection = S3Connection() bucket = Bucket(connection=connection, name=context['bucket']) key = Key(bucket=bucket, name=context['name']) if key.exists(): return 'done' else: return 'missing'
def _push_to_os(self, rel_path, source_file=None, from_string=None): """ Push the file pointed to by ``rel_path`` to the object store naming the key ``rel_path``. If ``source_file`` is provided, push that file instead while still using ``rel_path`` as the key name. If ``from_string`` is provided, set contents of the file to the value of the string. """ try: source_file = source_file if source_file else self._get_cache_path( rel_path) if os.path.exists(source_file): key = Key(self.bucket, rel_path) if os.path.getsize(source_file) == 0 and key.exists(): log.debug( "Wanted to push file '%s' to S3 key '%s' but its size is 0; skipping." % (source_file, rel_path)) return True if from_string: key.set_contents_from_string( from_string, reduced_redundancy=self.use_rr) log.debug("Pushed data from string '%s' to key '%s'" % (from_string, rel_path)) else: start_time = datetime.now() log.debug( "Pushing cache file '%s' of size %s bytes to key '%s'" % (source_file, os.path.getsize(source_file), rel_path)) mb_size = os.path.getsize(source_file) / 1e6 if mb_size < 10 or type(self) == SwiftObjectStore: self.transfer_progress = 0 # Reset transfer progress counter key.set_contents_from_filename( source_file, reduced_redundancy=self.use_rr, cb=self._transfer_cb, num_cb=10) else: multipart_upload(self.bucket, key.name, source_file, mb_size, self.access_key, self.secret_key, use_rr=self.use_rr) end_time = datetime.now() log.debug( "Pushed cache file '%s' to key '%s' (%s bytes transfered in %s sec)" % (source_file, rel_path, os.path.getsize(source_file), end_time - start_time)) return True else: log.error( "Tried updating key '%s' from source file '%s', but source file does not exist." % (rel_path, source_file)) except S3ResponseError, ex: log.error("Trouble pushing S3 key '%s' from file '%s': %s" % (rel_path, source_file, ex))
def test_get_all_keys_tree(self): """ test storing and retrieving a directory tree """ # 2011-12-04 -- s3 clips leading slash key_names = [ "aaa/b/cccc/1", "aaa/b/ccccccccc/1", "aaa/b/ccccccccc/2", "aaa/b/ccccccccc/3", "aaa/b/dddd/1", "aaa/b/dddd/2", "aaa/e/ccccccccc/1", "fff/e/ccccccccc/1", ] # create the bucket bucket = self._s3_connection.create_unique_bucket() self.assertTrue(bucket is not None) for key in bucket.list(): key.delete() # create some keys keys = list() for key_name in key_names: key = Key(bucket) # set the name key.name = key_name # upload some data test_string = os.urandom(1024) key.set_contents_from_string(test_string) self.assertTrue(key.exists()) keys.append(key) result_set = BucketListResultSet(bucket, prefix="aaa") self.assertEqual(len(list(result_set)), 7) result_set = BucketListResultSet(bucket, prefix="aaa/b") self.assertEqual(len(list(result_set)), 6) result_set = BucketListResultSet(bucket, prefix="aaa/b/ccccccccc/") self.assertEqual(len(list(result_set)), 3) result_set = BucketListResultSet(bucket, prefix="aaa/b/dddd") self.assertEqual(len(list(result_set)), 2) result_set = BucketListResultSet(bucket, prefix="aaa/e") self.assertEqual(len(list(result_set)), 1) # delete the keys for key in bucket.list(): key.delete() # delete the bucket self._s3_connection.delete_bucket(bucket.name)
def get(self, key_name): k = Key(self.bucket, key_name) if not k.exists(): return None content = k.get_contents_as_string() if self.decompress: return self.decompress(content) return content
def get(self, filepath): k = Key(self.bucket) k.key = self.make_fp(filepath) if k.exists(): s = k.get_contents_as_string() print s return json.loads(un_gz(s)) else: return None
def exists(self, _path): k = Key(self._bucket_api) k.key = _path.key try: return k.exists() except: logger.exception("head s3 file failed") return False
def exists(self, path): try: assert self.__initialized, "Storage not initialized." path = str(path) path_full = self.__path_expand(path, bool_file=True) path_full = self.__rm_lead_slash(path_full) k = Key(self.__connection_bucket) k.key = path_full if k.exists(): output = True else: k.key = path_full + "/" output = k.exists() logger.debug("exists " + str(path) + ": " + str(output)) return output except Exception as e: logger.error("Failed to check the existence. " + str(e)) raise ValueError("exists failed!")
def getFile(self,path,file,localfile): if self._conn == None: raise Exception("Must connect first.") k = Key(self._bconn) k.name = self.buildPath(path,file) if k.exists(): print ("Key name:%s" % k.name) k.get_contents_to_file(localfile)
def s3_exists(self, key_name): key = None try: logging.debug("Checking if key exists s3://%s%s" % (self.bucket_name, key_name)) key = Key(bucket=self.bucket, name=key_name) return key.exists() finally: if key: key.close()
def remove_folder(bucket, folder): key = Key(bucket) key.key = folder if key.exists(): if DRY_RUN: LOG.info('DRY_RUN: Removing {0}'.format(folder)) else: key.delete() else: LOG.warning('The key {0} does not exist'.format(folder))
def check_timestamp(): require('settings', provided_by=[production, staging]) bucket = utils.get_bucket(app_config.S3_BUCKET) k = Key(bucket) k.key = '%s/live-data/timestamp.json' % app_config.PROJECT_SLUG if k.exists(): return True else: return False