def put_bucket(self, bucket, data, header, accesskey, error_on_preexisting): """ Store a bucket """ if self.use_fs_directly: return fshandler.FilesystemHandler.put_bucket(self, bucket, data, header) # write /tmp/tmp/uploads/s3-UUID-bucket/bucket_meta.xml # write /tmp/tmp/uploads/s3-UUID-bucket/bucket_files.xml # run contrib-submit with delete after submitted if not self.is_safe_identifier(bucket): raise S3_Error("InvalidBucketName", "Bucket names should be valid archive identifiers; try someting matching" + ' this regular expression: ^[a-zA-Z0-9][a-zA-Z0-9_.-]{4,100}$' ) db = s3db.s3db() username = db.get_username(accesskey) if username == None : # db broken? by the time we get here # access key was already checked by s3server raise S3_Error meta_xml_etree = self.meta_xml_from_header(header, bucket, username) if not self.username_can_add_to_collections(username, meta_xml_etree): raise S3_Error("AccessDenied", "You lack sufficient privilages to write to those collections") meta_xml = xml.etree.ElementTree.tostring(meta_xml_etree, "UTF-8") bucket_exists = True try: self.get_bucket_host(bucket) except S3_Error, e: if e.code == "NoSuchBucket": bucket_exists = False
def assert_can_write_to_bucket(self, accesskey, bucket): """ returns true, or raises an access exception if not allowed """ db = s3db.s3db() # shortcut if they made the bucket if ((accesskey != None) and (accesskey == db.get_bucket_owner_accesskey(bucket))): return True username = db.get_username(accesskey) if username == None: # db broken? by the time we get here # access key was already checked by s3server raise S3_Error pipe = subprocess.Popen([ 'php', '-r', 'include "/petabox/setup.inc"; print Auth::usernameCanWriteToItem($argv[1], $argv[2]);', '--', username, bucket, ], stdout=subprocess.PIPE) output = pipe.communicate()[0] if (pipe.wait() != 0): raise S3_Error("InternalError", "Auth::usernameCanWriteToItem: " + output) if output == '1' or output == '2': return True raise S3_Error("AccessDenied", "You lack sufficient privilages to write to this item.")
def assert_not_overloaded(self, accesskey): """ make sure we are under the outstanding task limit limiting is as follows: if total put count exceeds hard_limit we return overloaded if total put count exceeds soft_limit than we do per user limiting if put_count[accesskey] exceeds user_limit then we return overloaded """ hard_limit = 1299 soft_limit = 999 user_limit = 29 db = s3db.s3db() (total) = db.get_running_put_counts() if total > hard_limit: time.sleep(2) raise S3_Error("SlowDown") return False if total > soft_limit: username = db.get_username(accesskey) (put_count) = db.get_running_put_count_for_user(username) if (put_count > user_limit): time.sleep(2) raise S3_Error("SlowDown") return False return True
def get_owner_accesskey(self, bucket, key): """ ownership from item's meta.xml + db """ path = self.bucket_to_path(bucket) # bucket must be a safe name, with no slashes assert(bucket.find('/') == -1) meta = xml.etree.ElementTree.parse(os.path.join(path, bucket + '_meta.xml')) username = meta.find('uploader').text db = s3db.s3db() accesskey = db.get_accesskey(username) return accesskey
def __init__(self, wsgi_env, wsgi_start_response): self.env = wsgi_env self.start_response = wsgi_start_response self.db = s3db.s3db() init_log = [] init_log.append( "SCRIPT_NAME: [%s] PATH_INFO: [%s] QUERY_STRING: [%s]" % (self.env['SCRIPT_NAME'], self.env['PATH_INFO'], self.env['QUERY_STRING'])) if self.env.has_key('CONTENT_TYPE'): init_log.append( "CONTENT_TYPE: [%s]" % (self.env['CONTENT_TYPE'])) self.path = '' self.path = self.env['REQUEST_URI'] self.command = self.env['REQUEST_METHOD'] self.data = [] self.response_headers = [] self.response_headers_dict = wsgiref.headers.Headers(self.response_headers) self.response = '500 Internal error' self.headers = {} init_log.append("\n-- headers:\n") for key in sorted(self.env.keys()): if not key.startswith('HTTP_'): continue value = self.env[key] newkey = key[5:] newkey = string.replace(newkey, '_', '-') newkey = string.lower(newkey) if newkey == 'cookie': value = 'REDACTED_BY_IA_S3' self.headers[newkey] = value # filter log if newkey == 'authorization': (access, sep, secret) = value.partition(':') init_log.append("%s:[%s] " % (newkey, access + ':REDACTED_BY_IA_S3')) if newkey != 'authorization': init_log.append("%s:[%s] " % (newkey, value)) init_log.append("-- end headers\n") if self.env.has_key('CONTENT_LENGTH'): self.headers['content-length'] = self.env['CONTENT_LENGTH'] if self.env.has_key('CONTENT_TYPE'): self.headers['content-type'] = self.env['CONTENT_TYPE'] (accesskey, signature) = S3RequestHandler.get_accesskey_signature(self) self._log = s3log.S3Log(self.env['wsgi.errors'], "s3accesskey-"+accesskey).log self.IFACE_CLASS = itemhandler.ItemHandler("/", self._log) for l in init_log: self._log(l)
def delkey(self, bucket, key, accesskey, cascade, keep_old_version): """delete a key from the s3 storage""" if self.use_fs_directly: return fshandler.FilesystemHandler.delkey(self, bucket, key) self.assert_bucket_exists(bucket) db = s3db.s3db() username = db.get_username(accesskey) if username == None : # db broken? by the time we get here # access key was already checked by s3server raise S3_Error for extension in ('', '_meta.txt'): job = [ self.ias3_path + '/' + "./deleteFromItem.py", "-s", username, "--priority", "1", "-i", bucket, "--comment", "s3-delete", "--file-to-delete", self.id_to_filename(key) + extension ] if cascade: job.append("--cascade") if keep_old_version: job.append("--s3-keep-old-version") pipe = subprocess.Popen(job, stdout=subprocess.PIPE) output = pipe.communicate()[0] assert (pipe.wait() == 0) self._log('put: called deleteFromItem.py: %s' % output)
def get_bucket_list(self, accesskey): """return a list of buckets at the endpoint""" db = s3db.s3db() return db.get_buckets(accesskey)
def put_key(self, bucket, key, filehandle, header, accesskey, queue_derive, keep_old_version): """ Store a key on disk """ if self.use_fs_directly: return fshandler.FilesystemHandler.put_key(self, bucket, key, filehandle, header, accesskey) self.assert_not_overloaded(accesskey) db = s3db.s3db() username = db.get_username(accesskey) if username == None : # db broken? by the time we get here # access key was already checked by s3server raise S3_Error #steps: # mkdir /tmp/tmp/uploads/s3-UUID-bucket/ # write /tmp/tmp/uploads/s3-UUID-bucket/key # write /tmp/tmp/uploads/s3-UUID-bucket/key_meta.txt # run contrib-submit with delete after submitted try: holdingdir = "/tmp/tmp/uploads/s3-" + str(uuid.uuid4()) + "-" + self.id_to_filename(bucket) key_file = holdingdir + "/" + self.id_to_filename(key) head_file = key_file + "_meta.txt" if not os.path.isdir(holdingdir): os.makedirs(holdingdir) k = open(key_file, "w") buffer = '' chunk = 32768 copied = 0 size = string.atoi(header['content-length']) md5_hash = hashlib.md5() while (copied < size and buffer != None): toread = min(chunk, (size - copied)) buffer = filehandle.read(toread) md5_hash.update(buffer) k.write(buffer) copied = copied + toread k.close() h = open(head_file, "w") header['ETag'] = '"' + md5_hash.hexdigest() + '"' h.write(self.head_data(header)) h.close() job = [ self.ias3_path + '/' + "./uploadItem.py", "--deleteAfter", "--priority", "-6", "-s", username, "--update", "-d", holdingdir, "-i", bucket, "--comment", "s3-put", ] if queue_derive: job.append("--derive") if keep_old_version: job.append("--s3-keep-old-version") pipe = subprocess.Popen(job, stdout=subprocess.PIPE) output = pipe.communicate()[0] if pipe.wait() != 0: shutil.rmtree(holdingdir, True) raise S3_Error("InternalError", "uploadItem.py: " + output) self._log('put_key: Created %s' % bucket) self._log('put_key: called upload_item: %s' % output) except OSError: self._log('put_key: Could not create %s' % bucket) raise S3_Error return md5_hash.hexdigest()