def put_bucket(self, bucket, data, header, accesskey, error_on_preexisting):
""" Store a bucket """
if self.use_fs_directly:
return fshandler.FilesystemHandler.put_bucket(self, bucket, data, header)
# write /tmp/tmp/uploads/s3-UUID-bucket/bucket_meta.xml
# write /tmp/tmp/uploads/s3-UUID-bucket/bucket_files.xml
# run contrib-submit with delete after submitted
if not self.is_safe_identifier(bucket):
raise S3_Error("InvalidBucketName", "Bucket names should be valid archive identifiers; try someting matching"
+ ' this regular expression: ^[a-zA-Z0-9][a-zA-Z0-9_.-]{4,100}$' )
db = s3db.s3db()
username = db.get_username(accesskey)
if username == None :
# db broken? by the time we get here
# access key was already checked by s3server
raise S3_Error
meta_xml_etree = self.meta_xml_from_header(header, bucket, username)
if not self.username_can_add_to_collections(username, meta_xml_etree):
raise S3_Error("AccessDenied", "You lack sufficient privilages to write to those collections")
meta_xml = xml.etree.ElementTree.tostring(meta_xml_etree, "UTF-8")
bucket_exists = True
try:
self.get_bucket_host(bucket)
except S3_Error, e:
if e.code == "NoSuchBucket":
bucket_exists = False
def assert_can_write_to_bucket(self, accesskey, bucket):
""" returns true, or raises an access exception if not allowed """
db = s3db.s3db()
# shortcut if they made the bucket
if ((accesskey != None) and (accesskey == db.get_bucket_owner_accesskey(bucket))):
return True
username = db.get_username(accesskey)
if username == None:
# db broken? by the time we get here
# access key was already checked by s3server
raise S3_Error
pipe = subprocess.Popen([
'php', '-r',
'include "/petabox/setup.inc"; print Auth::usernameCanWriteToItem($argv[1], $argv[2]);',
'--',
username,
bucket,
], stdout=subprocess.PIPE)
output = pipe.communicate()[0]
if (pipe.wait() != 0):
raise S3_Error("InternalError", "Auth::usernameCanWriteToItem: " + output)
if output == '1' or output == '2':
return True
raise S3_Error("AccessDenied", "You lack sufficient privilages to write to this item.")
def assert_not_overloaded(self, accesskey):
""" make sure we are under the outstanding task limit
limiting is as follows:
if total put count exceeds hard_limit we return overloaded
if total put count exceeds soft_limit than we do per user limiting
if put_count[accesskey] exceeds user_limit then we return overloaded
"""
hard_limit = 1299
soft_limit = 999
user_limit = 29
db = s3db.s3db()
(total) = db.get_running_put_counts()
if total > hard_limit:
time.sleep(2)
raise S3_Error("SlowDown")
return False
if total > soft_limit:
username = db.get_username(accesskey)
(put_count) = db.get_running_put_count_for_user(username)
if (put_count > user_limit):
time.sleep(2)
raise S3_Error("SlowDown")
return False
return True
def get_owner_accesskey(self, bucket, key):
""" ownership from item's meta.xml + db """
path = self.bucket_to_path(bucket)
# bucket must be a safe name, with no slashes
assert(bucket.find('/') == -1)
meta = xml.etree.ElementTree.parse(os.path.join(path, bucket + '_meta.xml'))
username = meta.find('uploader').text
db = s3db.s3db()
accesskey = db.get_accesskey(username)
return accesskey
def __init__(self, wsgi_env, wsgi_start_response):
self.env = wsgi_env
self.start_response = wsgi_start_response
self.db = s3db.s3db()
init_log = []
init_log.append( "SCRIPT_NAME: [%s] PATH_INFO: [%s] QUERY_STRING: [%s]" % (self.env['SCRIPT_NAME'], self.env['PATH_INFO'], self.env['QUERY_STRING']))
if self.env.has_key('CONTENT_TYPE'):
init_log.append( "CONTENT_TYPE: [%s]" % (self.env['CONTENT_TYPE']))
self.path = ''
self.path = self.env['REQUEST_URI']
self.command = self.env['REQUEST_METHOD']
self.data = []
self.response_headers = []
self.response_headers_dict = wsgiref.headers.Headers(self.response_headers)
self.response = '500 Internal error'
self.headers = {}
init_log.append("\n-- headers:\n")
for key in sorted(self.env.keys()):
if not key.startswith('HTTP_'):
continue
value = self.env[key]
newkey = key[5:]
newkey = string.replace(newkey, '_', '-')
newkey = string.lower(newkey)
if newkey == 'cookie':
value = 'REDACTED_BY_IA_S3'
self.headers[newkey] = value
# filter log
if newkey == 'authorization':
(access, sep, secret) = value.partition(':')
init_log.append("%s:[%s] " % (newkey, access + ':REDACTED_BY_IA_S3'))
if newkey != 'authorization':
init_log.append("%s:[%s] " % (newkey, value))
init_log.append("-- end headers\n")
if self.env.has_key('CONTENT_LENGTH'):
self.headers['content-length'] = self.env['CONTENT_LENGTH']
if self.env.has_key('CONTENT_TYPE'):
self.headers['content-type'] = self.env['CONTENT_TYPE']
(accesskey, signature) = S3RequestHandler.get_accesskey_signature(self)
self._log = s3log.S3Log(self.env['wsgi.errors'], "s3accesskey-"+accesskey).log
self.IFACE_CLASS = itemhandler.ItemHandler("/", self._log)
for l in init_log:
self._log(l)
def delkey(self, bucket, key, accesskey, cascade, keep_old_version):
"""delete a key from the s3 storage"""
if self.use_fs_directly:
return fshandler.FilesystemHandler.delkey(self, bucket, key)
self.assert_bucket_exists(bucket)
db = s3db.s3db()
username = db.get_username(accesskey)
if username == None :
# db broken? by the time we get here
# access key was already checked by s3server
raise S3_Error
for extension in ('', '_meta.txt'):
job = [
self.ias3_path + '/' + "./deleteFromItem.py",
"-s", username,
"--priority",
"1",
"-i", bucket,
"--comment", "s3-delete",
"--file-to-delete",
self.id_to_filename(key) + extension
]
if cascade:
job.append("--cascade")
if keep_old_version:
job.append("--s3-keep-old-version")
pipe = subprocess.Popen(job, stdout=subprocess.PIPE)
output = pipe.communicate()[0]
assert (pipe.wait() == 0)
self._log('put: called deleteFromItem.py: %s' % output)
def get_bucket_list(self, accesskey):
"""return a list of buckets at the endpoint"""
db = s3db.s3db()
return db.get_buckets(accesskey)
def put_key(self, bucket, key, filehandle, header, accesskey, queue_derive, keep_old_version):
""" Store a key on disk """
if self.use_fs_directly:
return fshandler.FilesystemHandler.put_key(self, bucket, key, filehandle, header, accesskey)
self.assert_not_overloaded(accesskey)
db = s3db.s3db()
username = db.get_username(accesskey)
if username == None :
# db broken? by the time we get here
# access key was already checked by s3server
raise S3_Error
#steps:
# mkdir /tmp/tmp/uploads/s3-UUID-bucket/
# write /tmp/tmp/uploads/s3-UUID-bucket/key
# write /tmp/tmp/uploads/s3-UUID-bucket/key_meta.txt
# run contrib-submit with delete after submitted
try:
holdingdir = "/tmp/tmp/uploads/s3-" + str(uuid.uuid4()) + "-" + self.id_to_filename(bucket)
key_file = holdingdir + "/" + self.id_to_filename(key)
head_file = key_file + "_meta.txt"
if not os.path.isdir(holdingdir):
os.makedirs(holdingdir)
k = open(key_file, "w")
buffer = ''
chunk = 32768
copied = 0
size = string.atoi(header['content-length'])
md5_hash = hashlib.md5()
while (copied < size and buffer != None):
toread = min(chunk, (size - copied))
buffer = filehandle.read(toread)
md5_hash.update(buffer)
k.write(buffer)
copied = copied + toread
k.close()
h = open(head_file, "w")
header['ETag'] = '"' + md5_hash.hexdigest() + '"'
h.write(self.head_data(header))
h.close()
job = [
self.ias3_path + '/' + "./uploadItem.py",
"--deleteAfter",
"--priority",
"-6",
"-s", username,
"--update",
"-d", holdingdir,
"-i", bucket,
"--comment",
"s3-put",
]
if queue_derive:
job.append("--derive")
if keep_old_version:
job.append("--s3-keep-old-version")
pipe = subprocess.Popen(job, stdout=subprocess.PIPE)
output = pipe.communicate()[0]
if pipe.wait() != 0:
shutil.rmtree(holdingdir, True)
raise S3_Error("InternalError", "uploadItem.py: " + output)
self._log('put_key: Created %s' % bucket)
self._log('put_key: called upload_item: %s' % output)
except OSError:
self._log('put_key: Could not create %s' % bucket)
raise S3_Error
return md5_hash.hexdigest()
