コード例 #1
0
ファイル: itemhandler.py プロジェクト: internetarchive/ias3
    def put_bucket(self, bucket, data, header, accesskey, error_on_preexisting):
        """ Store a bucket """
        if self.use_fs_directly:
            return fshandler.FilesystemHandler.put_bucket(self, bucket, data, header)

        # write /tmp/tmp/uploads/s3-UUID-bucket/bucket_meta.xml
        # write /tmp/tmp/uploads/s3-UUID-bucket/bucket_files.xml
        # run contrib-submit with delete after submitted

        if not self.is_safe_identifier(bucket):
            raise S3_Error("InvalidBucketName", "Bucket names should be valid archive identifiers; try someting matching"
                + ' this regular expression: ^[a-zA-Z0-9][a-zA-Z0-9_.-]{4,100}$' )

        db = s3db.s3db()
        username = db.get_username(accesskey)
        if username == None :
            # db broken? by the time we get here
            # access key was already checked by s3server
            raise S3_Error

        meta_xml_etree = self.meta_xml_from_header(header, bucket, username)
        
        if not self.username_can_add_to_collections(username, meta_xml_etree):
            raise S3_Error("AccessDenied", "You lack sufficient privilages to write to those collections")

        meta_xml = xml.etree.ElementTree.tostring(meta_xml_etree, "UTF-8")

        bucket_exists = True
        try:
            self.get_bucket_host(bucket) 
        except S3_Error, e:
            if e.code == "NoSuchBucket":
                bucket_exists = False
コード例 #2
0
ファイル: itemhandler.py プロジェクト: internetarchive/ias3
    def assert_can_write_to_bucket(self, accesskey, bucket):
        """ returns true, or raises an access exception if not allowed """
        db = s3db.s3db()

        # shortcut if they made the bucket
        if ((accesskey != None) and (accesskey == db.get_bucket_owner_accesskey(bucket))):
            return True

        username = db.get_username(accesskey)
        if username == None:
            # db broken? by the time we get here
            # access key was already checked by s3server
            raise S3_Error

        pipe = subprocess.Popen([
            'php', '-r', 
            'include "/petabox/setup.inc";  print Auth::usernameCanWriteToItem($argv[1], $argv[2]);',
            '--', 
            username,
            bucket,
            ], stdout=subprocess.PIPE)
        output = pipe.communicate()[0]
        if (pipe.wait() != 0):
            raise S3_Error("InternalError", "Auth::usernameCanWriteToItem: " + output)
        if output == '1' or output == '2':
            return True
        raise S3_Error("AccessDenied", "You lack sufficient privilages to write to this item.")
コード例 #3
0
ファイル: itemhandler.py プロジェクト: internetarchive/ias3
    def assert_not_overloaded(self, accesskey):
        """ make sure we are under the outstanding task limit 
            limiting is as follows:
            if total put count exceeds hard_limit we return overloaded
            if total put count exceeds soft_limit than we do per user limiting
            if put_count[accesskey] exceeds user_limit then we return overloaded
            """
        hard_limit = 1299
        soft_limit = 999
        user_limit = 29

        db = s3db.s3db()
        (total) = db.get_running_put_counts() 

        if total > hard_limit:
            time.sleep(2)
            raise S3_Error("SlowDown")
            return False

        if total > soft_limit:
            username = db.get_username(accesskey)
            (put_count) = db.get_running_put_count_for_user(username) 

            if (put_count > user_limit):
                time.sleep(2)
                raise S3_Error("SlowDown")
                return False

        return True
コード例 #4
0
ファイル: itemhandler.py プロジェクト: internetarchive/ias3
 def get_owner_accesskey(self, bucket, key):
     """ ownership from item's meta.xml + db """
     path = self.bucket_to_path(bucket)
     # bucket must be a safe name, with no slashes
     assert(bucket.find('/') == -1)
     meta = xml.etree.ElementTree.parse(os.path.join(path, bucket + '_meta.xml'))
     username = meta.find('uploader').text
     db = s3db.s3db()
     accesskey = db.get_accesskey(username)
     return accesskey
コード例 #5
0
ファイル: s3server.py プロジェクト: internetarchive/ias3
    def __init__(self, wsgi_env, wsgi_start_response):
        self.env = wsgi_env
        self.start_response = wsgi_start_response
        self.db = s3db.s3db()

        init_log = []
        init_log.append( "SCRIPT_NAME: [%s] PATH_INFO: [%s] QUERY_STRING: [%s]" % (self.env['SCRIPT_NAME'], self.env['PATH_INFO'], self.env['QUERY_STRING']))
        if self.env.has_key('CONTENT_TYPE'):
            init_log.append( "CONTENT_TYPE: [%s]" % (self.env['CONTENT_TYPE']))
        self.path = ''
        self.path = self.env['REQUEST_URI']
       
        self.command = self.env['REQUEST_METHOD']

        self.data = []
        self.response_headers = []
        self.response_headers_dict = wsgiref.headers.Headers(self.response_headers)
        self.response = '500 Internal error'
        
        self.headers = {}
        init_log.append("\n-- headers:\n")
        for key in sorted(self.env.keys()):
            if not key.startswith('HTTP_'):
                continue
            value = self.env[key]
            newkey = key[5:]
            newkey = string.replace(newkey, '_', '-')
            newkey = string.lower(newkey)
            if newkey == 'cookie':
                value = 'REDACTED_BY_IA_S3'
            self.headers[newkey] = value

            # filter log
            if newkey == 'authorization':
                (access, sep, secret) = value.partition(':')
                init_log.append("%s:[%s] " % (newkey, access + ':REDACTED_BY_IA_S3'))
            if newkey != 'authorization':
                init_log.append("%s:[%s] " % (newkey, value))

        init_log.append("-- end headers\n")

        if self.env.has_key('CONTENT_LENGTH'):
            self.headers['content-length'] = self.env['CONTENT_LENGTH']
        if self.env.has_key('CONTENT_TYPE'):
            self.headers['content-type'] = self.env['CONTENT_TYPE']

        (accesskey, signature) = S3RequestHandler.get_accesskey_signature(self)
        self._log = s3log.S3Log(self.env['wsgi.errors'], "s3accesskey-"+accesskey).log
        self.IFACE_CLASS = itemhandler.ItemHandler("/", self._log)
        for l in init_log:
            self._log(l)
コード例 #6
0
ファイル: itemhandler.py プロジェクト: internetarchive/ias3
    def delkey(self, bucket, key, accesskey, cascade, keep_old_version):
        """delete a key from the s3 storage"""

        if self.use_fs_directly:
            return fshandler.FilesystemHandler.delkey(self, bucket, key)

        self.assert_bucket_exists(bucket)

        db = s3db.s3db()
        username = db.get_username(accesskey)
        if username == None :
            # db broken? by the time we get here
            # access key was already checked by s3server
            raise S3_Error

        for extension in ('', '_meta.txt'):
            job = [
                self.ias3_path + '/' + "./deleteFromItem.py", 
                "-s", username,
                "--priority",
                "1",
                "-i", bucket, 
                "--comment", "s3-delete",
                "--file-to-delete", 
                self.id_to_filename(key) + extension
                ]

            if cascade:
                job.append("--cascade")
            if keep_old_version:
                job.append("--s3-keep-old-version")

            pipe = subprocess.Popen(job, stdout=subprocess.PIPE)
            output = pipe.communicate()[0]
            assert (pipe.wait() == 0)
            self._log('put: called deleteFromItem.py: %s' % output)
コード例 #7
0
ファイル: itemhandler.py プロジェクト: internetarchive/ias3
 def get_bucket_list(self, accesskey):
     """return a list of buckets at the endpoint"""
     db = s3db.s3db()
     return db.get_buckets(accesskey)
コード例 #8
0
ファイル: itemhandler.py プロジェクト: internetarchive/ias3
    def put_key(self, bucket, key, filehandle, header, accesskey, queue_derive, keep_old_version):
        """ Store a key on disk """
        if self.use_fs_directly:
            return fshandler.FilesystemHandler.put_key(self, bucket, key, filehandle, header, accesskey)

        self.assert_not_overloaded(accesskey)

        db = s3db.s3db()
        username = db.get_username(accesskey)
        if username == None :
            # db broken? by the time we get here
            # access key was already checked by s3server
            raise S3_Error

        #steps:
        # mkdir /tmp/tmp/uploads/s3-UUID-bucket/
        # write /tmp/tmp/uploads/s3-UUID-bucket/key
        # write /tmp/tmp/uploads/s3-UUID-bucket/key_meta.txt
        # run contrib-submit with delete after submitted

        try:
            holdingdir = "/tmp/tmp/uploads/s3-" + str(uuid.uuid4()) + "-" + self.id_to_filename(bucket)
            key_file = holdingdir + "/" + self.id_to_filename(key)
            head_file = key_file + "_meta.txt"
            if not os.path.isdir(holdingdir):
                os.makedirs(holdingdir)
            k = open(key_file, "w")
            buffer = ''
            chunk = 32768
            copied = 0
            size = string.atoi(header['content-length'])
            md5_hash = hashlib.md5()
            while (copied < size and buffer != None):
                toread = min(chunk, (size - copied))
                buffer = filehandle.read(toread)
                md5_hash.update(buffer)
                k.write(buffer)
                copied = copied + toread
            k.close()
            h = open(head_file, "w")
            header['ETag'] = '"' + md5_hash.hexdigest() + '"'
            h.write(self.head_data(header))
            h.close()
            job = [
                self.ias3_path + '/' + "./uploadItem.py", 
                "--deleteAfter", 
                "--priority",
                "-6",
                "-s", username, 
                "--update", 
                "-d", holdingdir, 
                "-i", bucket, 
                "--comment", 
                "s3-put", 
            ]
            if queue_derive:
                job.append("--derive")
            if keep_old_version:
                job.append("--s3-keep-old-version")
            pipe = subprocess.Popen(job, stdout=subprocess.PIPE)
            output = pipe.communicate()[0]
            if pipe.wait() != 0:
                shutil.rmtree(holdingdir, True)
                raise S3_Error("InternalError", "uploadItem.py: " + output)
 
            self._log('put_key: Created %s' % bucket)
            self._log('put_key: called upload_item: %s' % output)
        except OSError:
            self._log('put_key: Could not create %s' % bucket)
            raise S3_Error
        return md5_hash.hexdigest()