Ejemplo n.º 1
0
    def test_enhanced_ssl_connection(self):
        fingerprint = '92:81:FE:85:F7:0C:26:60:EC:D6:B3:BF:93:CF:F9:71:CC:07:7D:0A'

        https_pool = HTTPSConnectionPool(self.host,
                                         self.port,
                                         cert_reqs='CERT_REQUIRED',
                                         ca_certs=DEFAULT_CA,
                                         assert_fingerprint=fingerprint)
        self.addCleanup(https_pool.close)

        https_pool.urlopen('GET', '/')
Ejemplo n.º 2
0
 def request():
     try:
         pool = HTTPSConnectionPool(self.host, self.port, assert_fingerprint=fingerprint)
         response = pool.urlopen("GET", "/", preload_content=False, timeout=Timeout(connect=1, read=0.001))
         response.read()
     finally:
         pool.close()
Ejemplo n.º 3
0
 def request():
     try:
         pool = HTTPSConnectionPool(self.host, self.port,
                                    assert_fingerprint=fingerprint)
         response = pool.urlopen('GET', '/', preload_content=False,
                                 timeout=Timeout(connect=1, read=0.001))
         response.read()
     finally:
         pool.close()
Ejemplo n.º 4
0
    def test_ssl_read_timeout(self):
        timed_out = Event()

        def socket_handler(listener):
            sock = listener.accept()[0]
            ssl_sock = ssl.wrap_socket(sock,
                                       server_side=True,
                                       keyfile=DEFAULT_CERTS['keyfile'],
                                       certfile=DEFAULT_CERTS['certfile'],
                                       ca_certs=DEFAULT_CA)

            buf = b''
            while not buf.endswith(b'\r\n\r\n'):
                buf += ssl_sock.recv(65536)

            # Send incomplete message (note Content-Length)
            ssl_sock.send(('HTTP/1.1 200 OK\r\n'
                           'Content-Type: text/plain\r\n'
                           'Content-Length: 10\r\n'
                           '\r\n'
                           'Hi-').encode('utf-8'))
            timed_out.wait()

            sock.close()
            ssl_sock.close()

        self._start_server(socket_handler)
        pool = HTTPSConnectionPool(self.host, self.port)
        self.addCleanup(pool.close)

        response = pool.urlopen('GET',
                                '/',
                                retries=0,
                                preload_content=False,
                                timeout=Timeout(connect=1, read=0.001))
        try:
            self.assertRaises(ReadTimeoutError, response.read)
        finally:
            timed_out.set()
Ejemplo n.º 5
0
    def test_ssl_read_timeout(self):
        timed_out = Event()

        def socket_handler(listener):
            sock = listener.accept()[0]
            ssl_sock = ssl.wrap_socket(sock,
                                       server_side=True,
                                       keyfile=DEFAULT_CERTS['keyfile'],
                                       certfile=DEFAULT_CERTS['certfile'],
                                       ca_certs=DEFAULT_CA)

            buf = b''
            while not buf.endswith(b'\r\n\r\n'):
                buf += ssl_sock.recv(65536)

            # Send incomplete message (note Content-Length)
            ssl_sock.send((
                'HTTP/1.1 200 OK\r\n'
                'Content-Type: text/plain\r\n'
                'Content-Length: 10\r\n'
                '\r\n'
                'Hi-').encode('utf-8'))
            timed_out.wait()

            sock.close()
            ssl_sock.close()

        self._start_server(socket_handler)
        pool = HTTPSConnectionPool(self.host, self.port)
        self.addCleanup(pool.close)

        response = pool.urlopen('GET', '/', retries=0, preload_content=False,
                                timeout=Timeout(connect=1, read=0.001))
        try:
            self.assertRaises(ReadTimeoutError, response.read)
        finally:
            timed_out.set()
Ejemplo n.º 6
0
    def test_ssl_read_timeout(self):
        timed_out = Event()

        def socket_handler(listener):
            sock = listener.accept()[0]
            ssl_sock = ssl.wrap_socket(
                sock,
                server_side=True,
                keyfile=DEFAULT_CERTS["keyfile"],
                certfile=DEFAULT_CERTS["certfile"],
                ca_certs=DEFAULT_CA,
            )

            buf = b""
            while not buf.endswith(b"\r\n\r\n"):
                buf += ssl_sock.recv(65536)

            # Send incomplete message (note Content-Length)
            ssl_sock.send(
                ("HTTP/1.1 200 OK\r\n" "Content-Type: text/plain\r\n" "Content-Length: 10\r\n" "\r\n" "Hi-").encode(
                    "utf-8"
                )
            )
            timed_out.wait()

            sock.close()
            ssl_sock.close()

        self._start_server(socket_handler)
        pool = HTTPSConnectionPool(self.host, self.port)

        response = pool.urlopen("GET", "/", retries=0, preload_content=False, timeout=Timeout(connect=1, read=0.001))
        try:
            self.assertRaises(ReadTimeoutError, response.read)
        finally:
            timed_out.set()
Ejemplo n.º 7
0
class OttProbe():
    """
    This class manages interactions with the supervisor:
    registration, specification retrievement, and return of results    
    """

    def __init__(self, config):
        """
        Initiates a OTT probe for component-initiated workflow. 
        Command line parameters ported to config file equivalents.  
        """

        self.config = config
        self.supervisorhost = None
        if "supervisorhost" in config["module_ott"]:
            self.supervisorhost = config["module_ott"]["supervisorhost"]
        self.supervisorhost = self.supervisorhost or SUPERVISOR_IP4
        
        self.supervisorport = None
        if "supervisorport" in config["module_ott"]:
            self.supervisorport = config["module_ott"]["supervisorport"]
        self.supervisorport = self.supervisorport or SUPERVISOR_PORT
        
        self.ip4addr = None
        if "ip4addr" in config["module_ott"]:
            self.ip4addr = config["module_ott"]["ip4addr"]
        self.ip4addr = self.ip4addr or IP4ADDR    
        print(">>> supervisor = " + self.supervisorhost + ":" + str(self.supervisorport) + ", ip4addr = "+ str(self.ip4addr) + "\n")
            
        tls_state = mplane.tls.TlsState(config)
        self.dn = self.get_dn( tls_state._keyfile, tls_state)
        self.forged_identity = mplane.tls.TlsState.extract_local_identity(tls_state, None)
        
        headers={"content-type": "application/x-mplane+json"}
        if tls_state._keyfile is None:
            if( self.forged_identity is not None ):
                headers={"content-type": "application/x-mplane+json","Forged-MPlane-Identity": self.forged_identity}
            self.pool = HTTPConnectionPool(self.supervisorhost, self.supervisorport, headers=headers)
        else:
            self.pool = HTTPSConnectionPool(self.supervisorhost, self.supervisorport, key_file=tls_state._keyfile, cert_file=tls_state._certfile, ca_certs=tls_state._cafile, headers=headers)

        self.scheduler = mplane.scheduler.Scheduler(self.config)
        if self.ip4addr is not None:
            self.scheduler.add_service(OttService(ott_capability(self.ip4addr)))
            # self.scheduler.add_service(OttService(cap(self.ip4addr)))

        
    def register_capabilities(self):
        print( ">>> register_capabilities(): register with supervisor at " + self.supervisorhost + ":" + str(self.supervisorport) )
        
        caps_list = ""
        for key in self.scheduler.capability_keys():
            cap = self.scheduler.capability_for_key(key)
            #if (self.scheduler.ac.check_azn(cap._label, self.dn)):
            if (self.scheduler.azn.check(cap, self.dn)):
                caps_list = caps_list + mplane.model.unparse_json(cap) + ","
        caps_list = "[" + caps_list[:-1].replace("\n","") + "]"
        
        print( ">>> register_capabilities(): caps_list = \n" + caps_list )

        if self.forged_identity is None:
            self.forged_identity = ""
        print( ">>> register_capabilities(): self.forged_identity = " + self.forged_identity )

        while True:
            try:
                # print(">>> register_capabilities(): urlopen with headers={content-type: application/x-mplane+json, Forged-MPlane-Identity: " + self.forged_identity + "})\n")
                res = self.pool.urlopen('POST', "/" + REGISTRATION_PATH, 
                    body=caps_list.encode("utf-8"), 
                    headers={"content-type": "application/x-mplane+json", "Forged-MPlane-Identity": self.forged_identity})
                
                if res.status == 200:
                    body = json.loads(res.data.decode("utf-8"))
                    print("\nCapability registration outcome:")
                    for key in body:
                        if body[key]['registered'] == "ok":
                            print(key + ": Ok")
                        else:
                            print(key + ": Failed (" + body[key]['reason'] + ")")
                    print("")
                else:
                    print("Error registering capabilities, Supervisor said: " + str(res.status) + " - " + res.data.decode("utf-8"))
                    exit(1)
                break
            except:
                print("Supervisor unreachable. Retrying connection in 5 seconds")
                sleep(5)
    

    def check_for_specs(self):
        """
        Poll the supervisor for specifications
        
        """
        url = "/" + SPECIFICATION_PATH
        
        # send a request for specifications
        res = self.pool.request('GET', url, headers={"Forged-MPlane-Identity": self.forged_identity})
        if res.status == 200:
            
            # specs retrieved: split them if there is more than one
            specs = mplane.utils.split_stmt_list(res.data.decode("utf-8"))
            for spec in specs:
                
                # hand spec to scheduler
                reply = self.scheduler.receive_message(self.dn, spec)
                # return error if spec is not authorized
                if isinstance(reply, mplane.model.Exception):
                    result_url = "/" + RESULT_PATH
                    # send result to the Supervisor
                    myheaders={}
                    if self.forged_identity is None:
                       myheaders={"content-type": "application/x-mplane+json"}
                    else:
                       myheaders={"content-type": "application/x-mplane+json", "Forged-MPlane-Identity": self.forged_identity}
                    res = self.pool.urlopen('POST', result_url, 
                            body=mplane.model.unparse_json(reply).encode("utf-8"), 
                            headers=myheaders)
                    return
                
                # enqueue job
                job = self.scheduler.job_for_message(reply)
                
                # launch a thread to monitor the status of the running measurement
                t = threading.Thread(target=self.return_results, args=[job])
                t.start()
                
        # not registered on supervisor, need to re-register
        elif res.status == 428:
            print("\nRe-registering capabilities with supervisor")
            self.register_to_supervisor()
        pass

        
    def return_results(self, job):
      """
      Monitors a job, and as soon as it is complete sends it to the Supervisor
        
      """
      try:
        url = "/" + RESULT_PATH
        reply = job.get_reply()
        
        # check if job is completed
        while job.finished() is not True:
            if job.failed():
              try:
                reply = job.get_reply()
                break
              except:
                print("Unexpected error in return_results job.get_reply():", sys.exc_info())
                raise
            sleep(1)
        if isinstance (reply, mplane.model.Receipt):
            reply = job.get_reply()
        
        # send result to the Supervisor
        res = self.pool.urlopen('POST', url, 
                body=mplane.model.unparse_json(reply).encode("utf-8") ) 

        # handle response
        if res.status == 200:
#            print("Result for " + reply.get_label() + " successfully returned!")
            print("Result successfully returned!")
        else:
            print("Error returning Result for " + reply.get_label())
            print("Supervisor said: " + str(res.status) + " - " + res.data.decode("utf-8"))
        pass
      except:
        print("Unexpected error in return_results:", sys.exc_info())
        raise
    
             
    def get_dn(self, security, tlsState):
    # def get_dn(self, security, cert):
        """
        Extracts the DN from the request object. 
        If SSL is disabled (ie no _keyfile supplied), returns a dummy DN
    
        """
        if security:
        # if security == True:
            self._tls = tlsState
            dn = self._tls._identity
            # dn = ""
            # for elem in cert.get('subject'):
                # if dn == "":
                    # dn = dn + str(elem[0][1])
                # else: 
                    # dn = dn + "." + str(elem[0][1])
        else:
            dn = DUMMY_DN
        # print ( ">>> get_dn(): dn = " + dn )
        return dn

        
    def process_args(self):
        global args
        global config   # bnj
        
        # Read the command line parameters
        parser = argparse.ArgumentParser(description="Run an OTT-probe")
        parser.add_argument('--ip4addr', '-4', metavar="source-v4-address",
                            help="Ping from the given IPv4 address")
        parser.add_argument('--disable-ssl', action='store_true', default=False, dest='DISABLE_SSL',
                            help='Disable secure communication')
        parser.add_argument('--config', metavar="config-file-location",
                            help="Location of the configuration file for certificates")
        parser.add_argument('--supervisorhost', metavar="supervisorhost",
                            help="IP or host name where supervisor runs (default: localhost)")
        parser.add_argument('--supervisorport', metavar="supervisorport",
                            help="port on which supervisor listens (default: 8888)")
        parser.add_argument('--forged-mplane-identity', metavar="forged_identity",
                            help="ID to use in non-secure mode instead of certificate's subject")
        args = parser.parse_args()
        
        # Read the configuration file
        config = configparser.ConfigParser()
        config.optionxform = str
        config.read(mplane.utils.search_path(args.config))
        #print("process_args(): config = " + str(config) + " self.config=" + str(self.config))
        print ("repr(config) = " + repr(config))
        
        self.forged_identity = args.forged_identity
        self.supervisorhost = args.supervisorhost or SUPERVISOR_IP4
        self.supervisorport = args.supervisorport or SUPERVISOR_PORT
        print ("process_args(): self.supervisorport = " + str(self.supervisorport) )
        
        self.ip4addr = None

        if args.ip4addr:
            self.ip4addr = ip_address(args.ip4addr)
            if self.ip4addr.version != 4:
                raise ValueError("invalid IPv4 address")
        if self.ip4addr is None :
             iplist = []
             [iplist.append(ip) for ip in socket.gethostbyname_ex(socket.gethostname())[2] if not ip.startswith("127.")][:1]
             print("Source address not defined. Lists of IPs found: " + ''.join(iplist) + " Using first: " + iplist[0])
             self.ip4addr = ip_address(iplist[0])
             if self.ip4addr.version != 4:
                raise ValueError("need at least one source address to run")
    
        if not args.DISABLE_SSL:
            if args.config is None:
                raise ValueError("without --disable-ssl, need to specify cert file")
            else:
                #print( "process_args(): pwd: " + os.getcwd() + " - cert config file: " + args.config )
                mplane.utils.check_file(args.config)
                # TODO: use search_path
                self.certfile = mplane.utils.normalize_path(mplane.utils.read_setting(args.config, "cert"))
                self.key = mplane.utils.normalize_path(mplane.utils.read_setting(args.config, "key"))
                self.ca = mplane.utils.normalize_path(mplane.utils.read_setting(args.config, "ca-chain"))
                mplane.utils.check_file(self.certfile)
                mplane.utils.check_file(self.key)
                mplane.utils.check_file(self.ca)
                print("process_args(): (2) self.certfile=" + self.certfile + " self.key=" + self.key + " self.ca=" + self.ca)
        else:
            self.certfile = None
            self.key = None
            self.ca = None
Ejemplo n.º 8
0
class HandleInfrastructure(DOInfrastructure):
    """
    Specialization of the general Digital Object Infrastructure based on the Handle System.
    Connects to the Handle System via a RESTful interface.
    """ 
    
    
    def __init__(self, host, port, user, user_index, password, path, prefix = None, additional_identifier_element = None, unsafe_ssl=False):
        '''
        Constructor.

        :param host: Host name of the Handle server.
        :param port: Port for the Handle server, usually 443 or 8443.
        :param user: Full user Handle name to authenticate for administrative permissions for modifying Handles. The full user name will usually be of the form 'prefix/suffix'.
        :param user_index: The index to use within the user Handle.
        :param password: The user password.
        :param path: The server path for the API endpoint, e.g. 'api/handles'. Does not include the host name.
        :param prefix: The Handle prefix to use (without trailing slash). If not given, all operations will work
          nonetheless, except for random handle creation. Note that setting a prefix does not mean that identifier 
          strings can omit it - all identifiers must ALWAYS include the prefix, no matter what.
        :param additional_identifier_element: A string that is inserted inbetween Handle prefix and suffix, e.g. if set
          to "test-", 10876/identifier becomes 10876/test-identifier.
        :unsafe_ssl: If set to True, SSL certificate warnings will be ignored. Do not activate this in productive environments!
        '''
        super(HandleInfrastructure, self).__init__()
        self._host = host
        self._port = port
        self._path = path
        self._prefix = prefix
        if unsafe_ssl:
            disable_warnings()
            self.__connpool = HTTPSConnectionPool(host, port=port, assert_hostname=False, cert_reqs="CERT_NONE")
        else:
            self.__connpool = HTTPSConnectionPool(host, port=port)
        self.__user_handle = prefix+"/"+user
        self.__user_index = user_index
        self.__authstring = b64encode(user_index+"%3A"+user+":"+password)
        self.__http_headers = {"Content-Type": "application/json", "Authorization": "Basic %s" % self.__authstring}
        if not self._path.endswith("/"):
            self._path = self._path + "/"
        self._additional_identifier_element = additional_identifier_element
            
    def _generate_random_identifier(self):
        if not self._prefix:
            raise ValueError("Cannot generate random Handles if no _prefix is provided!")
        rid = super(HandleInfrastructure, self)._generate_random_identifier()
        return self._prefix+"/"+rid

    def _prepare_identifier(self, identifier):
        # check identifier string for validity
        if " " in identifier:
            raise ValueError("Illegal Handle identifier string character; spaces are not supported! (identifier: %s)" % identifier)
        identifier = identifier.strip()
        if (self._additional_identifier_element):
            # split identifier into _prefix and suffix, insert additional element inbetween 
            parts = identifier.split("/", 1)
            if len(parts) != 2:
                raise ValueError("Invalid identifier - no separating slash between _prefix and suffix: %s" % identifier)
            if (parts[1].startswith(self._additional_identifier_element)):
                return self._path+identifier, identifier
            newident = parts[0]+"/"+self._additional_identifier_element+parts[1]
            return self._path+newident, newident
        else:
            return self._path+identifier, identifier
    
    def __generate_admin_value(self):
        return {"index":100,"type":"HS_ADMIN","data":{"format":"admin","value":{"handle":self.__user_handle,"index":self.__user_index,"permissions":"011111110011"}}}
    
    def _acquire_pid(self, identifier):
        path, identifier_prep = self._prepare_identifier(identifier)
        # Try to create Handle, but do not ovewrite existing
        values = {"values": [self.__generate_admin_value()]}
        resp = self.__connpool.urlopen("PUT", path+"?overwrite=false", str(values), self.__http_headers)
        # status check; 409 = Conflict on existing Handle
        if (resp.status == 409):
            raise PIDAlreadyExistsError("Handle already exists: %s" % identifier_prep)
        if not(200 <= resp.status <= 299):
            raise IOError("Could not create Handle %s: %s" % (identifier_prep, resp.reason))
        return identifier_prep
    
    def _do_from_json(self, piddata, identifier, aliases):
        """
        Construct a DO instance from given JSON data.
        
        :param piddata: JSON loaded data.
        :param identifier: Identifier of the DO.
        :param aliases: A list of aliases that were used to get to this identifier (may be empty). The list must be
          ordered in the order of alias resolution, i.e. aliases[0] pointed to aliases[1] etc. The last entry pointed 
          to the actual identifier. 
        :returns: A fully fledged DigitalObject instance
        """
        # piddata is an array of dicts, where each dict has keys: index, type, data
        references = {}
        res_type = None
        if not "values" in piddata:
            raise IOError("Illegal format of JSON response from Handle server: 'values' not found in JSON record!")
        for ele in piddata["values"]:
            idx = int(ele["index"])
            if idx == 2:
                res_type = ele["data"]["value"]
                continue
            if ele["type"] == "HS_ADMIN":
                # ignore HS_ADMIN values; these are taken care of by the REST service server-side
                continue
            # no special circumstances --> assign to annotations or references
            if REFERENCE_INDEX_END >= idx >= REFERENCE_INDEX_START:
                # reference; first, parse element data using json to a list
                list_data = json.loads(ele["data"]["value"])
                if not isinstance(list_data, list):
                    raise IOError("Illegal format of JSON response from Handle server: Cannot load reference list! Input: %s" % ele["data"])
                if ele["type"] not in references:
                    references[ele["type"]] = list_data
                else:
                    references[ele["type"]].extend(list_data)
                continue
        # create special instances for special resource types
        if res_type == DigitalObjectSet.RESOURCE_TYPE:
            return DigitalObjectSet(self, identifier, references=references, alias_identifiers=aliases)
        if res_type == DigitalObjectArray.RESOURCE_TYPE:
            return DigitalObjectArray(self, identifier, references=references, alias_identifiers=aliases)
        if res_type == DigitalObjectLinkedList.RESOURCE_TYPE:
            return DigitalObjectLinkedList(self, identifier, references=references, alias_identifiers=aliases)
        return DigitalObject(self, identifier, references, alias_identifiers=aliases)
        
    def lookup_pid(self, identifier):
        aliases = []
        while True:
            path, identifier = self._prepare_identifier(identifier)
            resp = self.__connpool.request("GET", path, None, self.__http_headers)
            if resp.status == 404:
                # Handle not found
                if len(aliases) > 0:
                    raise PIDAliasBrokenError("Alias %s does not exist. Already resolved aliases: %s" % (identifier, aliases))
                return None
            elif not(200 <= resp.status <= 299):
                raise IOError("Failed to look up Handle %s due to the following reason (HTTP Code %s): %s" % (identifier, resp.status, resp.reason))
            else:
                # check for HS_ALIAS redirect
                piddata = json.loads(resp.data)
                isa, alias_id = self._check_json_for_alias(piddata)
                if isa:
                    # write down alias identifier and redo lookup with target identifier
                    aliases.append(identifier)
                    identifier = alias_id
                    continue                    
                dobj = self._do_from_json(piddata, identifier, aliases)
                return dobj            
        
    def _determine_index(self, identifier, handledata, key, index_start, index_end=None):
        """
        Finds an index in the Handle key-metadata record to store a value for the given key. If the key is already
        present, its index will be reused. If it is not present, a free index will be determined. 
        
        :param identifier: The current Handle.
        :param key: The key that will be assigned.
        :param index_start: At which index the search should start.
        :param index_end: Where should the search end? Use None to search all indices greater than the start index.
        :raises: :exc:`IndexError` if all possible indices are already taken by other keys.
        :returns: an index value. 
        """
        matching_values = []
        free_index = index_start
        taken_indices = []        
        if not "values" in handledata:
            raise IOError("Illegal format of JSON response from Handle server: 'values' not found in JSON record!")
        for ele in handledata["values"]:
            idx = int(ele["index"])
            if (index_end and (index_start <= idx <= index_end))\
            or (not index_end and (index_start <= idx)):
                taken_indices.append(idx)
            if ele["type"] == key:
                matching_values.append(ele)
        if len(matching_values) > 1:
            raise IllegalHandleStructureError("Handle %s contains more than one entry of type %s!" % (identifier, key))
        elif len(matching_values) == 1:
            return int(matching_values[0]["index"])
        else:
            # key not present in Handle; must assign a new index
            # check for free index within bounds
            if taken_indices == []:
                return index_start
            m = min(taken_indices)
            if m == index_end:
                raise IllegalHandleStructureError("Handle %s does not have any more available index slots between %s and %s!" % (index_start, index_end))
            return m
        
    def _write_pid_value(self, identifier, index, valuetype, value):
        """
        Writes a single (index, type, value) to the Handle with given identifier.
        
        :param identifier: The Handle identifier.
        :param index: Index (positive 32 bit int).
        :param valuetype: Type (arbitrary)
        :param value: Value (arbitrary)
        """
        path, identifier = self._prepare_identifier(identifier)
        if type(index) is not int:
            raise ValueError("Index must be an integer! (was: type %s, value %s)" % (type(index), index))
        # write the raw (index, type, value) triple
        data = json.dumps([{"index": index, "type": valuetype, "data": {"format": "string", "value": value}}])
        resp = self.__connpool.urlopen("PUT", path+"?index=various", data, self.__http_headers)
        if not(200 <= resp.status <= 299):
            raise IOError("Could not write raw value to Handle %s: %s" % (identifier, resp.reason))
    
    def _read_pid_value(self, identifier, index):
        """
        Reads a single indexed type and value from the Handle with given identifier.
        
        :returns: A tuple (type, value) or None if the given index is unassigned.
        :raises: :exc:`IOError` if no Handle with given identifier exists. 
        """
        path, identifier = self._prepare_identifier(str(identifier))
        if type(index) is not int:
            raise ValueError("Index must be an integer! (was: type %s, value %s)" % (type(index), index))
        # read only the given index
        resp = self.__connpool.request("GET", path+"?index=%s" % index, "", self.__http_headers)
        if resp.status == 404:
            # value not found; the Handle may exist, but the index is unused
            return None        
        if not(200 <= resp.status <= 299):
            raise IOError("Could not read raw value from Handle %s: %s" % (identifier, resp.reason))
        respdata = json.loads(resp.data)
        if not "values" in respdata:
            raise IOError("Illegal format of JSON response from Handle server: 'values' not found in JSON record!")
        for ele in respdata["values"]:
            if int(ele["index"]) == index:
                return (ele["type"], ele["data"]["value"])
        return None
        
    def _remove_pid_value(self, identifier, index):
        """
        Removes a single Handle value at Handle of given identifier at given index.
        
        :raises: :exc:`IOError` if no Handle with given identifier exists. 
        """
        path, identifier = self._prepare_identifier(str(identifier))
        if type(index) is not int:
            raise ValueError("Index must be an integer! (was: type %s, value %s)" % (type(index), index))
        # read only the given index
        resp = self.__connpool.urlopen("DELETE", path+"?index=%s" % index, "", self.__http_headers)
        if not(200 <= resp.status <= 299):
            raise IOError("Could not remove raw value from Handle %s: %s" % (identifier, resp.reason))

    def _read_all_pid_values(self, identifier):
        """
        Reads the full Handle record of given identifier.
        
        :return: a dict with indexes as keys and (type, value) tuples as values.
        """
        path, identifier = self._prepare_identifier(identifier)
        # read full record
        resp = self.__connpool.request("GET", path, "", self.__http_headers)
        if not(200 <= resp.status <= 299):
            raise IOError("Could not read raw values from Handle %s: %s" % (identifier, resp.reason))
        respdata = json.loads(resp.data)
        res = {}
        if not "values" in respdata:
            raise IOError("Illegal format of JSON response from Handle server: 'values' not found in JSON record!")
        for ele in respdata["values"]:
            res[int(ele["index"])] = (ele["type"], ele["data"]["value"])
        return res
    
    def _write_resource_information(self, identifier, resource_location, resource_type=None):
        path, identifier = self._prepare_identifier(identifier)
        handle_values = []
        if resource_location:
            handle_values = [{"index": INDEX_RESOURCE_LOCATION, "type": "URL", "data": {"format": "string", "value": resource_location}}]
        if resource_type:
            handle_values.append({"index": INDEX_RESOURCE_TYPE, "type": "", "data": {"format": "string", "value": resource_type}})
        data = json.dumps(handle_values)
        resp = self.__connpool.urlopen("PUT", path, data, self.__http_headers)
        if not(200 <= resp.status <= 299):
            raise IOError("Could not write resource location to Handle %s: %s" % (identifier, resp.reason))

    def delete_do(self, identifier):
        path, identifier = self._prepare_identifier(identifier)
        resp = self.__connpool.urlopen("DELETE", path, headers=self.__http_headers)
        if resp.status == 404:
            raise KeyError("Handle not found: %s" % identifier)
        if not(200 <= resp.status <= 299):
            raise IOError("Could not delete Handle %s: %s" % (identifier, resp.reason))

    def _write_reference(self, identifier, key, reference):
        path, identifier = self._prepare_identifier(identifier)
        # first, we need to determine the index to use by looking at the key
        resp = self.__connpool.request("GET", path, headers=self.__http_headers)
        if not(200 <= resp.status <= 299):
            raise IOError("Unknown Handle: %s" % identifier)
        dodata = json.loads(resp.data)
        index = self._determine_index(identifier, dodata, key, REFERENCE_INDEX_START, REFERENCE_INDEX_END)
        # now we can write the reference; note that reference may be a list. But this is okay, we
        # convert it to a string and take care of reconversion in the JSON-to-DO method
        reference_s = json.dumps(reference)
        data = json.dumps({"values": [{"index": index, "type": key, "data": {"format": "string", "value": reference_s}}]})
        resp = self.__connpool.urlopen("PUT", path+"?index=various", data, self.__http_headers)
        if not(200 <= resp.status <= 299):
            raise IOError("Could not write references to Handle %s: %s" % (identifier, resp.reason))
            
            
    def create_alias(self, original, alias_identifier):
        if isinstance(original, DigitalObject):
            original_identifier = original.identifier
        else: 
            original_identifier = str(original)
        path, identifier = self._prepare_identifier(alias_identifier)
        # check for existing Handle
        resp = self.__connpool.request("GET", path, None, self.__http_headers)
        if (resp.status == 200):
            # Handle already exists
            raise PIDAlreadyExistsError("Handle already exists, cannot use it as an alias: %s" % identifier)
        if (resp.status != 404):
            raise IOError("Failed to check for existing Handle %s (HTTP Code %s): %s" % (identifier, resp.status, resp.reason))
        # okay, alias is available. Now create it.
        values = {"values": [self.__generate_admin_value(), {"index": 1, "type": "HS_ALIAS", "data": {"format": "string", "value": str(original_identifier)}}]}
        resp = self.__connpool.urlopen("PUT", path, str(values), self.__http_headers)
        if not(200 <= resp.status <= 299):
            raise IOError("Could not create Alias Handle %s: %s" % (identifier, resp.reason))
        return identifier
    
    def delete_alias(self, alias_identifier):
        # resolve to check if this is really an alias
        isa = self.is_alias(alias_identifier)
        if not isa:
            return False
        self.delete_do(alias_identifier)
        return True
    
    def is_alias(self, alias_identifier):
        path, identifier = self._prepare_identifier(alias_identifier)
        resp = self.__connpool.request("GET", path, None, self.__http_headers)
        if resp.status == 404:
            raise KeyError("Handle not found: %s" % identifier)
        if not(200 <= resp.status <= 299):
            raise IOError("Failed to lookup Handle %s for alias check: %s" % (identifier, resp.reason))
        # parse JSON, but do not create a Digital Object instance, as this might cause inefficient subsequent calls
        isa, a_id = self._check_json_for_alias(json.loads(resp.data))
        return isa

    def _check_json_for_alias(self, piddata):
        """
        Checks the given JSON data structure for presence of an HS_ALIAS marker.
        
        :returns: a tuple (b, id) where b is True or False and if b is True, id is the Handle string of the target
          Handle.
        """
        res = (False, None)
        for ele in piddata["values"]:
            if ele["type"] == "HS_ALIAS":
                res = (True, ele["data"]["value"])
                break
        return res        

    def prefix_pid(self, suffix):
        """
        Prepends a given (incomplete) identifier with the current Handle _prefix.
        """
        return self._prefix + "/" + suffix
    
    def manufacture_hashmap(self, identifier, characteristic_segment_number):
        """
        Factory method. Constructs Handle-based Hashmap implementation objects.
        
        :identifier: The PID of the record that should hold the hash map.
        :param: characteristic_segment_number: Since there can be multiple hash maps in a single record, this number
          is used to separate them from each other. 
        """
        return HandleHashmapImpl(self, identifier, characteristic_segment_number)