def create_database(self, database_document): """ Creates a database @param database_document: has to be DatabaseDocument type """ if "Raven/DataDir" not in database_document.settings: raise exceptions.InvalidOperationException( "The Raven/DataDir setting is mandatory") db_name = database_document.database_id.replace( "Raven/Databases/", "") Utils.name_validation(db_name) path = "databases/{0}".format(Utils.quote_key(db_name)) response = self.requests_handler.http_request_handler( path, "PUT", database_document.to_json(), admin=True) if response.status_code == 502: raise exceptions.ErrorResponseException( "Connection failed please check your connection to {0}". format(self.requests_handler.url)) if response.status_code != 200: raise exceptions.ErrorResponseException( "Database with the name '{0}' already exists".format( database_document.database_id)) return response
def initialize(self): if not self._initialize: self._database_commands = database_commands.DatabaseCommands( self._requests_handler) if self.database is None: raise exceptions.InvalidOperationException( "None database is not valid") if not self.database.lower() == self.conventions.system_database: path = "Raven/Databases/{0}".format(self.database) response = self._requests_handler.check_database_exists( "docs?id=" + Utils.quote_key(path)) # here we unsure database exists if not create new one if response.status_code == 404: try: raise exceptions.ErrorResponseException( "Could not open database named: {0}, database does not exists" .format(self.database)) except exceptions.ErrorResponseException: print(traceback.format_exc()) self._database_commands.admin_commands.create_database( DatabaseDocument(self.database, { "Raven/DataDir": "~\\{0}".format(self.database) })) self._requests_handler.get_replication_topology() self.generator = HiloGenerator(self.conventions.max_ids_to_catch, self._database_commands) self._initialize = True
def put(self, key, document, metadata=None, etag=None): """ @param key: unique key under which document will be stored :type str @param document: document data :type dict @param metadata: document metadata :type dict @param etag: current document etag, used for concurrency checks (null to skip check) :type str @return: json file :rtype: dict """ headers = None if document is None: document = {} if metadata is None: metadata = {} if not isinstance(key, str): raise ValueError("key must be {0}".format(type(""))) if not isinstance(document, dict) or not isinstance(metadata, dict): raise ValueError("document and metadata must be dict") data = [{"Key": key, "Document": document, "Metadata": metadata, "AdditionalData": None, "Method": "PUT", "Etag": etag}] if etag: headers = {"if-None-Match": etag} response = self._requests_handler.http_request_handler("bulk_docs", "POST", data=data, headers=headers).json() if "Error" in response: if "ActualEtag" in response: raise exceptions.FetchConcurrencyException(response["Error"]) raise exceptions.ErrorResponseException(response["Error"][:85]) return response
def query(self, index_name, index_query, includes=None, metadata_only=False, index_entries_only=False, force_read_from_master=False): """ @param index_name: A name of an index to query @param force_read_from_master: If True the reading also will be from the master :type bool :type str @param index_query: A query definition containing all information required to query a specified index. :type IndexQuery @param includes: An array of relative paths that specify related documents ids which should be included in a query result. :type list @param metadata_only: True if returned documents should include only metadata without a document body. :type bool @param index_entries_only: True if query results should contain only index entries. :type bool @return:json :rtype:dict """ if not index_name: raise ValueError("index_name cannot be None or empty") if index_query is None: raise ValueError("None query is invalid") if not isinstance(index_query, IndexQuery): raise ValueError("query must be IndexQuery type") path = "indexes/{0}?".format(Utils.quote_key(index_name)) if index_query.default_operator is QueryOperator.AND: path += "&operator={0}".format(index_query.default_operator.value) if index_query.query: path += "&query={0}".format(Utils.quote_key(index_query.query)) if index_query.sort_hints: for hint in index_query.sort_hints: path += "&{0}".format(hint) if index_query.sort_fields: for field in index_query.sort_fields: path += "&sort={0}".format(field) if index_query.fetch: for item in index_query.fetch: path += "&fetch={0}".format(item) if metadata_only: path += "&metadata-only=true" if index_entries_only: path += "&debug=entries" if includes and len(includes) > 0: path += "".join("&include=" + item for item in includes) if index_query.start: path += "&start={0}".format(index_query.start) path += "&pageSize={0}".format(index_query.page_size) response = self._requests_handler.http_request_handler( path, "GET", force_read_from_master=force_read_from_master).json() if "Error" in response: raise exceptions.ErrorResponseException(response["Error"][:100]) return response
def set_response(self, response): if response is None: return None if response.status_code != 201: response = response.json() if "Error" in response: raise exceptions.ErrorResponseException(response["Error"]) return response.raw.data
def delete_database(self, db_name, hard_delete=False): db_name = db_name.replace("Rave/Databases/", "") path = "databases/{0}".format(Utils.quote_key(db_name)) if hard_delete: path += "?hard-delete=true" response = self.requests_handler.http_request_handler(path, "DELETE", admin=True) if response.content != '' and response.content != b'': raise exceptions.ErrorResponseException(response.content) return response
def call_hilo(self, type_tag_name, max_id, etag): headers = {"if-None-Match": "\"" + etag + "\""} put_url = "docs/Raven%2FHilo%2F{0}".format(type_tag_name) response = self.http_request_handler(put_url, "PUT", data={"Max": max_id}, headers=headers) if response.status_code == 409: raise exceptions.FetchConcurrencyException(response.json["Error"]) if response.status_code != 201: raise exceptions.ErrorResponseException( "Something is wrong with the request")
def set_response(self, response): if response is None: raise ValueError("Invalid response") response = response.json() if "Error" in response: raise exceptions.ErrorResponseException(response["Error"]) if "Databases" not in response: raise ValueError("Invalid response") return response["Databases"]
def delete(self, key, etag=None): if key is None: raise ValueError("None Key is not valid") if not isinstance(key, str): raise ValueError("key must be {0}".format(type(""))) headers = {} if etag is not None: headers["If-None-Match"] = etag key = Utils.quote_key(key) path = "docs/{0}".format(key) response = self._requests_handler.http_request_handler(path, "DELETE", headers=headers) if response.status_code != 204: raise exceptions.ErrorResponseException(response.json()["Error"])
def delete_by_index(self, index_name, query, options=None): """ @param index_name: name of an index to perform a query on :type str @param query: query that will be performed :type IndexQuery @param options: various operation options e.g. AllowStale or MaxOpsPerSec :type BulkOperationOptions @return: json :rtype: dict """ path = Utils.build_path(index_name, query, options) response = self._requests_handler.http_request_handler(path, "DELETE") if response.status_code != 200 and response.status_code != 202: try: raise exceptions.ErrorResponseException(response.json()["Error"][:100]) except ValueError: raise response.raise_for_status() return response.json()
def query(self, index_name, index_query, includes=None, metadata_only=False, index_entries_only=False, force_read_from_master=False): """ @param index_name: A name of an index to query @param force_read_from_master: If True the reading also will be from the master :type bool :type str @param index_query: A query definition containing all information required to query a specified index. :type IndexQuery @param includes: An array of relative paths that specify related documents ids which should be included in a query result. :type list @param metadata_only: True if returned documents should include only metadata without a document body. :type bool @param index_entries_only: True if query results should contain only index entries. :type bool @return:json :rtype:dict """ if not index_name: raise ValueError("index_name cannot be None or empty") if index_query is None: raise ValueError("None query is invalid") if not isinstance(index_query, IndexQuery): raise ValueError("index_query must be IndexQuery type") path = "indexes/{0}?".format(Utils.quote_key(index_name)) path += self._build_query_request_path( index_query=index_query, includes=includes, metadata_only=metadata_only, index_entries_only=index_entries_only) response = self._requests_handler.http_request_handler( path, "GET", force_read_from_master=force_read_from_master).json() if "Error" in response: raise exceptions.ErrorResponseException(response["Error"][:100]) return response
def check_replication_change(self, topology_file): try: response = self.http_request_handler("replication/topology", "GET") if response.status_code == 200: topology = response.json() with self.lock: if self.topology != topology: self.topology = topology self.update_replication(topology_file) elif response.status_code != 400 and response.status_code != 404 and not self.topology: raise exceptions.ErrorResponseException( "Could not connect to the database {0} please check the problem" .format(self._primary_database)) except exceptions.InvalidOperationException: pass if not self.database.lower() == self.convention.system_database: timer = Timer(60 * 5, lambda: self.check_replication_change(topology_file)) timer.daemon = True timer.start()
def open_session(self, database=None, api_key=None, force_read_from_master=False): self._assert_initialize() session_id = uuid.uuid4() database_commands_for_session = self._database_commands if database is not None: requests_handler = HttpRequestsFactory(self.url, database, self.conventions, force_get_topology=True, api_key=api_key) path = "Raven/Databases/{0}".format(database) response = requests_handler.check_database_exists( "docs?id=" + Utils.quote_key(path)) if response.status_code != 200: raise exceptions.ErrorResponseException( "Could not open database named:{0}".format(database)) database_commands_for_session = database_commands.DatabaseCommands( requests_handler) return documentsession(database, self, database_commands_for_session, session_id, force_read_from_master)
def _execute_with_replication(self, path, method, headers, data=None, admin=False, force_read_from_master=False, uri="databases", stream=False): second_api_key = None while True: index = None url = None if not force_read_from_master: if admin: url = "{0}/admin/{1}".format(self._primary_url, path) second_api_key = self.api_key else: if method == "GET": if (path == "replication/topology" or "Hilo" in path) and not self.primary: raise exceptions.InvalidOperationException( "Cant get access to {0} when {1}(primary) is Down" .format(path, self._primary_database)) elif self.convention.failover_behavior == Failover.read_from_all_servers: with self.lock: self.request_count += 1 index = self.request_count % ( len(self.replication_topology) + 1) if index != 0 or not self.primary: # if 0 use the primary index -= 1 destination = self.replication_topology.peek( index if index > 0 else 0) url = "{0}/{1}/{2}/{3}".format( destination["url"], uri, destination["database"], path) second_api_key = destination[ "credentials"].get("api_key", None) elif not self.primary: url = "{0}/{1}/{2}/{3}".format( self.url, uri, self.database, path) else: if not self.primary: if self.convention.failover_behavior == \ Failover.allow_reads_from_secondaries_and_writes_to_secondaries: url = "{0}/{1}/{2}/{3}".format( self.url, uri, self.database, path) else: raise exceptions.InvalidOperationException( "Cant write to server when the primary is down when failover = {0}" .format(self.convention.failover_behavior. name)) if url is None: if not self.primary: raise exceptions.InvalidOperationException( "Cant read or write to the master because {0} is down". format(self._primary_database)) url = "{0}/{1}/{2}/{3}".format(self._primary_url, uri, self._primary_database, path) if uri != "databases": url = "{0}/{1}".format(self._primary_url, path) second_api_key = self.api_key with requests.session() as session: if headers is None: headers = {} headers.update(self.headers) data = json.dumps(data, default=self.convention.json_default_method) response = session.request(method, url=url, data=data, headers=headers, stream=stream) if response.status_code == 412 or response.status_code == 401: try: oauth_source = response.headers.__getitem__( "OAuth-Source") except KeyError: raise exceptions.InvalidOperationException( "Something is not right please check your server settings (do you use the right api_key)" ) self.do_auth_request(self.api_key, oauth_source, second_api_key) continue if (response.status_code == 503 or response.status_code == 502) and \ not self.replication_topology.empty() and not ( path == "replication/topology" or "Hilo" in path): if self.primary: if self.convention.failover_behavior == Failover.fail_immediately or force_read_from_master: raise exceptions.ErrorResponseException( "Failed to get response from server") self.primary = False self.is_alive( { "url": self._primary_url, "database": self._primary_database }, primary=True) else: with self.lock: if not index: index = 0 peek_item = self.replication_topology.peek(index) if self.url == peek_item[ "url"] and self.database == peek_item[ "database"]: self.is_alive( self.replication_topology.get(index)) if self.replication_topology.empty(): raise exceptions.ErrorResponseException( "Please check your databases") destination = self.replication_topology.peek() self.database = destination["database"] self.url = destination["url"] second_api_key = destination["credentials"].get( "api_key", None) continue return response
def do_auth_request(self, api_key, oauth_source, second_api_key=None): api_name, secret = api_key.split('/', 1) tries = 1 headers = {"grant_type": "client_credentials"} data = None with requests.session() as session: while True: oath = session.request(method="POST", url=oauth_source, headers=headers, data=data) if oath.reason == "Precondition Failed": if tries > 1: if not (second_api_key and self.api_key != second_api_key and tries < 3): raise exceptions.ErrorResponseException( "Unauthorized") api_name, secret = second_api_key.split('/', 1) tries += 1 authenticate = oath.headers.__getitem__( "www-authenticate")[len("Raven "):] challenge_dict = dict( item.split("=", 1) for item in authenticate.split(',')) exponent_str = challenge_dict.get("exponent", None) modulus_str = challenge_dict.get("modulus", None) challenge = challenge_dict.get("challenge", None) exponent = bytes_to_long( base64.standard_b64decode(exponent_str)) modulus = bytes_to_long( base64.standard_b64decode(modulus_str)) rsa = RSA.construct((modulus, exponent)) cipher = PKCS1_OAEP.new(rsa) iv = get_random_bytes(16) key = get_random_bytes(32) encoder = PKCS7Encoder() cipher_text = cipher.encrypt(key + iv) results = [] results.extend(cipher_text) aes = AES.new(key, AES.MODE_CBC, iv) sub_data = Utils.dict_to_string({ "api key name": api_name, "challenge": challenge, "response": base64.b64encode( hashlib.sha1('{0};{1}'.format( challenge, secret).encode('utf-8')).digest()) }) results.extend(aes.encrypt(encoder.encode(sub_data))) data = Utils.dict_to_string({ "exponent": exponent_str, "modulus": modulus_str, "data": base64.standard_b64encode(bytearray(results)) }) if exponent is None or modulus is None or challenge is None: raise exceptions.InvalidOperationException( "Invalid response from server, could not parse raven authentication information:{0} " .format(authenticate)) tries += 1 elif oath.status_code == 200: oath_json = oath.json() body = oath_json["Body"] signature = oath_json["Signature"] if not sys.version_info.major > 2: body = body.encode('utf-8') signature = signature.encode('utf-8') with self.lock: self._token = "Bearer {0}".format({ "Body": body, "Signature": signature }) self.headers.update({"Authorization": self._token}) break else: raise exceptions.ErrorResponseException(oath.reason)