def create_database(self, database_document):
"""
Creates a database
@param database_document: has to be DatabaseDocument type
"""
if "Raven/DataDir" not in database_document.settings:
raise exceptions.InvalidOperationException(
"The Raven/DataDir setting is mandatory")
db_name = database_document.database_id.replace(
"Raven/Databases/", "")
Utils.name_validation(db_name)
path = "databases/{0}".format(Utils.quote_key(db_name))
response = self.requests_handler.http_request_handler(
path, "PUT", database_document.to_json(), admin=True)
if response.status_code == 502:
raise exceptions.ErrorResponseException(
"Connection failed please check your connection to {0}".
format(self.requests_handler.url))
if response.status_code != 200:
raise exceptions.ErrorResponseException(
"Database with the name '{0}' already exists".format(
database_document.database_id))
return response
def initialize(self):
if not self._initialize:
self._database_commands = database_commands.DatabaseCommands(
self._requests_handler)
if self.database is None:
raise exceptions.InvalidOperationException(
"None database is not valid")
if not self.database.lower() == self.conventions.system_database:
path = "Raven/Databases/{0}".format(self.database)
response = self._requests_handler.check_database_exists(
"docs?id=" + Utils.quote_key(path))
# here we unsure database exists if not create new one
if response.status_code == 404:
try:
raise exceptions.ErrorResponseException(
"Could not open database named: {0}, database does not exists"
.format(self.database))
except exceptions.ErrorResponseException:
print(traceback.format_exc())
self._database_commands.admin_commands.create_database(
DatabaseDocument(self.database, {
"Raven/DataDir":
"~\\{0}".format(self.database)
}))
self._requests_handler.get_replication_topology()
self.generator = HiloGenerator(self.conventions.max_ids_to_catch,
self._database_commands)
self._initialize = True
def put(self, key, document, metadata=None, etag=None):
"""
@param key: unique key under which document will be stored
:type str
@param document: document data
:type dict
@param metadata: document metadata
:type dict
@param etag: current document etag, used for concurrency checks (null to skip check)
:type str
@return: json file
:rtype: dict
"""
headers = None
if document is None:
document = {}
if metadata is None:
metadata = {}
if not isinstance(key, str):
raise ValueError("key must be {0}".format(type("")))
if not isinstance(document, dict) or not isinstance(metadata, dict):
raise ValueError("document and metadata must be dict")
data = [{"Key": key, "Document": document, "Metadata": metadata, "AdditionalData": None,
"Method": "PUT", "Etag": etag}]
if etag:
headers = {"if-None-Match": etag}
response = self._requests_handler.http_request_handler("bulk_docs", "POST", data=data, headers=headers).json()
if "Error" in response:
if "ActualEtag" in response:
raise exceptions.FetchConcurrencyException(response["Error"])
raise exceptions.ErrorResponseException(response["Error"][:85])
return response
def query(self,
index_name,
index_query,
includes=None,
metadata_only=False,
index_entries_only=False,
force_read_from_master=False):
"""
@param index_name: A name of an index to query
@param force_read_from_master: If True the reading also will be from the master
:type bool
:type str
@param index_query: A query definition containing all information required to query a specified index.
:type IndexQuery
@param includes: An array of relative paths that specify related documents ids
which should be included in a query result.
:type list
@param metadata_only: True if returned documents should include only metadata without a document body.
:type bool
@param index_entries_only: True if query results should contain only index entries.
:type bool
@return:json
:rtype:dict
"""
if not index_name:
raise ValueError("index_name cannot be None or empty")
if index_query is None:
raise ValueError("None query is invalid")
if not isinstance(index_query, IndexQuery):
raise ValueError("query must be IndexQuery type")
path = "indexes/{0}?".format(Utils.quote_key(index_name))
if index_query.default_operator is QueryOperator.AND:
path += "&operator={0}".format(index_query.default_operator.value)
if index_query.query:
path += "&query={0}".format(Utils.quote_key(index_query.query))
if index_query.sort_hints:
for hint in index_query.sort_hints:
path += "&{0}".format(hint)
if index_query.sort_fields:
for field in index_query.sort_fields:
path += "&sort={0}".format(field)
if index_query.fetch:
for item in index_query.fetch:
path += "&fetch={0}".format(item)
if metadata_only:
path += "&metadata-only=true"
if index_entries_only:
path += "&debug=entries"
if includes and len(includes) > 0:
path += "".join("&include=" + item for item in includes)
if index_query.start:
path += "&start={0}".format(index_query.start)
path += "&pageSize={0}".format(index_query.page_size)
response = self._requests_handler.http_request_handler(
path, "GET", force_read_from_master=force_read_from_master).json()
if "Error" in response:
raise exceptions.ErrorResponseException(response["Error"][:100])
return response
def set_response(self, response):
if response is None:
return None
if response.status_code != 201:
response = response.json()
if "Error" in response:
raise exceptions.ErrorResponseException(response["Error"])
return response.raw.data
def delete_database(self, db_name, hard_delete=False):
db_name = db_name.replace("Rave/Databases/", "")
path = "databases/{0}".format(Utils.quote_key(db_name))
if hard_delete:
path += "?hard-delete=true"
response = self.requests_handler.http_request_handler(path, "DELETE", admin=True)
if response.content != '' and response.content != b'':
raise exceptions.ErrorResponseException(response.content)
return response
def call_hilo(self, type_tag_name, max_id, etag):
headers = {"if-None-Match": "\"" + etag + "\""}
put_url = "docs/Raven%2FHilo%2F{0}".format(type_tag_name)
response = self.http_request_handler(put_url,
"PUT",
data={"Max": max_id},
headers=headers)
if response.status_code == 409:
raise exceptions.FetchConcurrencyException(response.json["Error"])
if response.status_code != 201:
raise exceptions.ErrorResponseException(
"Something is wrong with the request")
def set_response(self, response):
if response is None:
raise ValueError("Invalid response")
response = response.json()
if "Error" in response:
raise exceptions.ErrorResponseException(response["Error"])
if "Databases" not in response:
raise ValueError("Invalid response")
return response["Databases"]
def delete(self, key, etag=None):
if key is None:
raise ValueError("None Key is not valid")
if not isinstance(key, str):
raise ValueError("key must be {0}".format(type("")))
headers = {}
if etag is not None:
headers["If-None-Match"] = etag
key = Utils.quote_key(key)
path = "docs/{0}".format(key)
response = self._requests_handler.http_request_handler(path, "DELETE", headers=headers)
if response.status_code != 204:
raise exceptions.ErrorResponseException(response.json()["Error"])
def delete_by_index(self, index_name, query, options=None):
"""
@param index_name: name of an index to perform a query on
:type str
@param query: query that will be performed
:type IndexQuery
@param options: various operation options e.g. AllowStale or MaxOpsPerSec
:type BulkOperationOptions
@return: json
:rtype: dict
"""
path = Utils.build_path(index_name, query, options)
response = self._requests_handler.http_request_handler(path, "DELETE")
if response.status_code != 200 and response.status_code != 202:
try:
raise exceptions.ErrorResponseException(response.json()["Error"][:100])
except ValueError:
raise response.raise_for_status()
return response.json()
def query(self,
index_name,
index_query,
includes=None,
metadata_only=False,
index_entries_only=False,
force_read_from_master=False):
"""
@param index_name: A name of an index to query
@param force_read_from_master: If True the reading also will be from the master
:type bool
:type str
@param index_query: A query definition containing all information required to query a specified index.
:type IndexQuery
@param includes: An array of relative paths that specify related documents ids
which should be included in a query result.
:type list
@param metadata_only: True if returned documents should include only metadata without a document body.
:type bool
@param index_entries_only: True if query results should contain only index entries.
:type bool
@return:json
:rtype:dict
"""
if not index_name:
raise ValueError("index_name cannot be None or empty")
if index_query is None:
raise ValueError("None query is invalid")
if not isinstance(index_query, IndexQuery):
raise ValueError("index_query must be IndexQuery type")
path = "indexes/{0}?".format(Utils.quote_key(index_name))
path += self._build_query_request_path(
index_query=index_query,
includes=includes,
metadata_only=metadata_only,
index_entries_only=index_entries_only)
response = self._requests_handler.http_request_handler(
path, "GET", force_read_from_master=force_read_from_master).json()
if "Error" in response:
raise exceptions.ErrorResponseException(response["Error"][:100])
return response
def check_replication_change(self, topology_file):
try:
response = self.http_request_handler("replication/topology", "GET")
if response.status_code == 200:
topology = response.json()
with self.lock:
if self.topology != topology:
self.topology = topology
self.update_replication(topology_file)
elif response.status_code != 400 and response.status_code != 404 and not self.topology:
raise exceptions.ErrorResponseException(
"Could not connect to the database {0} please check the problem"
.format(self._primary_database))
except exceptions.InvalidOperationException:
pass
if not self.database.lower() == self.convention.system_database:
timer = Timer(60 * 5,
lambda: self.check_replication_change(topology_file))
timer.daemon = True
timer.start()
def open_session(self,
database=None,
api_key=None,
force_read_from_master=False):
self._assert_initialize()
session_id = uuid.uuid4()
database_commands_for_session = self._database_commands
if database is not None:
requests_handler = HttpRequestsFactory(self.url,
database,
self.conventions,
force_get_topology=True,
api_key=api_key)
path = "Raven/Databases/{0}".format(database)
response = requests_handler.check_database_exists(
"docs?id=" + Utils.quote_key(path))
if response.status_code != 200:
raise exceptions.ErrorResponseException(
"Could not open database named:{0}".format(database))
database_commands_for_session = database_commands.DatabaseCommands(
requests_handler)
return documentsession(database, self, database_commands_for_session,
session_id, force_read_from_master)
def _execute_with_replication(self,
path,
method,
headers,
data=None,
admin=False,
force_read_from_master=False,
uri="databases",
stream=False):
second_api_key = None
while True:
index = None
url = None
if not force_read_from_master:
if admin:
url = "{0}/admin/{1}".format(self._primary_url, path)
second_api_key = self.api_key
else:
if method == "GET":
if (path == "replication/topology"
or "Hilo" in path) and not self.primary:
raise exceptions.InvalidOperationException(
"Cant get access to {0} when {1}(primary) is Down"
.format(path, self._primary_database))
elif self.convention.failover_behavior == Failover.read_from_all_servers:
with self.lock:
self.request_count += 1
index = self.request_count % (
len(self.replication_topology) + 1)
if index != 0 or not self.primary: # if 0 use the primary
index -= 1
destination = self.replication_topology.peek(
index if index > 0 else 0)
url = "{0}/{1}/{2}/{3}".format(
destination["url"], uri,
destination["database"], path)
second_api_key = destination[
"credentials"].get("api_key", None)
elif not self.primary:
url = "{0}/{1}/{2}/{3}".format(
self.url, uri, self.database, path)
else:
if not self.primary:
if self.convention.failover_behavior == \
Failover.allow_reads_from_secondaries_and_writes_to_secondaries:
url = "{0}/{1}/{2}/{3}".format(
self.url, uri, self.database, path)
else:
raise exceptions.InvalidOperationException(
"Cant write to server when the primary is down when failover = {0}"
.format(self.convention.failover_behavior.
name))
if url is None:
if not self.primary:
raise exceptions.InvalidOperationException(
"Cant read or write to the master because {0} is down".
format(self._primary_database))
url = "{0}/{1}/{2}/{3}".format(self._primary_url, uri,
self._primary_database, path)
if uri != "databases":
url = "{0}/{1}".format(self._primary_url, path)
second_api_key = self.api_key
with requests.session() as session:
if headers is None:
headers = {}
headers.update(self.headers)
data = json.dumps(data,
default=self.convention.json_default_method)
response = session.request(method,
url=url,
data=data,
headers=headers,
stream=stream)
if response.status_code == 412 or response.status_code == 401:
try:
oauth_source = response.headers.__getitem__(
"OAuth-Source")
except KeyError:
raise exceptions.InvalidOperationException(
"Something is not right please check your server settings (do you use the right api_key)"
)
self.do_auth_request(self.api_key, oauth_source,
second_api_key)
continue
if (response.status_code == 503 or response.status_code == 502) and \
not self.replication_topology.empty() and not (
path == "replication/topology" or "Hilo" in path):
if self.primary:
if self.convention.failover_behavior == Failover.fail_immediately or force_read_from_master:
raise exceptions.ErrorResponseException(
"Failed to get response from server")
self.primary = False
self.is_alive(
{
"url": self._primary_url,
"database": self._primary_database
},
primary=True)
else:
with self.lock:
if not index:
index = 0
peek_item = self.replication_topology.peek(index)
if self.url == peek_item[
"url"] and self.database == peek_item[
"database"]:
self.is_alive(
self.replication_topology.get(index))
if self.replication_topology.empty():
raise exceptions.ErrorResponseException(
"Please check your databases")
destination = self.replication_topology.peek()
self.database = destination["database"]
self.url = destination["url"]
second_api_key = destination["credentials"].get(
"api_key", None)
continue
return response
def do_auth_request(self, api_key, oauth_source, second_api_key=None):
api_name, secret = api_key.split('/', 1)
tries = 1
headers = {"grant_type": "client_credentials"}
data = None
with requests.session() as session:
while True:
oath = session.request(method="POST",
url=oauth_source,
headers=headers,
data=data)
if oath.reason == "Precondition Failed":
if tries > 1:
if not (second_api_key and
self.api_key != second_api_key and tries < 3):
raise exceptions.ErrorResponseException(
"Unauthorized")
api_name, secret = second_api_key.split('/', 1)
tries += 1
authenticate = oath.headers.__getitem__(
"www-authenticate")[len("Raven "):]
challenge_dict = dict(
item.split("=", 1) for item in authenticate.split(','))
exponent_str = challenge_dict.get("exponent", None)
modulus_str = challenge_dict.get("modulus", None)
challenge = challenge_dict.get("challenge", None)
exponent = bytes_to_long(
base64.standard_b64decode(exponent_str))
modulus = bytes_to_long(
base64.standard_b64decode(modulus_str))
rsa = RSA.construct((modulus, exponent))
cipher = PKCS1_OAEP.new(rsa)
iv = get_random_bytes(16)
key = get_random_bytes(32)
encoder = PKCS7Encoder()
cipher_text = cipher.encrypt(key + iv)
results = []
results.extend(cipher_text)
aes = AES.new(key, AES.MODE_CBC, iv)
sub_data = Utils.dict_to_string({
"api key name":
api_name,
"challenge":
challenge,
"response":
base64.b64encode(
hashlib.sha1('{0};{1}'.format(
challenge, secret).encode('utf-8')).digest())
})
results.extend(aes.encrypt(encoder.encode(sub_data)))
data = Utils.dict_to_string({
"exponent":
exponent_str,
"modulus":
modulus_str,
"data":
base64.standard_b64encode(bytearray(results))
})
if exponent is None or modulus is None or challenge is None:
raise exceptions.InvalidOperationException(
"Invalid response from server, could not parse raven authentication information:{0} "
.format(authenticate))
tries += 1
elif oath.status_code == 200:
oath_json = oath.json()
body = oath_json["Body"]
signature = oath_json["Signature"]
if not sys.version_info.major > 2:
body = body.encode('utf-8')
signature = signature.encode('utf-8')
with self.lock:
self._token = "Bearer {0}".format({
"Body":
body,
"Signature":
signature
})
self.headers.update({"Authorization": self._token})
break
else:
raise exceptions.ErrorResponseException(oath.reason)
