def delete(self, key_or_entity):
"""
@param key_or_entity:can be the key or the entity we like to delete
:type str or object:
"""
if key_or_entity is None:
raise ValueError("None key is invalid")
if not isinstance(key_or_entity, str):
self.delete_by_entity(key_or_entity)
self._known_missing_ids.add(key_or_entity)
if key_or_entity in self._entities_by_key:
entity = self._entities_by_key[key_or_entity]
if self._has_change(entity):
raise exceptions.InvalidOperationException(
"Can't delete changed entity using identifier. Use delete_by_entity(entity) instead."
)
if entity not in self._entities_and_metadata:
raise exceptions.InvalidOperationException(
"{0} is not associated with the session, cannot delete unknown entity instance"
.format(entity))
if "Raven-Read-Only" in self._entities_and_metadata[entity][
"original_metadata"]:
raise exceptions.InvalidOperationException(
"{0} is marked as read only and cannot be deleted".format(
entity))
self.delete_by_entity(entity)
return
self._defer_commands.add(
commands_data.DeleteCommandData(key_or_entity))
def store(self,
entity,
key=None,
etag=None,
force_concurrency_check=False):
"""
@param entity: Entity that will be stored
:type object:
@param key: Entity will be stored under this key, (None to generate automatically)
:type str:
@param etag: Current entity etag, used for concurrency checks (null to skip check)
:type str
@param force_concurrency_check:
:type bool
"""
if entity is None:
raise ValueError("None entity value is invalid")
if entity in self._entities_and_metadata:
if etag is not None:
self._entities_and_metadata[entity]["etag"] = etag
self._entities_and_metadata[entity][
"force_concurrency_check"] = force_concurrency_check
return
if key is None:
entity_id = GenerateEntityIdOnTheClient.try_get_id_from_instance(
entity)
else:
GenerateEntityIdOnTheClient.try_set_id_on_entity(entity, key)
entity_id = key
self.assert_no_non_unique_instance(entity, entity_id)
if not entity_id:
entity_id = self.document_store.generate_id(entity)
GenerateEntityIdOnTheClient.try_set_id_on_entity(entity, entity_id)
for command in self._defer_commands:
if command.key == entity_id:
raise exceptions.InvalidOperationException(
"Can't store document, there is a deferred command registered for this document in the session. "
"Document id: " + entity_id)
if entity in self._deleted_entities:
raise exceptions.InvalidOperationException(
"Can't store object, it was already deleted in this session. Document id: "
+ entity_id)
metadata = self.conventions.build_default_metadata(entity)
metadata["etag"] = etag
self._deleted_entities.discard(entity)
self.save_entity(entity_id,
entity, {},
metadata, {},
force_concurrency_check=force_concurrency_check)
def delete_by_entity(self, entity):
if entity is None:
raise ValueError("None entity is invalid")
if entity not in self._entities_and_metadata:
raise exceptions.InvalidOperationException(
"{0} is not associated with the session, cannot delete unknown entity instance"
.format(entity))
if "Raven-Read-Only" in self._entities_and_metadata[entity][
"original_metadata"]:
raise exceptions.InvalidOperationException(
"{0} is marked as read only and cannot be deleted".format(
entity))
self._deleted_entities.add(entity)
self._known_missing_ids.add(self._entities_and_metadata[entity]["key"])
def initialize(self):
if not self._initialize:
self._database_commands = database_commands.DatabaseCommands(
self._requests_handler)
if self.database is None:
raise exceptions.InvalidOperationException(
"None database is not valid")
if not self.database.lower() == self.conventions.system_database:
path = "Raven/Databases/{0}".format(self.database)
response = self._requests_handler.check_database_exists(
"docs?id=" + Utils.quote_key(path))
# here we unsure database exists if not create new one
if response.status_code == 404:
try:
raise exceptions.ErrorResponseException(
"Could not open database named: {0}, database does not exists"
.format(self.database))
except exceptions.ErrorResponseException:
print(traceback.format_exc())
self._database_commands.admin_commands.create_database(
DatabaseDocument(self.database, {
"Raven/DataDir":
"~\\{0}".format(self.database)
}))
self._requests_handler.get_replication_topology()
self.generator = HiloGenerator(self.conventions.max_ids_to_catch,
self._database_commands)
self._initialize = True
def create_database(self, database_document):
"""
Creates a database
@param database_document: has to be DatabaseDocument type
"""
if "Raven/DataDir" not in database_document.settings:
raise exceptions.InvalidOperationException(
"The Raven/DataDir setting is mandatory")
db_name = database_document.database_id.replace(
"Raven/Databases/", "")
Utils.name_validation(db_name)
path = "databases/{0}".format(Utils.quote_key(db_name))
response = self.requests_handler.http_request_handler(
path, "PUT", database_document.to_json(), admin=True)
if response.status_code == 502:
raise exceptions.ErrorResponseException(
"Connection failed please check your connection to {0}".
format(self.requests_handler.url))
if response.status_code != 200:
raise exceptions.ErrorResponseException(
"Database with the name '{0}' already exists".format(
database_document.database_id))
return response
def is_alive(self, destination, primary=False):
with requests.session() as session:
while True:
response = session.request(
"GET",
"{0}/databases/{1}/replication/topology?check-server-reachable"
.format(destination["url"], destination["database"]),
headers=self.headers)
if response.status_code == 412 or response.status_code == 401:
try:
try:
oauth_source = response.headers.__getitem__(
"OAuth-Source")
except KeyError:
raise exceptions.InvalidOperationException(
"Something is not right please check your server settings"
)
self.do_auth_request(self.api_key, oauth_source)
except exceptions.ErrorResponseException:
break
if response.status_code == 200:
if primary:
self.primary = True
else:
self.replication_topology.put(destination)
return
else:
break
is_alive_timer = Timer(5,
lambda: self.is_alive(destination, primary))
is_alive_timer.daemon = True
is_alive_timer.start()
def name_validation(name):
if name is None:
raise ValueError("None name is not valid")
result = re.match(r'([A-Za-z0-9_\-\.]+)', name, re.IGNORECASE)
if not result:
raise exceptions.InvalidOperationException(
"Database name can only contain only A-Z, a-z, \"_\", \".\" or \"-\" but was: " + name)
def convert_to_entity(document, object_type, conventions, nested_object_types=None, fetch=False):
metadata = document.pop("@metadata")
original_metadata = metadata.copy()
type_from_metadata = conventions.try_get_type_from_metadata(metadata)
if object_type == dict:
return document, metadata, original_metadata
entity = _DynamicStructure(**document)
if not fetch:
object_from_metadata = None
if type_from_metadata is not None:
object_from_metadata = Utils.import_class(type_from_metadata)
if object_from_metadata is None:
if object_type is not None:
entity.__class__ = object_type
metadata["Raven-Python-Type"] = "{0}.{1}".format(object_type.__module__, object_type.__name__)
else:
if object_type and not Utils.is_inherit(object_type, object_from_metadata):
raise exceptions.InvalidOperationException(
"Unable to cast object of type {0} to type {1}".format(object_from_metadata, object_type))
entity.__class__ = object_from_metadata
# Checking the class for initialize
entity_initialize_dict = Utils.make_initialize_dict(document, entity.__class__.__init__)
temp_entity = entity.__class__(**entity_initialize_dict)
for key, value in entity.__dict__.items():
if key not in entity_initialize_dict:
if hasattr(temp_entity, key):
setattr(temp_entity, key, value)
entity = temp_entity
if nested_object_types:
for key in nested_object_types:
attr = getattr(entity, key)
if attr:
try:
if isinstance(attr, list):
nested_list = []
for attribute in attr:
nested_list.append(nested_object_types[key](
**Utils.make_initialize_dict(attribute, nested_object_types[key].__init__)))
setattr(entity, key, nested_list)
elif nested_object_types[key] is datetime:
setattr(entity, key, Utils.string_to_datetime(attr))
elif nested_object_types[key] is timedelta:
setattr(entity, key, Utils.string_to_timedelta(attr))
else:
setattr(entity, key, nested_object_types[key](
**Utils.make_initialize_dict(attr, nested_object_types[key].__init__)))
except TypeError:
pass
if 'Id' in entity.__dict__:
entity.Id = metadata.get('@id', None)
return entity, metadata, original_metadata
def increment_requests_count(self):
self._number_of_requests_in_session += 1
if self._number_of_requests_in_session > self.conventions.max_number_of_request_per_session:
raise exceptions.InvalidOperationException(
"The maximum number of requests ({0}) allowed for this session has been reached. Raven limits the number \
of remote calls that a session is allowed to make as an early warning system. Sessions are expected to \
be short lived, and Raven provides facilities like batch saves (call save_changes() only once).\
You can increase the limit by setting DocumentConvention.\
MaxNumberOfRequestsPerSession or MaxNumberOfRequestsPerSession, but it is advisable \
that you'll look into reducing the number of remote calls first, \
since that will speed up your application significantly and result in a\
more responsive application.".format(
self.conventions.max_number_of_request_per_session))
def wait_for_operation_complete(self, operation_id, timeout=None):
start_time = time.time()
try:
path = "operation/status?id={0}".format(operation_id)
while True:
response = self.request_handler.http_request_handler(
path, "GET")
if response.status_code == 200:
response = response.json()
if timeout and time.time() - start_time > timeout:
raise exceptions.TimeoutException(
"The operation did not finish before the timeout end")
if response["Faulted"]:
if "Error" in response["State"]:
error = response["State"]["Error"]
else:
error = "Something went wrong with the operation"
raise exceptions.InvalidOperationException(error)
if response["Completed"]:
return response
time.sleep(0.5)
except ValueError as e:
raise exceptions.InvalidOperationException(e)
def save_changes(self):
data = _SaveChangesData(list(self._defer_commands),
len(self._defer_commands))
self._defer_commands.clear()
self._prepare_for_delete_commands(data)
self._prepare_for_puts_commands(data)
if len(data.commands) == 0:
return
self.increment_requests_count()
batch_result = self.database_commands.batch(data.commands)
if batch_result is None:
raise exceptions.InvalidOperationException(
"Cannot call Save Changes after the document store was disposed."
)
self._update_batch_result(batch_result, data)
def put_index(self, index_name, index_def, overwrite=False):
"""
@param index_name:The name of the index
@param index_def: IndexDefinition class a definition of a RavenIndex
@param overwrite: if set to True overwrite
"""
if index_name is None:
raise ValueError("None index_name is not valid")
if not isinstance(index_def, IndexDefinition):
raise ValueError("index_def must be IndexDefinition type")
index_name = Utils.quote_key(index_name)
path = "indexes/{0}?definition=yes".format(index_name)
response = self._requests_handler.http_request_handler(path, "GET")
if not overwrite and response.status_code != 404:
raise exceptions.InvalidOperationException("Cannot put index:{0},index already exists".format(index_name))
data = index_def.to_json()
return self._requests_handler.http_request_handler(path, "PUT", data=data).json()
def _assert_initialize(self):
if not self._initialize:
raise exceptions.InvalidOperationException(
"You cannot open a session or access the database commands before initializing the document store.\
Did you forget calling initialize()?")
def _execute_with_replication(self,
path,
method,
headers,
data=None,
admin=False,
force_read_from_master=False,
uri="databases",
stream=False):
second_api_key = None
while True:
index = None
url = None
if not force_read_from_master:
if admin:
url = "{0}/admin/{1}".format(self._primary_url, path)
second_api_key = self.api_key
else:
if method == "GET":
if (path == "replication/topology"
or "Hilo" in path) and not self.primary:
raise exceptions.InvalidOperationException(
"Cant get access to {0} when {1}(primary) is Down"
.format(path, self._primary_database))
elif self.convention.failover_behavior == Failover.read_from_all_servers:
with self.lock:
self.request_count += 1
index = self.request_count % (
len(self.replication_topology) + 1)
if index != 0 or not self.primary: # if 0 use the primary
index -= 1
destination = self.replication_topology.peek(
index if index > 0 else 0)
url = "{0}/{1}/{2}/{3}".format(
destination["url"], uri,
destination["database"], path)
second_api_key = destination[
"credentials"].get("api_key", None)
elif not self.primary:
url = "{0}/{1}/{2}/{3}".format(
self.url, uri, self.database, path)
else:
if not self.primary:
if self.convention.failover_behavior == \
Failover.allow_reads_from_secondaries_and_writes_to_secondaries:
url = "{0}/{1}/{2}/{3}".format(
self.url, uri, self.database, path)
else:
raise exceptions.InvalidOperationException(
"Cant write to server when the primary is down when failover = {0}"
.format(self.convention.failover_behavior.
name))
if url is None:
if not self.primary:
raise exceptions.InvalidOperationException(
"Cant read or write to the master because {0} is down".
format(self._primary_database))
url = "{0}/{1}/{2}/{3}".format(self._primary_url, uri,
self._primary_database, path)
if uri != "databases":
url = "{0}/{1}".format(self._primary_url, path)
second_api_key = self.api_key
with requests.session() as session:
if headers is None:
headers = {}
headers.update(self.headers)
data = json.dumps(data,
default=self.convention.json_default_method)
response = session.request(method,
url=url,
data=data,
headers=headers,
stream=stream)
if response.status_code == 412 or response.status_code == 401:
try:
oauth_source = response.headers.__getitem__(
"OAuth-Source")
except KeyError:
raise exceptions.InvalidOperationException(
"Something is not right please check your server settings (do you use the right api_key)"
)
self.do_auth_request(self.api_key, oauth_source,
second_api_key)
continue
if (response.status_code == 503 or response.status_code == 502) and \
not self.replication_topology.empty() and not (
path == "replication/topology" or "Hilo" in path):
if self.primary:
if self.convention.failover_behavior == Failover.fail_immediately or force_read_from_master:
raise exceptions.ErrorResponseException(
"Failed to get response from server")
self.primary = False
self.is_alive(
{
"url": self._primary_url,
"database": self._primary_database
},
primary=True)
else:
with self.lock:
if not index:
index = 0
peek_item = self.replication_topology.peek(index)
if self.url == peek_item[
"url"] and self.database == peek_item[
"database"]:
self.is_alive(
self.replication_topology.get(index))
if self.replication_topology.empty():
raise exceptions.ErrorResponseException(
"Please check your databases")
destination = self.replication_topology.peek()
self.database = destination["database"]
self.url = destination["url"]
second_api_key = destination["credentials"].get(
"api_key", None)
continue
return response
def do_auth_request(self, api_key, oauth_source, second_api_key=None):
api_name, secret = api_key.split('/', 1)
tries = 1
headers = {"grant_type": "client_credentials"}
data = None
with requests.session() as session:
while True:
oath = session.request(method="POST",
url=oauth_source,
headers=headers,
data=data)
if oath.reason == "Precondition Failed":
if tries > 1:
if not (second_api_key and
self.api_key != second_api_key and tries < 3):
raise exceptions.ErrorResponseException(
"Unauthorized")
api_name, secret = second_api_key.split('/', 1)
tries += 1
authenticate = oath.headers.__getitem__(
"www-authenticate")[len("Raven "):]
challenge_dict = dict(
item.split("=", 1) for item in authenticate.split(','))
exponent_str = challenge_dict.get("exponent", None)
modulus_str = challenge_dict.get("modulus", None)
challenge = challenge_dict.get("challenge", None)
exponent = bytes_to_long(
base64.standard_b64decode(exponent_str))
modulus = bytes_to_long(
base64.standard_b64decode(modulus_str))
rsa = RSA.construct((modulus, exponent))
cipher = PKCS1_OAEP.new(rsa)
iv = get_random_bytes(16)
key = get_random_bytes(32)
encoder = PKCS7Encoder()
cipher_text = cipher.encrypt(key + iv)
results = []
results.extend(cipher_text)
aes = AES.new(key, AES.MODE_CBC, iv)
sub_data = Utils.dict_to_string({
"api key name":
api_name,
"challenge":
challenge,
"response":
base64.b64encode(
hashlib.sha1('{0};{1}'.format(
challenge, secret).encode('utf-8')).digest())
})
results.extend(aes.encrypt(encoder.encode(sub_data)))
data = Utils.dict_to_string({
"exponent":
exponent_str,
"modulus":
modulus_str,
"data":
base64.standard_b64encode(bytearray(results))
})
if exponent is None or modulus is None or challenge is None:
raise exceptions.InvalidOperationException(
"Invalid response from server, could not parse raven authentication information:{0} "
.format(authenticate))
tries += 1
elif oath.status_code == 200:
oath_json = oath.json()
body = oath_json["Body"]
signature = oath_json["Signature"]
if not sys.version_info.major > 2:
body = body.encode('utf-8')
signature = signature.encode('utf-8')
with self.lock:
self._token = "Bearer {0}".format({
"Body":
body,
"Signature":
signature
})
self.headers.update({"Authorization": self._token})
break
else:
raise exceptions.ErrorResponseException(oath.reason)
