def delete(self, key_or_entity): """ @param key_or_entity:can be the key or the entity we like to delete :type str or object: """ if key_or_entity is None: raise ValueError("None key is invalid") if not isinstance(key_or_entity, str): self.delete_by_entity(key_or_entity) self._known_missing_ids.add(key_or_entity) if key_or_entity in self._entities_by_key: entity = self._entities_by_key[key_or_entity] if self._has_change(entity): raise exceptions.InvalidOperationException( "Can't delete changed entity using identifier. Use delete_by_entity(entity) instead." ) if entity not in self._entities_and_metadata: raise exceptions.InvalidOperationException( "{0} is not associated with the session, cannot delete unknown entity instance" .format(entity)) if "Raven-Read-Only" in self._entities_and_metadata[entity][ "original_metadata"]: raise exceptions.InvalidOperationException( "{0} is marked as read only and cannot be deleted".format( entity)) self.delete_by_entity(entity) return self._defer_commands.add( commands_data.DeleteCommandData(key_or_entity))
def store(self, entity, key=None, etag=None, force_concurrency_check=False): """ @param entity: Entity that will be stored :type object: @param key: Entity will be stored under this key, (None to generate automatically) :type str: @param etag: Current entity etag, used for concurrency checks (null to skip check) :type str @param force_concurrency_check: :type bool """ if entity is None: raise ValueError("None entity value is invalid") if entity in self._entities_and_metadata: if etag is not None: self._entities_and_metadata[entity]["etag"] = etag self._entities_and_metadata[entity][ "force_concurrency_check"] = force_concurrency_check return if key is None: entity_id = GenerateEntityIdOnTheClient.try_get_id_from_instance( entity) else: GenerateEntityIdOnTheClient.try_set_id_on_entity(entity, key) entity_id = key self.assert_no_non_unique_instance(entity, entity_id) if not entity_id: entity_id = self.document_store.generate_id(entity) GenerateEntityIdOnTheClient.try_set_id_on_entity(entity, entity_id) for command in self._defer_commands: if command.key == entity_id: raise exceptions.InvalidOperationException( "Can't store document, there is a deferred command registered for this document in the session. " "Document id: " + entity_id) if entity in self._deleted_entities: raise exceptions.InvalidOperationException( "Can't store object, it was already deleted in this session. Document id: " + entity_id) metadata = self.conventions.build_default_metadata(entity) metadata["etag"] = etag self._deleted_entities.discard(entity) self.save_entity(entity_id, entity, {}, metadata, {}, force_concurrency_check=force_concurrency_check)
def delete_by_entity(self, entity): if entity is None: raise ValueError("None entity is invalid") if entity not in self._entities_and_metadata: raise exceptions.InvalidOperationException( "{0} is not associated with the session, cannot delete unknown entity instance" .format(entity)) if "Raven-Read-Only" in self._entities_and_metadata[entity][ "original_metadata"]: raise exceptions.InvalidOperationException( "{0} is marked as read only and cannot be deleted".format( entity)) self._deleted_entities.add(entity) self._known_missing_ids.add(self._entities_and_metadata[entity]["key"])
def initialize(self): if not self._initialize: self._database_commands = database_commands.DatabaseCommands( self._requests_handler) if self.database is None: raise exceptions.InvalidOperationException( "None database is not valid") if not self.database.lower() == self.conventions.system_database: path = "Raven/Databases/{0}".format(self.database) response = self._requests_handler.check_database_exists( "docs?id=" + Utils.quote_key(path)) # here we unsure database exists if not create new one if response.status_code == 404: try: raise exceptions.ErrorResponseException( "Could not open database named: {0}, database does not exists" .format(self.database)) except exceptions.ErrorResponseException: print(traceback.format_exc()) self._database_commands.admin_commands.create_database( DatabaseDocument(self.database, { "Raven/DataDir": "~\\{0}".format(self.database) })) self._requests_handler.get_replication_topology() self.generator = HiloGenerator(self.conventions.max_ids_to_catch, self._database_commands) self._initialize = True
def create_database(self, database_document): """ Creates a database @param database_document: has to be DatabaseDocument type """ if "Raven/DataDir" not in database_document.settings: raise exceptions.InvalidOperationException( "The Raven/DataDir setting is mandatory") db_name = database_document.database_id.replace( "Raven/Databases/", "") Utils.name_validation(db_name) path = "databases/{0}".format(Utils.quote_key(db_name)) response = self.requests_handler.http_request_handler( path, "PUT", database_document.to_json(), admin=True) if response.status_code == 502: raise exceptions.ErrorResponseException( "Connection failed please check your connection to {0}". format(self.requests_handler.url)) if response.status_code != 200: raise exceptions.ErrorResponseException( "Database with the name '{0}' already exists".format( database_document.database_id)) return response
def is_alive(self, destination, primary=False): with requests.session() as session: while True: response = session.request( "GET", "{0}/databases/{1}/replication/topology?check-server-reachable" .format(destination["url"], destination["database"]), headers=self.headers) if response.status_code == 412 or response.status_code == 401: try: try: oauth_source = response.headers.__getitem__( "OAuth-Source") except KeyError: raise exceptions.InvalidOperationException( "Something is not right please check your server settings" ) self.do_auth_request(self.api_key, oauth_source) except exceptions.ErrorResponseException: break if response.status_code == 200: if primary: self.primary = True else: self.replication_topology.put(destination) return else: break is_alive_timer = Timer(5, lambda: self.is_alive(destination, primary)) is_alive_timer.daemon = True is_alive_timer.start()
def name_validation(name): if name is None: raise ValueError("None name is not valid") result = re.match(r'([A-Za-z0-9_\-\.]+)', name, re.IGNORECASE) if not result: raise exceptions.InvalidOperationException( "Database name can only contain only A-Z, a-z, \"_\", \".\" or \"-\" but was: " + name)
def convert_to_entity(document, object_type, conventions, nested_object_types=None, fetch=False): metadata = document.pop("@metadata") original_metadata = metadata.copy() type_from_metadata = conventions.try_get_type_from_metadata(metadata) if object_type == dict: return document, metadata, original_metadata entity = _DynamicStructure(**document) if not fetch: object_from_metadata = None if type_from_metadata is not None: object_from_metadata = Utils.import_class(type_from_metadata) if object_from_metadata is None: if object_type is not None: entity.__class__ = object_type metadata["Raven-Python-Type"] = "{0}.{1}".format(object_type.__module__, object_type.__name__) else: if object_type and not Utils.is_inherit(object_type, object_from_metadata): raise exceptions.InvalidOperationException( "Unable to cast object of type {0} to type {1}".format(object_from_metadata, object_type)) entity.__class__ = object_from_metadata # Checking the class for initialize entity_initialize_dict = Utils.make_initialize_dict(document, entity.__class__.__init__) temp_entity = entity.__class__(**entity_initialize_dict) for key, value in entity.__dict__.items(): if key not in entity_initialize_dict: if hasattr(temp_entity, key): setattr(temp_entity, key, value) entity = temp_entity if nested_object_types: for key in nested_object_types: attr = getattr(entity, key) if attr: try: if isinstance(attr, list): nested_list = [] for attribute in attr: nested_list.append(nested_object_types[key]( **Utils.make_initialize_dict(attribute, nested_object_types[key].__init__))) setattr(entity, key, nested_list) elif nested_object_types[key] is datetime: setattr(entity, key, Utils.string_to_datetime(attr)) elif nested_object_types[key] is timedelta: setattr(entity, key, Utils.string_to_timedelta(attr)) else: setattr(entity, key, nested_object_types[key]( **Utils.make_initialize_dict(attr, nested_object_types[key].__init__))) except TypeError: pass if 'Id' in entity.__dict__: entity.Id = metadata.get('@id', None) return entity, metadata, original_metadata
def increment_requests_count(self): self._number_of_requests_in_session += 1 if self._number_of_requests_in_session > self.conventions.max_number_of_request_per_session: raise exceptions.InvalidOperationException( "The maximum number of requests ({0}) allowed for this session has been reached. Raven limits the number \ of remote calls that a session is allowed to make as an early warning system. Sessions are expected to \ be short lived, and Raven provides facilities like batch saves (call save_changes() only once).\ You can increase the limit by setting DocumentConvention.\ MaxNumberOfRequestsPerSession or MaxNumberOfRequestsPerSession, but it is advisable \ that you'll look into reducing the number of remote calls first, \ since that will speed up your application significantly and result in a\ more responsive application.".format( self.conventions.max_number_of_request_per_session))
def wait_for_operation_complete(self, operation_id, timeout=None): start_time = time.time() try: path = "operation/status?id={0}".format(operation_id) while True: response = self.request_handler.http_request_handler( path, "GET") if response.status_code == 200: response = response.json() if timeout and time.time() - start_time > timeout: raise exceptions.TimeoutException( "The operation did not finish before the timeout end") if response["Faulted"]: if "Error" in response["State"]: error = response["State"]["Error"] else: error = "Something went wrong with the operation" raise exceptions.InvalidOperationException(error) if response["Completed"]: return response time.sleep(0.5) except ValueError as e: raise exceptions.InvalidOperationException(e)
def save_changes(self): data = _SaveChangesData(list(self._defer_commands), len(self._defer_commands)) self._defer_commands.clear() self._prepare_for_delete_commands(data) self._prepare_for_puts_commands(data) if len(data.commands) == 0: return self.increment_requests_count() batch_result = self.database_commands.batch(data.commands) if batch_result is None: raise exceptions.InvalidOperationException( "Cannot call Save Changes after the document store was disposed." ) self._update_batch_result(batch_result, data)
def put_index(self, index_name, index_def, overwrite=False): """ @param index_name:The name of the index @param index_def: IndexDefinition class a definition of a RavenIndex @param overwrite: if set to True overwrite """ if index_name is None: raise ValueError("None index_name is not valid") if not isinstance(index_def, IndexDefinition): raise ValueError("index_def must be IndexDefinition type") index_name = Utils.quote_key(index_name) path = "indexes/{0}?definition=yes".format(index_name) response = self._requests_handler.http_request_handler(path, "GET") if not overwrite and response.status_code != 404: raise exceptions.InvalidOperationException("Cannot put index:{0},index already exists".format(index_name)) data = index_def.to_json() return self._requests_handler.http_request_handler(path, "PUT", data=data).json()
def _assert_initialize(self): if not self._initialize: raise exceptions.InvalidOperationException( "You cannot open a session or access the database commands before initializing the document store.\ Did you forget calling initialize()?")
def _execute_with_replication(self, path, method, headers, data=None, admin=False, force_read_from_master=False, uri="databases", stream=False): second_api_key = None while True: index = None url = None if not force_read_from_master: if admin: url = "{0}/admin/{1}".format(self._primary_url, path) second_api_key = self.api_key else: if method == "GET": if (path == "replication/topology" or "Hilo" in path) and not self.primary: raise exceptions.InvalidOperationException( "Cant get access to {0} when {1}(primary) is Down" .format(path, self._primary_database)) elif self.convention.failover_behavior == Failover.read_from_all_servers: with self.lock: self.request_count += 1 index = self.request_count % ( len(self.replication_topology) + 1) if index != 0 or not self.primary: # if 0 use the primary index -= 1 destination = self.replication_topology.peek( index if index > 0 else 0) url = "{0}/{1}/{2}/{3}".format( destination["url"], uri, destination["database"], path) second_api_key = destination[ "credentials"].get("api_key", None) elif not self.primary: url = "{0}/{1}/{2}/{3}".format( self.url, uri, self.database, path) else: if not self.primary: if self.convention.failover_behavior == \ Failover.allow_reads_from_secondaries_and_writes_to_secondaries: url = "{0}/{1}/{2}/{3}".format( self.url, uri, self.database, path) else: raise exceptions.InvalidOperationException( "Cant write to server when the primary is down when failover = {0}" .format(self.convention.failover_behavior. name)) if url is None: if not self.primary: raise exceptions.InvalidOperationException( "Cant read or write to the master because {0} is down". format(self._primary_database)) url = "{0}/{1}/{2}/{3}".format(self._primary_url, uri, self._primary_database, path) if uri != "databases": url = "{0}/{1}".format(self._primary_url, path) second_api_key = self.api_key with requests.session() as session: if headers is None: headers = {} headers.update(self.headers) data = json.dumps(data, default=self.convention.json_default_method) response = session.request(method, url=url, data=data, headers=headers, stream=stream) if response.status_code == 412 or response.status_code == 401: try: oauth_source = response.headers.__getitem__( "OAuth-Source") except KeyError: raise exceptions.InvalidOperationException( "Something is not right please check your server settings (do you use the right api_key)" ) self.do_auth_request(self.api_key, oauth_source, second_api_key) continue if (response.status_code == 503 or response.status_code == 502) and \ not self.replication_topology.empty() and not ( path == "replication/topology" or "Hilo" in path): if self.primary: if self.convention.failover_behavior == Failover.fail_immediately or force_read_from_master: raise exceptions.ErrorResponseException( "Failed to get response from server") self.primary = False self.is_alive( { "url": self._primary_url, "database": self._primary_database }, primary=True) else: with self.lock: if not index: index = 0 peek_item = self.replication_topology.peek(index) if self.url == peek_item[ "url"] and self.database == peek_item[ "database"]: self.is_alive( self.replication_topology.get(index)) if self.replication_topology.empty(): raise exceptions.ErrorResponseException( "Please check your databases") destination = self.replication_topology.peek() self.database = destination["database"] self.url = destination["url"] second_api_key = destination["credentials"].get( "api_key", None) continue return response
def do_auth_request(self, api_key, oauth_source, second_api_key=None): api_name, secret = api_key.split('/', 1) tries = 1 headers = {"grant_type": "client_credentials"} data = None with requests.session() as session: while True: oath = session.request(method="POST", url=oauth_source, headers=headers, data=data) if oath.reason == "Precondition Failed": if tries > 1: if not (second_api_key and self.api_key != second_api_key and tries < 3): raise exceptions.ErrorResponseException( "Unauthorized") api_name, secret = second_api_key.split('/', 1) tries += 1 authenticate = oath.headers.__getitem__( "www-authenticate")[len("Raven "):] challenge_dict = dict( item.split("=", 1) for item in authenticate.split(',')) exponent_str = challenge_dict.get("exponent", None) modulus_str = challenge_dict.get("modulus", None) challenge = challenge_dict.get("challenge", None) exponent = bytes_to_long( base64.standard_b64decode(exponent_str)) modulus = bytes_to_long( base64.standard_b64decode(modulus_str)) rsa = RSA.construct((modulus, exponent)) cipher = PKCS1_OAEP.new(rsa) iv = get_random_bytes(16) key = get_random_bytes(32) encoder = PKCS7Encoder() cipher_text = cipher.encrypt(key + iv) results = [] results.extend(cipher_text) aes = AES.new(key, AES.MODE_CBC, iv) sub_data = Utils.dict_to_string({ "api key name": api_name, "challenge": challenge, "response": base64.b64encode( hashlib.sha1('{0};{1}'.format( challenge, secret).encode('utf-8')).digest()) }) results.extend(aes.encrypt(encoder.encode(sub_data))) data = Utils.dict_to_string({ "exponent": exponent_str, "modulus": modulus_str, "data": base64.standard_b64encode(bytearray(results)) }) if exponent is None or modulus is None or challenge is None: raise exceptions.InvalidOperationException( "Invalid response from server, could not parse raven authentication information:{0} " .format(authenticate)) tries += 1 elif oath.status_code == 200: oath_json = oath.json() body = oath_json["Body"] signature = oath_json["Signature"] if not sys.version_info.major > 2: body = body.encode('utf-8') signature = signature.encode('utf-8') with self.lock: self._token = "Bearer {0}".format({ "Body": body, "Signature": signature }) self.headers.update({"Authorization": self._token}) break else: raise exceptions.ErrorResponseException(oath.reason)