Пример #1
0
 def _replace_index(self, new_index, alias_list):
     """Replace an existing index based on an alias."""
     update = dict([('actions', [])])
     old_index = []
     reference_alias = next(iter(alias_list or []), None)
     prefix = "{0}_".format(reference_alias)
     try:
         alias_indexes = self._alias_index_list().get(reference_alias)
         if alias_indexes:
             old_index = [x for x in alias_indexes if x.startswith(prefix)]
         for alias in alias_list:
             update["actions"].append(
                 {"add": {
                     "index": new_index,
                     "alias": alias
                 }})
             for elem in old_index:
                 update["actions"].append(
                     {"remove": {
                         "index": elem,
                         "alias": alias
                     }})
         self.es.indices.update_aliases(body=json.dumps(update))
         self._index_delete(old_index)
         app_logger.info("Replace index finished.")
     except Exception as error:
         app_logger.error('Something is wrong: {0}'.format(error))
         raise
     finally:
         return json.dumps(update)
Пример #2
0
    def __call__(self, message):
        """Process the RPC Payload.

        :param Message message:
        :return:
        """
        try:
            processed_message = self._handle_message(message)
        except Exception as e:
            app_logger.error('Something went wrong: {0}'.format(e))
            properties = {'correlation_id': message.correlation_id}
            error_message = "Error Type: {0}, with message: {1}".format(
                e.__class__.__name__, e.message)
            message_data = json.loads(message.body)
            processed_message = response_message(message_data["provenance"],
                                                 status="error",
                                                 status_messsage=error_message)
            response = Message.create(
                message.channel,
                str(
                    json.dumps(processed_message,
                               indent=4,
                               separators=(',', ': '))), properties)
            response.publish(message.reply_to)
            message.reject(requeue=False)
        else:
            properties = {'correlation_id': message.correlation_id}
            response = Message.create(message.channel, processed_message,
                                      properties)
            response.publish(message.reply_to)
            message.ack()
Пример #3
0
 def _bulk_index(self, doc_type, data, target_index):
     """Index one or more documents via the bulk operation."""
     try:
         if doc_type is None:
             doc_type = "General"
         self.es.bulk(index=target_index,
                      doc_type=doc_type,
                      body=data,
                      refresh=True)
         app_logger.info(
             "Bulk Index in Elasticsearch at index: \"{0}\" with type: \"{1}\"."
             .format(target_index, doc_type))
     except Exception as error:
         app_logger.error('Something is wrong: {0}'.format(error))
         raise
Пример #4
0
 def _api_index(self, doc_id, doc_type, data, target_index):
     """Index single document from an ."""
     try:
         if doc_type is None:
             doc_type = "General"
         self.es.index(index=target_index,
                       id=doc_id,
                       doc_type=doc_type,
                       body=data,
                       refresh=True)
         app_logger.info(
             "Index document {0} in Elasticsearch at index: \"{1}\" with type: \"{2}\"."
             .format(doc_id, target_index, doc_type))
     except Exception as error:
         app_logger.error('Something is wrong: {0}'.format(error))
         raise
Пример #5
0
 def _add_alias(self, new_index, alias_list):
     """Add index based on an alias."""
     update = dict([('actions', [])])
     try:
         for alias in alias_list:
             update["actions"].append(
                 {"add": {
                     "index": new_index,
                     "alias": alias
                 }})
         self.es.indices.update_aliases(body=json.dumps(update))
         app_logger.info("Add alias finished.")
     except Exception as error:
         app_logger.error('Something is wrong: {0}'.format(error))
         raise
     finally:
         return json.dumps(update)
Пример #6
0
def add_message(message_data, replace_index, alias_list):
    """Index data and associate aliases to the indexes."""
    startTime = datetime.now().strftime('%Y-%m-%dT%H:%M:%S')
    PUBLISHER = Publisher(broker['host'], broker['user'], broker['pass'], broker['provqueue'])
    elastic = ElasticIndex()
    try:
        elastic._add_alias(replace_index, alias_list)
        endTime = datetime.now().strftime('%Y-%m-%dT%H:%M:%S')
        if bool(message_data["provenance"]):
            PUBLISHER.push(prov_message(message_data, "success", startTime, endTime, replace_index))
        app_logger.info('Indexed data with: {0} aliases'.format(alias_list))
        return json.dumps(response_message(message_data["provenance"], status="success"), indent=4, separators=(',', ': '))
    except Exception as error:
        endTime = datetime.now().strftime('%Y-%m-%dT%H:%M:%S')
        PUBLISHER.push(prov_message(message_data, "error", startTime, endTime, replace_index))
        app_logger.error('Something is wrong: {0}'.format(error))
        raise
Пример #7
0
 def _index_create(self, reference_alias, target_index=None):
     """Create a new index if it does not exist."""
     try:
         if target_index is not None and not (
                 self.es.indices.exists(target_index)):
             self.es.indices.create(index=target_index,
                                    ignore=400,
                                    refresh=True)
         else:
             target_index = "{0}_service_{1}".format(
                 reference_alias,
                 datetime.now().strftime('%Y%m%d_%H%M%S'))
         app_logger.info("New index created: \"{0}\".".format(target_index))
     except Exception as error:
         app_logger.error('Something is wrong: {0}'.format(error))
         raise
     finally:
         return target_index
Пример #8
0
def replace_message(message_data, replace_index, alias_list):
    """Replace an old index with a new index for a given alias list."""
    startTime = datetime.now().strftime('%Y-%m-%dT%H:%M:%S')
    PUBLISHER = Publisher(broker['host'], broker['user'], broker['pass'], broker['provqueue'])
    elastic = ElasticIndex()
    try:
        # This is what makes it a replace operation.
        elastic._replace_index(replace_index, alias_list)
        endTime = datetime.now().strftime('%Y-%m-%dT%H:%M:%S')
        if bool(message_data["provenance"]):
            PUBLISHER.push(prov_message(message_data, "success", startTime, endTime, replace_index))
        app_logger.info('Replaced Indexed data with: {0} aliases'.format(alias_list))
        return json.dumps(response_message(message_data["provenance"], status="success"), indent=4, separators=(',', ': '))
    except Exception as error:
        endTime = datetime.now().strftime('%Y-%m-%dT%H:%M:%S')
        PUBLISHER.push(prov_message(message_data, "error", startTime, endTime, replace_index))
        app_logger.error('Something is wrong: {0}'.format(error))
        raise
Пример #9
0
def retrieve_data(inputType, input_data):
    """Retrieve data from a specific URI."""
    s = requests.Session()
    allowed = ('http', 'https', 'ftp')
    local = ('file')
    if inputType == "Data":
        return input_data
    elif inputType == "URI":
        try:
            if urlparse(input_data).scheme in allowed:
                request = s.get(input_data, timeout=1)
                return request.text
            elif urlparse(input_data).scheme in local:
                s.mount('file://', FileAdapter())
                request = s.get(input_data)
                return handle_fileAdapter(request, input_data)
        except Exception as error:
            app_logger.error('Something is wrong: {0}'.format(error))
            raise
Пример #10
0
    def start_server(self):
        """Start the RPC Server.

        :return:
        """
        self._stopped.clear()
        if not self._connection or self._connection.is_closed:
            self._create_connection()
        while not self._stopped.is_set():
            try:
                # Check our connection for errors.
                self._connection.check_for_errors()
                self._update_consumers()
            except amqpstorm.AMQPError as why:
                # If an error occurs, re-connect and let update_consumers
                # re-open the channels.
                app_logger.error(why)
                self._stop_consumers()
                self._create_connection()
            time.sleep(1)
Пример #11
0
 def wrapper(message_data, *args, **kwargs):
     """Wrap it nicely."""
     aliases = message_data["payload"]["indexingServiceInput"]["targetAlias"]
     alias_list = [str(r) for r in aliases]
     # If there are multiple aliases in the message we need the first one for reference.
     # The reference alias will act as a unique identified for the index to be replaced.
     reference_alias = next(iter(alias_list or []), None)
     source_data = message_data["payload"]["indexingServiceInput"]["sourceData"]
     elastic = ElasticIndex()
     # We will be using only one index for all data sources.
     replace_index = elastic._index_create(reference_alias)
     try:
         for source in iter(source_data or []):
             data = retrieve_data(source["inputType"], source["input"])
             if source["useBulk"] and source["useBulk"] is True:
                 elastic._bulk_index(source["docType"], data, replace_index)
             elif source["documentID"]:
                 elastic._api_index(source["documentID"], source["docType"], data, replace_index)
             else:
                 raise KeyError("Missing useBulk operation or documentID for single document indexing.")
         return func(message_data, replace_index=replace_index, alias_list=alias_list)
     except Exception as error:
         app_logger.error('Something is wrong: {0}'.format(error))
         raise
Пример #12
0
    def _create_connection(self):
        """Create a connection.

        :return:
        """
        attempts = 0
        while True:
            attempts += 1
            if self._stopped.is_set():
                break
            try:
                self._connection = Connection(self.hostname, self.username,
                                              self.password)
                app_logger.info(
                    'Established connection with AMQP server {0}'.format(
                        self._connection))
                break
            except amqpstorm.AMQPError as why:
                app_logger.error(why)
                if self.max_retries and attempts > self.max_retries:
                    raise Exception('max number of retries reached')
                time.sleep(min(attempts * 2, 30))
            except KeyboardInterrupt:
                break