Example #1
0
 def execute(self, query, *args, **kwargs):
     connection = self.connect()
     try:
         # If connection is permanent is possible that have many changes of db
         db = format_value(kwargs, "db", default_value=format_value(self.attrib, "db", default_value=""))
         connection.select_db(db)
         result = None
         with closing(connection.cursor()) as cursor:
             if format_value(kwargs, "is_sp", default_value=False):
                 cursor.callproc(query, args)
             else:
                 cursor.execute(query, args)
             if format_value(kwargs, "details", default_value=False):
                 result = {"affected_rows": connection.affected_rows(), "insert_id": connection.insert_id()}
             else:
                 result = cursor.fetchall()
         # Log the warning information
         for item in connection.show_warnings():
             # 1329L, 'No data - zero rows fetched, selected, or processed'
             if item[1] == 1329:
                 log.debug(item[2])
             else:
                 log.warning("%s - %s" % (item[1], item[2]))
         return result
     except OperationalError, oe:
         # Range 2000~2999 used to client error codes
         self.is_necessary_reprocess = oe[0] > 1999 and oe[0] < 3000
         raise oe
Example #2
0
    def run(self, process, action_value, *args, **kwargs):
        """
        Will process the information passed in action_value.
        """
        python_path = format_value(process, "path", default_value=self.python_path)
        if not exists(python_path):
            raise ValueError("The directory [%s] not exist." % self.python_path)
        path_exist = python_path in path
        try:
            if not path_exist: path.append(python_path)

            # Check the module
            module = format_value(process, "module", default_value=None)
            if module is None: raise ValueError("The module was not set in %s that use the connection [%s]" % (process["tag"], process["connection_name"]))
            if not (exists(join(python_path, module + ".py")) or exists(join(python_path, module + ".pyc"))):
                raise ValueError("The module [%s] not exist in the path [%s]" % (module, python_path))
            class_name = format_value(process, "class", default_value=None)
            method = format_value(process, "method", default_value="run")
            module_ref = __import__(module, fromlist=None if class_name is None else [class_name, ])
            instance = (None, 1, "Was not implemented yet!")
            if class_name:
                class_ref = getattr(module_ref, class_name)()
                instance = getattr(class_ref, method)(process, action_value, *args, **kwargs)
            else:
                instance = getattr(module_ref, method)(process, action_value, *args, **kwargs)
            return instance
        except Exception, e:
            return (None, settings.SYSTEM_ERROR, e)
Example #3
0
 def run(self, process, action_value, *args, **kwargs):
     """
     Will process the information passed in action_value.
     """
     field_position = format_value(process, "field_position", default_value=None)
     run_method = format_value(
         process, "run_method", default_value="query" if process["tag"] == "origin" else "update"
     ).lower()
     query = format_value(process, "text", default_value="")
     try:
         values = action_value if field_position is None else tuple([action_value[item] for item in field_position])
     except:
         # Probably the action value don't have all the items
         return (None, 1, "Values not compatible with field_positions!")
     other_settings = {}
     db = format_value(process, "db_name", default_value=None)
     if db is not None:
         other_settings["db"] = db
     other_settings["details"] = run_method in ["update", "sp_update"]
     other_settings["run_method"] = run_method in ["sp", "sp_update"]
     try:
         result = self.execute(query, is_sp=run_method == "sp", *values, **other_settings)
         if run_method in ["update", "sp_update"]:
             result = (result["affected_rows"], result["insert_id"])
         return (result, 0, "Success!")
     except OperationalError, oe:
         # Range 2000~2999 used to client error codes
         self.is_necessary_reprocess = oe[0] > 1999 and oe[0] < 3000
         return (
             None,
             settings.SYSTEM_ERROR if self.is_necessary_reprocess else settings.ACTION_ERROR,
             "Problem when executed the row [%s]!" % oe[1],
         )
Example #4
0
 def _action_parse(self, name, element):
     attrib = element["attrib"]
     # Validate the necessary variables
     attrib["tag"] = element["tag"]
     attrib["reprocess_time"] = format_value(attrib, "reprocess_time", 180)
     attrib["retry"] = format_value(attrib, "retry", 0)
     attrib["origin"] = None
     attrib["destination_list"] = []
     # Will be use to set a default in case have multiple destination
     attrib["stop_on_error"] = True
     self[name] = attrib
     for item in element["list"]:
         self._process_internal(name, item)
     # Each destination already have the information set, so don't need the key anymore
     self[name].pop("stop_on_error")
Example #5
0
 def _replication_parse(self, name, element):
     attrib = element["attrib"]
     replicate_to = format_value(attrib, "to", None)
     if replicate_to is None: raise StandardError("The attribute [to] in the replicate [%s] need be declared" % name)
     replicate_to = [ item.strip() for item in replicate_to.split(",") if item.strip() != "" ]
     # Check if all the actions exist
     for action in replicate_to:
         if action not in self.keys():
             raise ValueError("The action [%s] in the replicate [%s] not exist in the action list" % (action, name))
     self[name] = {
                   "tag": element["tag"],
                   "replicate_to": replicate_to[:],
                   "reprocess_time": format_value(attrib, "reprocess_time", 180),
                   "retry": format_value(attrib, "reprocess_time", 0)
                   }
Example #6
0
 def __init__(self, file_name, *args, **kwargs):
     dict.__init__(self, *args, **kwargs)
     if not exists(file_name):
         raise StandardError("The file that contain the action list [%s] was not created" % file_name)
     try:
         request = XMLParserToDict(file_name)
         # For the first tags
         for item in request:
             # Don't accept if don't have name of the action
             name = format_value(item["attrib"], "name", None)
             if name is None: continue
             if item["tag"] == "action":
                 self._action_parse(name, item)
                 # Check if have a less one destination
                 if len(self[name]["destination_list"]) < 1:
                     raise StandardError("The action [%s] need have a less one destination." % name)
                 # Check if request to duplicate the action
                 duplicate = format_value(item["attrib"], "duplicate", "")
                 if len(duplicate) > 0:
                     # Check the necessary quantity information that need be set
                     # The number 1 means the action name
                     qtd = 1 + (0 if self[name]["origin"] is None else 1)
                     dest_qtd = len(self[name]["destination_list"])
                     # First split the list of actions
                     action_list = duplicate.split(";")
                     for action in action_list:
                         if len(action) < 1: continue
                         attr = action.split(",")
                         if attr[0] == "": raise StandardError("The duplicate name in action [%s] is necessary declarate." % name)
                         if len(attr) != qtd + dest_qtd:
                             raise StandardError("The duplicate [%s] in action [%s] have incomplete information." % (attr[0], name,))
                         # Clone the dictionary
                         clone = deepcopy(self[name])
                         clone.pop("duplicate")
                         # Change the name
                         clone["name"] = attr[0]
                         if self[name]["origin"] is not None and attr[1] != "":
                             clone["origin"]["connection_name"] = attr[1]
                         for position in range(len(clone["destination_list"])):
                             if attr[qtd + position] != "":
                                 clone["destination_list"][position]["connection_name"] = attr[qtd + position]
                         self[clone["name"]] = clone
                     # Remove the duplicate
                     item["attrib"].pop("duplicate")
             if item["tag"] == "redirect":
                 self._replication_parse(name, item)
     except Exception, e:
         raise StandardError("The file [%s] was malformed: [%s] " % (file_name, e))
Example #7
0
 def _process_internal(self, parent_name, element):
     tag = element["tag"]
     # In case of destinations just set to be a list
     if tag == "destinations":
         # In case the user set in destinations
         self[parent_name]["stop_on_error"] = format_value(element, "stop_on_error", True)
         # Call the list inside
         for item in element["list"]:
             self._process_internal(parent_name, item)
         return
     attrib = element["attrib"]
     # Adding in the attribute: tag and text to manipulate only one dictionary
     attrib["tag"] = format_value(element, "tag", None)
     attrib["text"] = format_value(element, "text", None)
     attrib["connection_name"] = format_value(attrib, "connection_name", None)
     if attrib["connection_name"] is None:
         raise ValueError("One of tag origin/destination in the action [%s] was set the run_method with query but without connection_name" % parent_name)
     # Using to origin/destination
     attrib["field_position"] = format_value(attrib, "field_position", None)
     if attrib["field_position"] is not None:
         attrib["field_position"] = [ int(item) for item in attrib["field_position"].split(",") ]
     if tag == "destination":
         attrib["stop_on_error"] = format_value(attrib, "stop_on_error", self[parent_name]["stop_on_error"])
     # Add the information in the parent_name
     if tag == "destination":
         # Check if have multiple connections
         connections = attrib["connection_name"].split(",")
         destination_list = []
         for connection in connections:
             destination = attrib.copy()
             destination["connection_name"] = connection.strip()
             destination_list.append(destination.copy())
         self[parent_name]["destination_list"].extend(destination_list)
     else:
         self[parent_name][tag] = attrib.copy()
Example #8
0
 def run(self):
     """
     Run the process and wait when don't have any
     """
     PROCESS_ID, ACTION_NAME = range(2)
     try:
         sleep_time = 0
         process = ProcessManager()
         # Execute while don't have a action to close this thread (join method)
         while not self.__stop_event.isSet():
             try:
                 have_process = self.have_request()
                 if have_process is None:
                     sleep_time = sleep_time + self.attrib["sleep_sum"] if self.attrib["sleep_limit"] > sleep_time else sleep_time
                     self.__stop_event.wait(sleep_time)
                     continue
                 else:
                     sleep_time = 0
                 for row in have_process:
                     action_name = row[ACTION_NAME]
                     action = format_value(settings.ACTION_DICT, action_name, default_value=None)
                     if action is None:
                         # Get the first column in the first row that need be the queue identify
                         self.update_request(action_name, row[PROCESS_ID], settings.CONFIG_ERROR, "Action [%s] not exist!" % action_name, 0, 180)
                         continue
                     # Check the function call
                     if action["tag"] == 'redirect':
                         # For this event we need a transaction to guarantee the replication will make for all
                         self.manager.connection.autocommit(False)
                         # update_param will call the update_request and send in case have a error
                         update_param = None
                         try:
                             for item in action["redirect_to"]:
                                 result = self.redirect_request(item, *have_process)
                                 if result != 1:
                                     raise ValueError("The replication expect that was one row affected, returned [%s]" % result)
                             # If don' had a problem, so finish the request
                             self.update_request(action_name, row[PROCESS_ID], settings.PROCESS_SUCCESS, "Success!", action["retry"], action["reprocess_time"])
                             self.manager.connection.commit()
                         except OperationalError, oe:
                             update_param = (action_name, row[PROCESS_ID], settings.ACTION_ERROR, "Action Error [%s]!" % oe, action["retry"], action["reprocess_time"])
                             self.manager.connection.rollback()
                         except Exception, e:
                             update_param = (action_name, row[PROCESS_ID], settings.SYSTEM_ERROR, "System Error [%s]!" % e, action["retry"], action["reprocess_time"])
                             self.manager.connection.rollback()
                         finally:
                             self.manager.connection.autocommit(True)
                         if update_param is not None:
                             self.update_request(*update_param)
Example #9
0
         self.manager.connection.commit()
     except OperationalError, oe:
         update_param = (action_name, row[PROCESS_ID], settings.ACTION_ERROR, "Action Error [%s]!" % oe, action["retry"], action["reprocess_time"])
         self.manager.connection.rollback()
     except Exception, e:
         update_param = (action_name, row[PROCESS_ID], settings.SYSTEM_ERROR, "System Error [%s]!" % e, action["retry"], action["reprocess_time"])
         self.manager.connection.rollback()
     finally:
         self.manager.connection.autocommit(True)
     if update_param is not None:
         self.update_request(*update_param)
 elif action["tag"] == 'action':
     main_message = "[%s/%s] - " % (self.name, action_name)
     log.info("%s Processing: %s" % (main_message, str(row)), verbose=1)
     # Check if have origin exist other else use the same information of have_request
     origin = format_value(action, "origin", default_value=None)
     if origin is None:
         # Return to a row because we threat like a result of query
         origin = (row,)
     else:
         origin, result_code, error_message = process.execute(origin, row)
         if result_code != 0:
             self.update_request(action_name, row[PROCESS_ID], result_code, "Origin - %s" % (error_message,), action["retry"], action["reprocess_time"])
             continue
         log.info("%s Retrieve origin: %s" % (main_message, str(origin)), verbose=1)
     have_error = False
     # each row that return need run in destination
     for origin_row in origin:
         # In case a list of error, will store the error in case of the stop_on_error = False
         error_result = None
         for pos, destination_item in enumerate(format_value(action, "destination_list", default_value=[])):
Example #10
0
         # If don' had a problem, so finish the request
         self.update_request(action_name, row[PROCESS_ID], settings.PROCESS_SUCCESS, "Success!", action["retry"], action["reprocess_time"])
         self.manager.connection.commit()
     except OperationalError, oe:
         update_param = (action_name, row[PROCESS_ID], settings.ACTION_ERROR, "Action Error [%s]!" % oe, action["retry"], action["reprocess_time"])
         self.manager.connection.rollback()
     except Exception, e:
         update_param = (action_name, row[PROCESS_ID], settings.SYSTEM_ERROR, "System Error [%s]!" % e, action["retry"], action["reprocess_time"])
         self.manager.connection.rollback()
     finally:
         self.manager.connection.autocommit(True)
     if update_param is not None:
         self.update_request(*update_param)
 elif action["tag"] == 'action':
     # Check if have origin exist other else use the same information of have_request
     origin = format_value(action, "origin", default_value=None)
     if origin is None:
         # Return to a row because we threat like a result of query
         origin = (row,)
     else:
         origin, result_code, error_message = process.execute(origin, row)
         if result_code != 0:
             self.update_request(action_name, row[PROCESS_ID], result_code, error_message, action["retry"], action["reprocess_time"])
             continue
     log.info("Thread [%s], processing action [%s] - Value: [%s] - Origin Result [%s]" % (self.name, action_name, str(row), str(origin)), verbose=1)
     have_error = False
     # each row that return need run in destination
     for origin_row in origin:
         # In case a list of error, will store the error in case of the stop_on_error = False
         error_result = None
         for destination_item in format_value(action, "destination_list", default_value=[]):