def execute(self, query, *args, **kwargs): connection = self.connect() try: # If connection is permanent is possible that have many changes of db db = format_value(kwargs, "db", default_value=format_value(self.attrib, "db", default_value="")) connection.select_db(db) result = None with closing(connection.cursor()) as cursor: if format_value(kwargs, "is_sp", default_value=False): cursor.callproc(query, args) else: cursor.execute(query, args) if format_value(kwargs, "details", default_value=False): result = {"affected_rows": connection.affected_rows(), "insert_id": connection.insert_id()} else: result = cursor.fetchall() # Log the warning information for item in connection.show_warnings(): # 1329L, 'No data - zero rows fetched, selected, or processed' if item[1] == 1329: log.debug(item[2]) else: log.warning("%s - %s" % (item[1], item[2])) return result except OperationalError, oe: # Range 2000~2999 used to client error codes self.is_necessary_reprocess = oe[0] > 1999 and oe[0] < 3000 raise oe
def run(self, process, action_value, *args, **kwargs): """ Will process the information passed in action_value. """ python_path = format_value(process, "path", default_value=self.python_path) if not exists(python_path): raise ValueError("The directory [%s] not exist." % self.python_path) path_exist = python_path in path try: if not path_exist: path.append(python_path) # Check the module module = format_value(process, "module", default_value=None) if module is None: raise ValueError("The module was not set in %s that use the connection [%s]" % (process["tag"], process["connection_name"])) if not (exists(join(python_path, module + ".py")) or exists(join(python_path, module + ".pyc"))): raise ValueError("The module [%s] not exist in the path [%s]" % (module, python_path)) class_name = format_value(process, "class", default_value=None) method = format_value(process, "method", default_value="run") module_ref = __import__(module, fromlist=None if class_name is None else [class_name, ]) instance = (None, 1, "Was not implemented yet!") if class_name: class_ref = getattr(module_ref, class_name)() instance = getattr(class_ref, method)(process, action_value, *args, **kwargs) else: instance = getattr(module_ref, method)(process, action_value, *args, **kwargs) return instance except Exception, e: return (None, settings.SYSTEM_ERROR, e)
def run(self, process, action_value, *args, **kwargs): """ Will process the information passed in action_value. """ field_position = format_value(process, "field_position", default_value=None) run_method = format_value( process, "run_method", default_value="query" if process["tag"] == "origin" else "update" ).lower() query = format_value(process, "text", default_value="") try: values = action_value if field_position is None else tuple([action_value[item] for item in field_position]) except: # Probably the action value don't have all the items return (None, 1, "Values not compatible with field_positions!") other_settings = {} db = format_value(process, "db_name", default_value=None) if db is not None: other_settings["db"] = db other_settings["details"] = run_method in ["update", "sp_update"] other_settings["run_method"] = run_method in ["sp", "sp_update"] try: result = self.execute(query, is_sp=run_method == "sp", *values, **other_settings) if run_method in ["update", "sp_update"]: result = (result["affected_rows"], result["insert_id"]) return (result, 0, "Success!") except OperationalError, oe: # Range 2000~2999 used to client error codes self.is_necessary_reprocess = oe[0] > 1999 and oe[0] < 3000 return ( None, settings.SYSTEM_ERROR if self.is_necessary_reprocess else settings.ACTION_ERROR, "Problem when executed the row [%s]!" % oe[1], )
def _action_parse(self, name, element): attrib = element["attrib"] # Validate the necessary variables attrib["tag"] = element["tag"] attrib["reprocess_time"] = format_value(attrib, "reprocess_time", 180) attrib["retry"] = format_value(attrib, "retry", 0) attrib["origin"] = None attrib["destination_list"] = [] # Will be use to set a default in case have multiple destination attrib["stop_on_error"] = True self[name] = attrib for item in element["list"]: self._process_internal(name, item) # Each destination already have the information set, so don't need the key anymore self[name].pop("stop_on_error")
def _replication_parse(self, name, element): attrib = element["attrib"] replicate_to = format_value(attrib, "to", None) if replicate_to is None: raise StandardError("The attribute [to] in the replicate [%s] need be declared" % name) replicate_to = [ item.strip() for item in replicate_to.split(",") if item.strip() != "" ] # Check if all the actions exist for action in replicate_to: if action not in self.keys(): raise ValueError("The action [%s] in the replicate [%s] not exist in the action list" % (action, name)) self[name] = { "tag": element["tag"], "replicate_to": replicate_to[:], "reprocess_time": format_value(attrib, "reprocess_time", 180), "retry": format_value(attrib, "reprocess_time", 0) }
def __init__(self, file_name, *args, **kwargs): dict.__init__(self, *args, **kwargs) if not exists(file_name): raise StandardError("The file that contain the action list [%s] was not created" % file_name) try: request = XMLParserToDict(file_name) # For the first tags for item in request: # Don't accept if don't have name of the action name = format_value(item["attrib"], "name", None) if name is None: continue if item["tag"] == "action": self._action_parse(name, item) # Check if have a less one destination if len(self[name]["destination_list"]) < 1: raise StandardError("The action [%s] need have a less one destination." % name) # Check if request to duplicate the action duplicate = format_value(item["attrib"], "duplicate", "") if len(duplicate) > 0: # Check the necessary quantity information that need be set # The number 1 means the action name qtd = 1 + (0 if self[name]["origin"] is None else 1) dest_qtd = len(self[name]["destination_list"]) # First split the list of actions action_list = duplicate.split(";") for action in action_list: if len(action) < 1: continue attr = action.split(",") if attr[0] == "": raise StandardError("The duplicate name in action [%s] is necessary declarate." % name) if len(attr) != qtd + dest_qtd: raise StandardError("The duplicate [%s] in action [%s] have incomplete information." % (attr[0], name,)) # Clone the dictionary clone = deepcopy(self[name]) clone.pop("duplicate") # Change the name clone["name"] = attr[0] if self[name]["origin"] is not None and attr[1] != "": clone["origin"]["connection_name"] = attr[1] for position in range(len(clone["destination_list"])): if attr[qtd + position] != "": clone["destination_list"][position]["connection_name"] = attr[qtd + position] self[clone["name"]] = clone # Remove the duplicate item["attrib"].pop("duplicate") if item["tag"] == "redirect": self._replication_parse(name, item) except Exception, e: raise StandardError("The file [%s] was malformed: [%s] " % (file_name, e))
def _process_internal(self, parent_name, element): tag = element["tag"] # In case of destinations just set to be a list if tag == "destinations": # In case the user set in destinations self[parent_name]["stop_on_error"] = format_value(element, "stop_on_error", True) # Call the list inside for item in element["list"]: self._process_internal(parent_name, item) return attrib = element["attrib"] # Adding in the attribute: tag and text to manipulate only one dictionary attrib["tag"] = format_value(element, "tag", None) attrib["text"] = format_value(element, "text", None) attrib["connection_name"] = format_value(attrib, "connection_name", None) if attrib["connection_name"] is None: raise ValueError("One of tag origin/destination in the action [%s] was set the run_method with query but without connection_name" % parent_name) # Using to origin/destination attrib["field_position"] = format_value(attrib, "field_position", None) if attrib["field_position"] is not None: attrib["field_position"] = [ int(item) for item in attrib["field_position"].split(",") ] if tag == "destination": attrib["stop_on_error"] = format_value(attrib, "stop_on_error", self[parent_name]["stop_on_error"]) # Add the information in the parent_name if tag == "destination": # Check if have multiple connections connections = attrib["connection_name"].split(",") destination_list = [] for connection in connections: destination = attrib.copy() destination["connection_name"] = connection.strip() destination_list.append(destination.copy()) self[parent_name]["destination_list"].extend(destination_list) else: self[parent_name][tag] = attrib.copy()
def run(self): """ Run the process and wait when don't have any """ PROCESS_ID, ACTION_NAME = range(2) try: sleep_time = 0 process = ProcessManager() # Execute while don't have a action to close this thread (join method) while not self.__stop_event.isSet(): try: have_process = self.have_request() if have_process is None: sleep_time = sleep_time + self.attrib["sleep_sum"] if self.attrib["sleep_limit"] > sleep_time else sleep_time self.__stop_event.wait(sleep_time) continue else: sleep_time = 0 for row in have_process: action_name = row[ACTION_NAME] action = format_value(settings.ACTION_DICT, action_name, default_value=None) if action is None: # Get the first column in the first row that need be the queue identify self.update_request(action_name, row[PROCESS_ID], settings.CONFIG_ERROR, "Action [%s] not exist!" % action_name, 0, 180) continue # Check the function call if action["tag"] == 'redirect': # For this event we need a transaction to guarantee the replication will make for all self.manager.connection.autocommit(False) # update_param will call the update_request and send in case have a error update_param = None try: for item in action["redirect_to"]: result = self.redirect_request(item, *have_process) if result != 1: raise ValueError("The replication expect that was one row affected, returned [%s]" % result) # If don' had a problem, so finish the request self.update_request(action_name, row[PROCESS_ID], settings.PROCESS_SUCCESS, "Success!", action["retry"], action["reprocess_time"]) self.manager.connection.commit() except OperationalError, oe: update_param = (action_name, row[PROCESS_ID], settings.ACTION_ERROR, "Action Error [%s]!" % oe, action["retry"], action["reprocess_time"]) self.manager.connection.rollback() except Exception, e: update_param = (action_name, row[PROCESS_ID], settings.SYSTEM_ERROR, "System Error [%s]!" % e, action["retry"], action["reprocess_time"]) self.manager.connection.rollback() finally: self.manager.connection.autocommit(True) if update_param is not None: self.update_request(*update_param)
self.manager.connection.commit() except OperationalError, oe: update_param = (action_name, row[PROCESS_ID], settings.ACTION_ERROR, "Action Error [%s]!" % oe, action["retry"], action["reprocess_time"]) self.manager.connection.rollback() except Exception, e: update_param = (action_name, row[PROCESS_ID], settings.SYSTEM_ERROR, "System Error [%s]!" % e, action["retry"], action["reprocess_time"]) self.manager.connection.rollback() finally: self.manager.connection.autocommit(True) if update_param is not None: self.update_request(*update_param) elif action["tag"] == 'action': main_message = "[%s/%s] - " % (self.name, action_name) log.info("%s Processing: %s" % (main_message, str(row)), verbose=1) # Check if have origin exist other else use the same information of have_request origin = format_value(action, "origin", default_value=None) if origin is None: # Return to a row because we threat like a result of query origin = (row,) else: origin, result_code, error_message = process.execute(origin, row) if result_code != 0: self.update_request(action_name, row[PROCESS_ID], result_code, "Origin - %s" % (error_message,), action["retry"], action["reprocess_time"]) continue log.info("%s Retrieve origin: %s" % (main_message, str(origin)), verbose=1) have_error = False # each row that return need run in destination for origin_row in origin: # In case a list of error, will store the error in case of the stop_on_error = False error_result = None for pos, destination_item in enumerate(format_value(action, "destination_list", default_value=[])):
# If don' had a problem, so finish the request self.update_request(action_name, row[PROCESS_ID], settings.PROCESS_SUCCESS, "Success!", action["retry"], action["reprocess_time"]) self.manager.connection.commit() except OperationalError, oe: update_param = (action_name, row[PROCESS_ID], settings.ACTION_ERROR, "Action Error [%s]!" % oe, action["retry"], action["reprocess_time"]) self.manager.connection.rollback() except Exception, e: update_param = (action_name, row[PROCESS_ID], settings.SYSTEM_ERROR, "System Error [%s]!" % e, action["retry"], action["reprocess_time"]) self.manager.connection.rollback() finally: self.manager.connection.autocommit(True) if update_param is not None: self.update_request(*update_param) elif action["tag"] == 'action': # Check if have origin exist other else use the same information of have_request origin = format_value(action, "origin", default_value=None) if origin is None: # Return to a row because we threat like a result of query origin = (row,) else: origin, result_code, error_message = process.execute(origin, row) if result_code != 0: self.update_request(action_name, row[PROCESS_ID], result_code, error_message, action["retry"], action["reprocess_time"]) continue log.info("Thread [%s], processing action [%s] - Value: [%s] - Origin Result [%s]" % (self.name, action_name, str(row), str(origin)), verbose=1) have_error = False # each row that return need run in destination for origin_row in origin: # In case a list of error, will store the error in case of the stop_on_error = False error_result = None for destination_item in format_value(action, "destination_list", default_value=[]):