def new_cmd(command, args_dict): cmd_template = avaliable_cmds[command] cmd = {'cmd':command} for k in cmd_template: cmd[k] = args_dict[k] if k in args_dict else cmd_template[k]['value'] if ('validation' in cmd_template[k] and cmd_template[k]['validation']): if (not cmd_template[k]['validation'](cmd[k])): raise Exception("%s: %s failed validation"%(k, cmd[k])) cmd['cmd_hash'] = hash_cmd(cmd) return cmd
def flush_cmd(bulk, data_type, template, redis_config): try: node_coordinator = NodeCoordinator(redis_config=redis_config) qsizes = node_coordinator.node_qsizes() logger.debug(qsizes) node_queues = {} for element in bulk: if data_type == "ids" and type(element) == int: user_id = element elif data_type =="users" and type(element) == dict and "id" in element: user_id = element['id'] t = copy.copy(template) t["user_id"] = int(user_id) t["depth"] = int(t["depth"]) -1 node_id = get_keys_by_min_value(qsizes)[0] if (node_id in node_queues): node_queue = node_queues[node_id] else: node_queue = NodeQueue(node_id, redis_config=redis_config) node_queues[node_id] = node_queue t['cmd_hash'] = hash_cmd(t) node_queue.put(t) qsizes[node_id] += 1 logger.debug("send [%s] to node: %s"%(json.dumps(t),node_id)) # intend to close all redis connections, but not sure yet... node_queues.clear() del node_coordinator except Exception as exc: logger.error('error during flush: %s'%exc) return True
def flush_cmd(bulk, data_type, template, redis_config): try: node_coordinator = NodeCoordinator(redis_config=redis_config) qsizes = node_coordinator.node_qsizes() logger.debug(qsizes) node_queues = {} for element in bulk: if data_type == "ids" and type(element) == int: user_id = element elif data_type == "users" and type( element) == dict and "id" in element: user_id = element['id'] t = copy.copy(template) t["user_id"] = int(user_id) t["depth"] = int(t["depth"]) - 1 node_id = get_keys_by_min_value(qsizes)[0] if (node_id in node_queues): node_queue = node_queues[node_id] else: node_queue = NodeQueue(node_id, redis_config=redis_config) node_queues[node_id] = node_queue t['cmd_hash'] = hash_cmd(t) node_queue.put(t) qsizes[node_id] += 1 logger.debug("send [%s] to node: %s" % (json.dumps(t), node_id)) # intend to close all redis connections, but not sure yet... node_queues.clear() del node_coordinator except Exception as exc: logger.error('error during flush: %s' % exc) return True