def select_task_impala_load_not_ready(self): try: con = MySQLdb.connect(host=self.mysqlinfo["host"], port=string.atoi(self.mysqlinfo["port"]), user=self.mysqlinfo["user"], passwd=self.mysqlinfo["passwd"], db=self.mysqlinfo["dbname"]) cursor = con.cursor() task_str = self.select_task_impala_taskname_list() sql = "select task_name,node_num,batch_id,h_db_name,h_table_name,h_table_part,i_db_name,i_table_name,i_table_part,insert_type,count(1) as num \ from task_impala_load where status!='SUCCESS' \ and task_name in {task_str}\ group by task_name,node_num,batch_id,h_db_name,h_table_name,h_table_part,i_db_name,i_table_name,i_table_part,insert_type having num=node_num;" sql = sql.replace("{task_str}", task_str) print( "select_task_impala_load_not_ready functions run! select sql => " + sql) cursor.execute(sql) results = cursor.fetchall() if (len(results) != 0): print( "select_task_impala_load_not_ready sql result : \n r(0): " + str(results[0])) else: print("result is null! ") return results except Error, e: Logger.error("mysql error select_task_impala_load_ready" + e.args[0].__str__() + ":" + e.args[1].__str__()) print "mysql error select_task_impala_load_ready%d: %s" % ( e.args[0], e.args[1])
def select_task_impala_taskname_list(self): try: con = MySQLdb.connect(host=self.mysqlinfo["host"], port=string.atoi(self.mysqlinfo["port"]), user=self.mysqlinfo["user"], passwd=self.mysqlinfo["passwd"], db=self.mysqlinfo["dbname"]) cursor = con.cursor() sql = "select task_name from task_impala_sql" cursor.execute(sql) taskname_list = cursor.fetchall() task_str = "(" for i in range(0, len(taskname_list)): for j in range(0, len(taskname_list[i])): if (i == (len(taskname_list) - 1)): task_str = task_str + "'" + str( taskname_list[i][j]) + "'" else: task_str = str(task_str + "'" + taskname_list[i][j]) + "'" + "," task_str = task_str + ")" return task_str except Error, e: Logger.error("select_task_impala_taskname_list" + e.args[0].__str__() + ":" + e.args[1].__str__()) print "mysql error select_task_impala_taskname_list%d: %s" % ( e.args[0], e.args[1])
def getConfigFileName(taskName): configRoot = getConfigRoot() if (configRoot == errorCode): Logger.error("failed to get the root dir of config file " + taskName) return errorCode configFileName = configRoot + str(taskName) + ".cfg" return configFileName
def save(relation): if ArtistRelationFactory.__use_db: artistA = relation.artistA.key() artistB = relation.artistB.key() if artistA > artistB: artistC = artistB artistB = artistA artistA = artistC ArtistRelationFactory.__db.execute( u"REPLACE INTO artist_relation "\ "(artistA, artistB, relation) VALUES (?, ?, ?)", ( artistA, artistB, Binary(pickle.dumps(relation, -1)))) ArtistRelationFactory.__db.commit() elif TrackRelationFactory.__use_fs: pathA = join(ArtistRelationFactory.__path, relation.artistA.key()) fileA = u"%s.pkl" % join(pathA, relation.artistB.key()) pathB = join(ArtistRelationFactory.__path, relation.artistB.key()) fileB = u"%s.pkl" % join(pathB, relation.artistA.key()) if not isdir(pathA): makedirs(pathA) if not isdir(pathB): makedirs(pathB) if fileA > fileB: fileC = fileB fileB = fileA fileA = fileC ArtistRelationFactory._save(relation, fileA) if fileA != fileB: if exists(fileB): remove(fileB) try: link(fileA, fileB) except OSError: Logger.error(u"Failed to link %s to %s" % (fileA, fileB)) relation.lastmodified = getmtime(fileA)
def getVoucherUrl(driver, waitJobid, ticketPlatform, ticketNum): # Load logger voucherlogger = Logger(ticketPlatform, waitJobid, botid='1', method='wait_voucher', logger='waitVoucher').getlog() voucherlogger.info("wait_vocher jobid=" + waitJobid) voucherlogger.info("order ticketplatform=" + ticketPlatform) voucherlogger.info("wait_vocher ticketNumber=" + ticketNum) singleorderurl = 'http://www.travelmart.com.cn/singleOrder.asp?id=' try: driver.get(singleorderurl + ticketNum) except: voucherlogger.error("Don't open singleorderurl") return False try: voucherurl = driver.find_element_by_link_text('查看打印凭证').get_attribute( 'href') except: voucherlogger.error("[Voucherurl] don't find element") return False return voucherurl
def getSqoopJobDict(): jobDict = {} Logger = log.Logger conf = ConfigParser.RawConfigParser() #获取sqoop配置文件名称 sqoopConfFileName = getConfigFileName("sqoop") try: conf.read(sqoopConfFileName) except: Logger.error("read sqoop configuration file error " + sqoopConfFileName) return errorCode jobNames_str = conf.get("jobsName", "jobs") if (jobNames_str is None): Logger.info("there is no job in the configuration!") return errorCode jobNames = jobNames_str.split(",") for jobName in jobNames: sqoopcmd = getEnv("sqoop") + " " for (key, value) in conf.items(jobName): #如果数据来自sql语句查询的结果,那么sql语句放在子sql里面 防止sql中包含“=”号 # if(str(key) == "-e" or str(key) == "--query"): # value = getSqoopSubSql(value) sqoopcmd = sqoopcmd + " " + key + " " + value jobDict[jobName] = sqoopcmd return jobDict
class Audit(object): """ A class for handling audit information """ def __init__(self, db, enabled=True): self.db = db self.enabled = enabled self.log = Logger(system=self) self.action_ids = { "GET_PROFILE": 0, "GET_CONTRACT": 1, "GET_LISTINGS": 2, # Click Store tab "GET_FOLLOWING": 3, "GET_FOLLOWERS": 4, "GET_RATINGS": 5 } def record(self, guid, action_id, contract_hash=None): if self.enabled is not True: return self.log.info("Recording Audit Event [%s]" % action_id) if action_id in self.action_ids: self.db.audit_shopping.set(guid, self.action_ids[action_id], contract_hash) else: self.log.error("Could not identify this action id")
class BaiduMap: def __init__(self): self._logger = Logger(__file__) self.status ={0:"正常", 1:"服务器内部错误", 2:"请求参数非法", 3:"权限校验失败", 4:"配额校验失败", 5:"ak不存在或者非法", 101:"服务禁用", 102:"不通过白名单或者安全码不对", 200:"无权限", 300:"配额错误"} self.geoprefix = "http://api.map.baidu.com/geocoder/v2/?address=" self.revprefix = "http://api.map.baidu.com/geocoder/v2/?location=" self.suffix = "&output=json&ak=" self.headers = {} self.headers["User-Agent"]="Mozilla/5.0 (X11; Linux x86_64; rv:45.0) Gecko/20100101 Firefox/45.0" def access(self,url): """get Json object from specified url """ try: req = urllib2.Request(url,headers=self.headers) response = urllib2.urlopen(req) return json.loads(response.read()) except Exception,e: self._logger.error("error occured when get geo data") return None
class Crawler(threading.Thread): def __init__(self, cookie, manager): threading.Thread.__init__(self) cookie_handler = urllib2.HTTPCookieProcessor(cookie) self._opener = urllib2.build_opener(cookie_handler) self._manager = manager self._logger = Logger(u"crawler.log") def run(self): while True: try: # fetch url url = self._manager.get_url() self._logger.debug("Fetch a url " + url) # fetch page request = urllib2.Request(url) page = self._opener.open(request) self._logger.debug("Fetch the page for url " + url) # insert page while True: try: self._manager.insert_page(url, page) self._logger.debug("Insert the page") break except PageQueueFullForNowError: time.sleep(0.1) except urllib2.URLError: self._manager.insert_page(url, None) self._logger.error("URLError for " + url) except UrlQueueEmptyForNowError: time.sleep(0.1) except NoUrlToCrawlError: break
def getSqoopPoolNumber(): poolNumber = getValueByKey("sqoop", "sqoopTask", "threadPoolNumber") if (poolNumber != errorCode): return poolNumber else: Logger.error("failed to get sqoop pool number!") return errorCode
def load(input): for i in range(len(Factories.__lib)): Logger.debug(u"Factories.load(): loading %s" % i) k = pickle.load(input) if k != i: # FIXME: we should die here Logger.error(u"Factories.load(): key mismatch %d vs %d" % (i, k)) Factories.__lib[k].setstate(pickle.load(input))
class Server(object): def __init__(self, kserver, signing_key, database): """ A high level class for sending direct, market messages to other nodes. A node will need one of these to participate in buying and selling. Should be initialized after the Kademlia server. """ self.kserver = kserver self.signing_key = signing_key self.router = kserver.protocol.router self.db = database self.log = Logger(system=self) self.protocol = MarketProtocol(kserver.node, self.router, signing_key, database) # TODO: we need a loop here that republishes keywords when they are about to expire # TODO: we also need a loop here to delete expiring contract (if they are set to expire) def querySeed(self, list_seed_pubkey): """ Query an HTTP seed for known vendors and save the vendors to the db. Args: Receives a list of one or more tuples Example [(seed, pubkey)] seed: A `string` consisting of "ip:port" or "hostname:port" pubkey: The hex encoded public key to verify the signature on the response """ for sp in list_seed_pubkey: seed, pubkey = sp try: self.log.debug("querying %s for vendors" % seed) c = httplib.HTTPConnection(seed) c.request("GET", "/?type=vendors") response = c.getresponse() self.log.debug("Http response from %s: %s, %s" % (seed, response.status, response.reason)) data = response.read() reread_data = data.decode("zlib") proto = peers.PeerSeeds() proto.ParseFromString(reread_data) verify_key = nacl.signing.VerifyKey( pubkey, encoder=nacl.encoding.HexEncoder) verify_key.verify("".join(proto.serializedNode), proto.signature) v = self.db.VendorStore() for peer in proto.serializedNode: try: n = objects.Node() n.ParseFromString(peer) v.save_vendor(n.guid.encode("hex"), peer) except Exception: pass except Exception, e: self.log.error("failed to query seed: %s" % str(e))
class SMTPNotification(object): """ A class for sending SMTP notifications """ def __init__(self, db): self.db = db self.server = 'localhost:25' self.sender = 'OpenBazaar' self.recipient = '' self.username = None self.password = None self.log = Logger(system=self) self.get_smtp_settings() def get_smtp_settings(self): settings = self.db.settings.get() self.server = settings[15] self.sender = settings[16] self.recipient = settings[17] self.username = settings[18] self.password = settings[19] def send(self, subject, body): settings = self.db.settings.get() is_enabled = True if settings[14] == 1 else False if is_enabled: # Construct MIME message msg = MIMEMultipart('alternative') msg['Subject'] = subject msg['From'] = self.sender msg['To'] = self.recipient html_body = MIMEText(body, 'html') msg.attach(html_body) try: server = smtplib.SMTP(self.server) server.starttls() # Authenticate if username/password set if self.username and self.password: server.login(self.username, self.password) server.sendmail(self.sender, self.recipient, msg.as_string()) except SMTPAuthenticationError as e: self.log.error(e) print e server.quit()
class Server(object): def __init__(self, kserver, signing_key, database): """ A high level class for sending direct, market messages to other nodes. A node will need one of these to participate in buying and selling. Should be initialized after the Kademlia server. """ self.kserver = kserver self.signing_key = signing_key self.router = kserver.protocol.router self.db = database self.log = Logger(system=self) self.protocol = MarketProtocol(kserver.node, self.router, signing_key, database) # TODO: we need a loop here that republishes keywords when they are about to expire # TODO: we also need a loop here to delete expiring contract (if they are set to expire) def querySeed(self, list_seed_pubkey): """ Query an HTTP seed for known vendors and save the vendors to the db. Args: Receives a list of one or more tuples Example [(seed, pubkey)] seed: A `string` consisting of "ip:port" or "hostname:port" pubkey: The hex encoded public key to verify the signature on the response """ for sp in list_seed_pubkey: seed, pubkey = sp try: self.log.debug("querying %s for vendors" % seed) c = httplib.HTTPConnection(seed) c.request("GET", "/?type=vendors") response = c.getresponse() self.log.debug("Http response from %s: %s, %s" % (seed, response.status, response.reason)) data = response.read() reread_data = data.decode("zlib") proto = peers.PeerSeeds() proto.ParseFromString(reread_data) verify_key = nacl.signing.VerifyKey(pubkey, encoder=nacl.encoding.HexEncoder) verify_key.verify("".join(proto.serializedNode), proto.signature) v = self.db.VendorStore() for peer in proto.serializedNode: try: n = objects.Node() n.ParseFromString(peer) v.save_vendor(n.guid.encode("hex"), peer) except Exception: pass except Exception, e: self.log.error("failed to query seed: %s" % str(e))
def getEnv(task): conf = ConfigParser.RawConfigParser() #通过任务类型获取对应的配置文件 ConfFileName = getConfigFileName(task) try: conf.read(ConfFileName) except: #email() Logger.error("failed to read sqoop config file!" + ConfFileName) return errorCode #获取该命令所在机器的环境变量 cmd = conf.get(task + "Env", "cmdPath") return cmd
def getSqoopJobListByName(jobName): conf = ConfigParser.RawConfigParser() sqoopConfFileName = getConfigFileName("sqoop") sqoopCmd = getEnv("sqoop") try: conf.read(sqoopConfFileName) except: #email() Logger.error("failed to read config file " + sqoopConfFileName) return errorCode for (key, value) in conf.items(jobName): sqoopCmd = sqoopCmd + " " + key + " " + value return sqoopCmd
def send_message(contact, message): try: contact.get_chat().send_message(message) except Exception as e: if (is_time_out_error(e)): print(f"Error trying to send message because time out - {e}") internet.wait_until_connection_becames_available() Logger.error( f"Error trying to send message because time out - {e}") else: print(f"unknown error {e}") Logger.error(f"Error trying to send message - {e}") pass time.sleep(2)
def getValueByKey(taskName, section, key): conf = ConfigParser.RawConfigParser() configFile = getConfigFileName(taskName) if (configFile == errorCode): Logger.error( "Failed to get configFile from function getConfigFileName(" + taskName + ")") return errorCode try: conf.read(configFile) except: Logger.error("read config file failed ! logfile dir is " + configFile) return errorCode value = conf.get(section, key) return value
def insert_task_log(self, values): try: con = MySQLdb.connect(host=self.mysqlinfo["host"], port=string.atoi(self.mysqlinfo["port"]), user=self.mysqlinfo["user"], passwd=self.mysqlinfo["passwd"], db=self.mysqlinfo["dbname"]) cursor = con.cursor() sql = "insert into task_log(task_id,task_type,start_time,end_time,cost,ifnormal,run_date) values(%s,%s,%s,%s,%s,%s,%s)" cursor.execute(sql, values) con.commit() except Error, e: Logger.error("mysql error insert_task_log" + e.args[0].__str__() + ":" + e.args[1].__str__()) print "mysql error insert_task_log %d: %s" % (e.args[0], e.args[1])
def getKeyValueBySection(taskName, section): conf = ConfigParser.RawConfigParser() configFile = getConfigFileName(taskName) if (configFile == errorCode): Logger.error( "Failed to get configFile from function getConfigFileName(" + taskName + ")") return errorCode try: conf.read(configFile) except: Logger.error("read config file failed ! logfile dir is " + configFile) return errorCode dict_kv = conf.items(section) return dict_kv
class DictionaryReader(): word_list = None logger = None filename = None def __init__(self): #TODO: implement config self.logger = Logger() # TODO: refractor try: filepath = os.path.expanduser(config.get('dictionary location')) self.read_file(filepath) except FileNotFoundError as err: self.logger.error(err) # TODO: refractor def reload_dictionary(self): self.logger.info("Reloading dictionary") self.read_file(self.filename) def read_file(self, filename): self.filename = filename try: with open(filename, 'r') as f: content = f.read() splitted = content.split('\n') only_unique = (list(set(splitted))) sorted_data = sorted(only_unique) self.word_list = sorted_data except FileNotFoundError as err: self.logger.error(err) #TODO: refractor try: with open(filename, 'w') as f: f.write('Sample\ndata\nin the\ndictionary') except IOError as err: self.logger.error(err) except PermissionError as err: self.logger.error(err) #TODO: refractor except OSError as err: self.logger.error(err) #TODO: refractor def get_random_word(self): if len(self.word_list) > 0: random_entry = random.randrange(0, len(self.word_list), 1) word = self.word_list[random_entry] if config.get('remove words from pool'): self.word_list.remove(word) return word else: self.logger.info('Dictionary emptied') return 'Greetings! You emptied the pool!'
def send_media(driver, path, chat_id, caption, contact): try: driver.send_media(path, chat_id, caption) except Exception as e: print(f"Error trying to send media -") if (is_time_out_error(e)): print(f"Error trying to send media because time out - {e}") Logger.error(f"Error trying to send media because time out - {e}") internet.wait_until_connection_becames_available() else: print(f"unknown error {e}") Logger.error(f"Error trying to send media - {e}") pass send_message( contact, f"Não foi possível enviar a media do {contact.get_safe_name()}")
def select_task_sceduling(self, run_date): try: con = MySQLdb.connect(host=self.mysqlinfo["host"], port=string.atoi(self.mysqlinfo["port"]), user=self.mysqlinfo["user"], passwd=self.mysqlinfo["passwd"], db=self.mysqlinfo["dbname"]) cursor = con.cursor() sql = "select task_id,runstatus,ifnormal from task_sceduling where date(run_date)='" + run_date + "'" cursor.execute(sql) results = cursor.fetchall() return results except Error, e: Logger.error("mysql error select_task_sceduling" + e.args[0].__str__() + ":" + e.args[1].__str__()) print "mysql error select_task_sceduling%d: %s" % (e.args[0], e.args[1])
class IWatchdog(threading.Thread): """ interface class needed to instanciate a watchdog class """ def __init__(self): """ constructor """ threading.Thread.__init__(self) self.logger = Logger("IWatchdog") def run(self): """ redefinition of run method of threading.Thread class :return: None """ self.logger.error("undefined watchdog launcher")
class MySQL: # the mysql error code error_code = '' # the instance of this class _instance = None # the database connection object _conn = None # the cursor _cur = None # the default timeout _TIMEOUT = 30 _timecount = 0 def __init__(self, dbconfig): """create database connection with specified arguments # Parameters: dbconfig: the configuration argument: host: user: passwd: port: charset: # Return: """ self._logger = Logger(__file__) try: self._conn = MySQLdb.connect(host=dbconfig['host'], port=dbconfig['port'], user=dbconfig['user'], passwd=dbconfig['passwd'], db=dbconfig['db'], charset=dbconfig['charset']) except MySQLdb.Error, e: self.error_code = str(e.args[0]) error_msg = 'MySQL error! ', str(e.args[0]), e.args[1] print error_msg # if not exceed the default timeout, try again if self._timecount < self._TIMEOUT: interval = 5 self._timecount += interval time.sleep(interval) self.__init__(dbconfig) else: self._logger.error(str(e.args[0]) + e.args[1]) raise Exception(error_msg) self._cur = self._conn.cursor()
class BaiduMap: def __init__(self): self._logger = Logger(__file__) self.headers = {} self.headers[ "User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64; rv:45.0) Gecko/20100101 Firefox/45.0" def access(self, url): """get Json object from specified url """ try: req = urllib2.Request(url, headers=self.headers) response = urllib2.urlopen(req) return json.loads(response.read()) except Exception, e: self._logger.error("error occured when get geo data") return None
class MessageListenerImpl(object): implements(MessageListener) def __init__(self, web_socket_factory, database): self.ws = web_socket_factory self.db = database self.log = Logger(system=self) def notify(self, plaintext, signature): try: success = self.db.messages.save_message( plaintext.sender_guid.encode("hex"), plaintext.handle, plaintext.pubkey, plaintext.subject, PlaintextMessage.Type.Name(plaintext.type), plaintext.message, plaintext.timestamp, plaintext.avatar_hash, signature, False, ) if plaintext.subject != "": self.db.purchases.update_unread(plaintext.subject) self.db.sales.update_unread(plaintext.subject) self.db.cases.update_unread(plaintext.subject) if success: message_json = { "message": { "sender": plaintext.sender_guid.encode("hex"), "subject": plaintext.subject, "message_type": PlaintextMessage.Type.Name(plaintext.type), "message": plaintext.message, "timestamp": plaintext.timestamp, "avatar_hash": plaintext.avatar_hash.encode("hex"), "public_key": plaintext.pubkey.encode("hex"), } } if plaintext.handle: message_json["message"]["handle"] = plaintext.handle self.ws.push(json.dumps(sanitize_html(message_json), indent=4)) except Exception as e: self.log.error("Market.Listener.notify Exception: %s" % e)
def getWorkflow(): conf = ConfigParser.RawConfigParser() workflowConfigFile = getConfigFileName("workflow") if (workflowConfigFile == errorCode): Logger.error("failed to get workflow config file!") return errorCode try: conf.read(workflowConfigFile) except: send_mail("failed to read workflow config file!", "time: " + str(time.time())) Logger.error("failed to read workflow config file!") return errorCode workDict = conf.items("workflow") if (workDict is not None): return workDict else: return errorCode
def update_task_impala_load_satus(self, vals): try: con = MySQLdb.connect(host=self.mysqlinfo["host"], port=string.atoi(self.mysqlinfo["port"]), user=self.mysqlinfo["user"], passwd=self.mysqlinfo["passwd"], db=self.mysqlinfo["dbname"]) cursor = con.cursor() sql = "update task_impala_load set status=%s where task_name=%s and batch_id=%s and h_db_name=%s and h_table_name=%s \ and h_table_part=%s and i_db_name=%s and i_table_name=%s and i_table_part=%s and insert_type=%s" cursor.execute(sql, vals) con.commit() except Error, e: Logger.error("mysql error update_task_impala_load_satus" + e.args[0].__str__() + ":" + e.args[1].__str__()) print "mysql error update_task_impala_load_satus%d: %s" % ( e.args[0], e.args[1])
def select_task_impala_sql(self, task_name): try: con = MySQLdb.connect(host=self.mysqlinfo["host"], port=string.atoi(self.mysqlinfo["port"]), user=self.mysqlinfo["user"], passwd=self.mysqlinfo["passwd"], db=self.mysqlinfo["dbname"]) cursor = con.cursor() sql = "select sub_sql from task_impala_sql where task_name='" + task_name + "'" print("impala_sql" + sql) cursor.execute(sql) result = cursor.fetchall() return result except Error, e: Logger.error("mysql error select_task_impala_sql" + e.args[0].__str__() + ":" + e.args[1].__str__()) print "mysql error select_task_impala_sql%d: %s" % (e.args[0], e.args[1])
def execmd(cmd): result = {} result["type"] = cmd # 数据统一用json串发送 try: if cmd == "cpu": result["data"] = get_cpu_rate() elif cmd == "memory": result["data"] = get_mem_rate() elif cmd == "diskio": result["data"] = get_diskio_rate() elif cmd == "network": result["data"] = get_net_rate() elif cmd == "hostinfo": result["data"] = get_host_info() result["hostname"] = socket.gethostname() Logger.info("Collected %s Messages" % cmd) except Exception, e: Logger.error(e)
class ShellLib(object): log = None command = None def __init__(self): self.logger = Logger() def newCommand(self): self.command = Command() def setWorkingDirectory(self, path): self.command.setWorkingDirectory(path) def addParameter(self, value): self.command.addParameter(value) self.logger.info(str(self.command)) def enableShellMode(self): self.command.setShellMode(True) def disableShellMode(self): self.command.setShellMode(False) def execute(self): self.logger.info("Executing.. [%s] wd=%s" % (self.command, self.command.getWorkingDirectory())) p = subprocess.Popen(self.command.getParameters(), shell=self.command.getShellMode(), cwd=self.command.getWorkingDirectory(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() if out: self.logger.info(out) if err: self.logger.error(err) if p.returncode != 0: raise Exception( "Error occurred while trying to execute console command RC=%d" % p.returncode)
def execmd(cmd): result = {} result['type'] = cmd # 数据统一用json串发送 try: if cmd == "cpu": result['data'] = get_cpu_rate() elif cmd == "memory": result['data'] = get_mem_rate() elif cmd == "diskio": result['data'] = get_diskio_rate() elif cmd == "network": result['data'] = get_net_rate() elif cmd == 'hostinfo': result['data'] = get_host_info() result['hostname'] = socket.gethostname() Logger.info('Collected %s Messages' % cmd) except Exception, e: Logger.error(e)
def update_task_runstatus(self, task_id, run_date, runstatus): try: con = MySQLdb.connect(host=self.mysqlinfo["host"], port=string.atoi(self.mysqlinfo["port"]), user=self.mysqlinfo["user"], passwd=self.mysqlinfo["passwd"], db=self.mysqlinfo["dbname"]) cursor = con.cursor() sql = "update task_sceduling set runstatus=%s where task_id=%s and run_date=%s" if runstatus: cursor.execute(sql, ["1", task_id, run_date]) else: cursor.execute(sql, ["0", task_id, run_date]) con.commit() except Error, e: Logger.error("mysql error update_task_sceduling" + e.args[0].__str__() + ":" + e.args[1].__str__()) print "mysql error update_task_sceduling%d: %s" % (e.args[0], e.args[1])
def get_event_message(script_path=None): logger = Logger().get_logger() if script_path is None: logger.warning("script path is empty!") script = LoadFromYaml().save_load(script_path=script_path) if not script: logger.error("script file:{0} load failed".format(script_path)) return "" try: script_conf = script["script config"] # logger.info(script_conf) return script_conf except KeyError: event_cls_list = LaunchExtractor().do(script_path=script_path) message = "" user_input_event_list = [] for event in event_cls_list: # logger.info(event.event_type) if isinstance(event, UserInputEvent): user_input_event_list.append(event) user_message = "input@{0} >> {1}".format( event.window["class name"], event.message) # logger.info(user_message) message += user_message message += '\n' message_body = '*'*20 + 'Script' + '*'*20 + '\n' message_body += "<script>\n" message_body += "delay=1\n" message_body += "script@{0}\n".format(script_path) message_body += message message_body += "</script>\n" logger.error(message_body) message_head = '*'*20 + 'Head' + '*'*20 + '\n' message_head += "<head>\n" message_head += "script_run_count = 1\n" # message_head += "script_index = script_list[1]\n" message_head += "script_delay = 1\n" message_head += "</head>\n" return message_head + message_body
def write(driverNumber, queue, group_id): #time.sleep(40) while True: #print("write thread live") if (config.reset == True): statusThread["write" + driverNumber] = False while (config.reset == True): time.sleep(1) statusThread["write" + driverNumber] = True print(f"Write {driverNumber} voltando a funcionar") if not queue.empty(): msg_type, path, caption = queue.get() print(f"Removed from queue: {msg_type}-{path}-{caption}") try: contact = config.driver[driverNumber].get_contact_from_id( group_id) except Exception as identifier: #config.reset=True print(f"Error in function get_contact_from_id") Logger.error(f"[writer-{driverNumber}] {identifier}") if msg_type == "chat": send_message(contact, caption) elif msg_type in ['document', 'image', 'video', 'ptt', 'audio']: chat_id = contact.get_chat().id print(f"write in path: {path}") if msg_type in ['document', 'ptt', 'audio']: send_media(config.driver[driverNumber], path, chat_id, "", contact) time.sleep(1) contact.get_chat().send_message(caption) else: send_media(config.driver[driverNumber], path, chat_id, caption, contact) os.remove(path) print(f"Deleted: {path}") elif msg_type == "sticker": pass write_on_backup_file(queue_dict[queue], "write", msg_type, path, caption) print(f"Writed: {msg_type}-{path}-{caption}")
class ShellLib(object): log = None command = None def __init__(self): self.logger = Logger() def newCommand(self): self.command = Command() def setWorkingDirectory(self, path): self.command.setWorkingDirectory(path) def addParameter(self, value): self.command.addParameter(value) self.logger.info(str(self.command)) def enableShellMode(self): self.command.setShellMode(True) def disableShellMode(self): self.command.setShellMode(False) def execute(self): self.logger.info("Executing.. [%s] wd=%s" % (self.command, self.command.getWorkingDirectory())) p = subprocess.Popen(self.command.getParameters(), shell=self.command.getShellMode(), cwd=self.command.getWorkingDirectory(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() if out: self.logger.info(out) if err: self.logger.error(err) if p.returncode != 0: raise Exception("Error occurred while trying to execute console command RC=%d" % p.returncode)
def handle(self): while True: self.data = self.request.recv(1024).strip() # cur_thread = threading.currentThread() info = "RECV from ", self.client_address[0], self.client_address[1] Logger.info(info) if self.data == None or len(self.data) == 0: Logger.error("empty command") break Logger.debug(self.data) # business logic here try: cmd = command.decode(self.data) except: Logger.error("format error") break # try: response = application.dispatch(cmd) Logger.debug(response) self.send(response) if response["code"] == -1: # server.shutdown() break
def save(relation): if TrackRelationFactory.__use_db: trackA = relation.trackA.key() trackB = relation.trackB.key() if trackA > trackB: trackC = trackB trackB = trackA trackA = trackC TrackRelationFactory.__db.execute( u"REPLACE INTO track_relation "\ "(trackA, trackB, relation) values (?, ?, ?)", ( trackA, trackB, Binary(pickle.dumps(relation, -1)))) TrackRelationFactory.__db.commit() elif TrackRelationFactory.__use_fs: pathA = join(TrackRelationFactory.__path, relation.trackA.artist.key(), relation.trackA.key()) fileA = u"%s.pkl" % join(pathA, relation.key()) pathB = join(TrackRelationFactory.__path, relation.trackB.artist.key(), relation.trackB.key()) fileB = u"%s.pkl" % join(pathB, relation.key()) if not isdir(pathA): makedirs(pathA) if not isdir(pathB): makedirs(pathB) if fileA > fileB: fileC = fileB fileB = fileA fileA = fileC TrackRelationFactory._save(relation, fileA) if fileA != fileB: if exists(fileB): remove(fileB) try: link(fileA, fileB) except OSError: Logger.error(u"Failed to link %s to %s" % (fileA, fileB)) relation.lastmodified = getmtime(fileA)
def load_file(filename): try: input = open(filename, 'rb') relation = pickle.load(input) input.close() if ArtistRelationFactory.register(relation): relation.lastmodified = getmtime(filename) return True # we forget inactive / unplayed files, so invalid relations are probably # OK Logger.error(u"ArtistRelationFactory.load_file(%s): invalid relation" % filename) except IOError: Logger.error('IOError') Logger.error(format_exc()) except EOFError: Logger.error('EOFError') Logger.error(format_exc()) except AttributeError: Logger.error('AttributeError: Format changed?') Logger.error(format_exc()) return False
class RPCProtocol: """ This is an abstract class for processing and sending rpc messages. A class that implements the `MessageProcessor` interface probably should extend this as it does most of the work of keeping track of messages. """ __metaclass__ = abc.ABCMeta def __init__(self, proto, router, waitTimeout=5): """ Args: proto: A protobuf `Node` object containing info about this node. router: A `RoutingTable` object from dht.routing. Implies a `network.Server` object must be started first. waitTimeout: Consider it a connetion failure if no response within this time window. noisy: Whether or not to log the output for this class. testnet: The network parameters to use. """ self.proto = proto self.router = router self._waitTimeout = waitTimeout self._outstanding = {} self.log = Logger(system=self) def receive_message(self, datagram, connection): m = Message() try: m.ParseFromString(datagram) sender = node.Node(m.sender.guid, m.sender.ip, m.sender.port, m.sender.signedPublicKey, m.sender.vendor) except Exception: # If message isn't formatted property then ignore self.log.warning("received unknown message from %s, ignoring" % str(connection.dest_addr)) return False if m.testnet != self.multiplexer.testnet: self.log.warning("received message from %s with incorrect network parameters." % str(connection.dest_addr)) connection.shutdown() return False if m.protoVer < PROTOCOL_VERSION: self.log.warning("received message from %s with incompatible protocol version." % str(connection.dest_addr)) connection.shutdown() return False # Check that the GUID is valid. If not, ignore if self.router.isNewNode(sender): try: pubkey = m.sender.signedPublicKey[len(m.sender.signedPublicKey) - 32:] verify_key = nacl.signing.VerifyKey(pubkey) verify_key.verify(m.sender.signedPublicKey) h = nacl.hash.sha512(m.sender.signedPublicKey) pow_hash = h[64:128] if int(pow_hash[:6], 16) >= 50 or hexlify(m.sender.guid) != h[:40]: raise Exception('Invalid GUID') except Exception: self.log.warning("received message from sender with invalid GUID, ignoring") connection.shutdown() return False if m.sender.vendor: self.db.VendorStore().save_vendor(m.sender.guid, m.sender.ip, m.sender.port, m.sender.signedPublicKey) msgID = m.messageID if m.command == NOT_FOUND: data = None else: data = tuple(m.arguments) if msgID in self._outstanding: self._acceptResponse(msgID, data, sender) elif m.command != NOT_FOUND: self._acceptRequest(msgID, str(Command.Name(m.command)).lower(), data, sender, connection) def _acceptResponse(self, msgID, data, sender): if data is not None: msgargs = (b64encode(msgID), sender) self.log.debug("received response for message id %s from %s" % msgargs) else: self.log.warning("received 404 error response from %s" % sender) d, timeout = self._outstanding[msgID] timeout.cancel() d.callback((True, data)) del self._outstanding[msgID] def _acceptRequest(self, msgID, funcname, args, sender, connection): self.log.debug("received request from %s, command %s" % (sender, funcname.upper())) f = getattr(self, "rpc_%s" % funcname, None) if f is None or not callable(f): msgargs = (self.__class__.__name__, funcname) self.log.error("%s has no callable method rpc_%s; ignoring request" % msgargs) return False if funcname == "hole_punch": f(sender, *args) else: d = defer.maybeDeferred(f, sender, *args) d.addCallback(self._sendResponse, funcname, msgID, sender, connection) def _sendResponse(self, response, funcname, msgID, sender, connection): self.log.debug("sending response for msg id %s to %s" % (b64encode(msgID), sender)) m = Message() m.messageID = msgID m.sender.MergeFrom(self.proto) m.protoVer = PROTOCOL_VERSION m.testnet = self.multiplexer.testnet if response is None: m.command = NOT_FOUND else: m.command = Command.Value(funcname.upper()) for arg in response: m.arguments.append(str(arg)) data = m.SerializeToString() connection.send_message(data) def _timeout(self, msgID, address): """ If a message times out we are first going to try hole punching because the node may be behind a restricted NAT. If it is successful, the original should get through. This timeout will only fire if the hole punching fails. """ # pylint: disable=pointless-string-statement """ Hole punching disabled for now seed = SEED_NODE_TESTNET if self.multiplexer.testnet else SEED_NODE if not hp and self.multiplexer.ip_address[0] != seed[0]: args = (address[0], address[1], b64encode(msgID)) self.log.debug("did not receive reply from %s:%s for msgID %s, trying hole punching..." % args) self.hole_punch(seed, address[0], address[1], "True") timeout = reactor.callLater(self._waitTimeout, self._timeout, msgID, address, True) self._outstanding[msgID][1] = timeout else: """ args = (b64encode(msgID), self._waitTimeout) self.log.warning("did not receive reply for msg id %s within %i seconds" % args) self._outstanding[msgID][0].callback((False, None)) del self._outstanding[msgID] self.multiplexer[address].shutdown() def rpc_hole_punch(self, sender, ip, port, relay="False"): """ A method for handling an incoming HOLE_PUNCH message. Relay the message to the correct node if it's not for us. Otherwise sent a datagram to allow the other node to punch through our NAT. """ if relay == "True": self.hole_punch((ip, int(port)), sender.ip, sender.port) else: self.log.debug("punching through NAT for %s:%s" % (ip, port)) self.multiplexer.send_datagram(" ", (ip, int(port))) def _get_waitTimeout(self, command): if command == GET_IMAGE or command == GET_CONTRACT: return 100 else: return self._waitTimeout def __getattr__(self, name): if name.startswith("_") or name.startswith("rpc_"): return object.__getattr__(self, name) try: return object.__getattr__(self, name) except AttributeError: pass def func(address, *args): msgID = sha1(str(random.getrandbits(255))).digest() m = Message() m.messageID = msgID m.sender.MergeFrom(self.proto) m.command = Command.Value(name.upper()) m.protoVer = PROTOCOL_VERSION for arg in args: m.arguments.append(str(arg)) m.testnet = self.multiplexer.testnet data = m.SerializeToString() d = defer.Deferred() timeout = reactor.callLater(self._get_waitTimeout(m.command), self._timeout, msgID, address) self._outstanding[msgID] = [d, timeout] self.multiplexer.send_message(data, address) self.log.debug("calling remote function %s on %s (msgid %s)" % (name, address, b64encode(msgID))) return d return func
def by_key(key): if TrackRelationFactory.__lib.has_key(key): return TrackRelationFactory.__lib[key] Logger.error(u"TrackRelationFactory.by_key(%s): key not found" % key) return None
cursor.close() conn.commit() def callback(ch, method, properties, body): print " [x] %r" % (body,) data = json.loads(body) print data write_sql(data) try: connection = pika.BlockingConnection(pika.ConnectionParameters(host="localhost")) channel = connection.channel() channel.exchange_declare(exchange="logs", type="fanout") except Exception, e: Logger.error(e) result = channel.queue_declare(exclusive=True) queue_name = result.method.queue channel.queue_bind(exchange="logs", queue=queue_name) print " [*] Waiting for logs. To exit press CTRL+C" channel.basic_consume(callback, queue=queue_name, no_ack=True) channel.start_consuming()
class Global: """ Class represents parser for global.conf """ # options common for all connectors conf_ams = {'AMS': ['Host', 'Token', 'Project', 'Topic', 'Bulk', 'PackSingleMsg']} conf_general = {'General': ['PublishAms', 'WriteAvro']} conf_auth = {'Authentication': ['HostKey', 'HostCert', 'CAPath', 'CAFile', 'VerifyServerCert', 'UsePlainHttpAuth', 'HttpUser', 'HttpPass']} conf_conn = {'Connection': ['Timeout', 'Retry', 'SleepRetry']} conf_state = {'InputState': ['SaveDir', 'Days']} # options specific for every connector conf_topo_schemas = {'AvroSchemas': ['TopologyGroupOfEndpoints', 'TopologyGroupOfGroups']} conf_topo_output = {'Output': ['TopologyGroupOfEndpoints', 'TopologyGroupOfGroups']} conf_downtimes_schemas = {'AvroSchemas': ['Downtimes']} conf_downtimes_output = {'Output': ['Downtimes']} conf_weights_schemas = {'AvroSchemas': ['Weights']} conf_weights_output = {'Output': ['Weights']} conf_poem_output = {'Output': ['Poem']} conf_poem_schemas = {'AvroSchemas': ['Poem']} def __init__(self, caller, confpath=None, **kwargs): self.optional = dict() self.logger = Logger(str(self.__class__)) self._filename = '/etc/argo-egi-connectors/global.conf' if not confpath else confpath self._checkpath = kwargs['checkpath'] if 'checkpath' in kwargs.keys() else False self.optional.update(self._lowercase_dict(self.conf_ams)) self.optional.update(self._lowercase_dict(self.conf_auth)) self.shared_secopts = self._merge_dict(self.conf_ams, self.conf_general, self.conf_auth, self.conf_conn, self.conf_state) self.secopts = {'topology-gocdb-connector.py': self._merge_dict(self.shared_secopts, self.conf_topo_schemas, self.conf_topo_output), 'downtimes-gocdb-connector.py': self._merge_dict(self.shared_secopts, self.conf_downtimes_schemas, self.conf_downtimes_output), 'weights-vapor-connector.py': self._merge_dict(self.shared_secopts, self.conf_weights_schemas, self.conf_weights_output), 'poem-connector.py': self._merge_dict(self.shared_secopts, self.conf_poem_schemas, self.conf_poem_output) } if caller: self.caller_secopts = self.secopts[os.path.basename(caller)] else: self.caller_secopts = self.shared_secopts def _merge_dict(self, *args): newd = dict() for d in args: newd.update(d) return newd def _lowercase_dict(self, d): newd = dict() for k in d.iterkeys(): opts = [o.lower() for o in d[k]] newd[k.lower()] = opts return newd def merge_opts(self, custopt, section): newd = custopt.copy() opts = [o for o in self.options.keys() if o.startswith(section)] for o in opts: if o in newd: continue newd.update({o: self.options[o]}) return newd def is_complete(self, opts, section): all = set([section+o for o in self.optional[section]]) diff = all.symmetric_difference(opts.keys()) if diff: return (False, diff) return (True, None) def _concat_sectopt(self, d): opts = list() for k in d.iterkeys(): for v in d[k]: opts.append(k+v) return opts def _one_active(self, options): loweropts = self._lowercase_dict(options) lval = [eval(self.options[k]) for k in self._concat_sectopt(loweropts)] if any(lval): return True else: return False def parse(self): config = ConfigParser.ConfigParser() if not os.path.exists(self._filename): self.logger.error('Could not find %s' % self._filename) raise SystemExit(1) config.read(self._filename) options = {} lower_section = [sec.lower() for sec in config.sections()] try: for sect, opts in self.caller_secopts.items(): if (sect.lower() not in lower_section and sect.lower() not in self.optional.keys()): raise ConfigParser.NoSectionError(sect.lower()) for opt in opts: for section in config.sections(): if section.lower().startswith(sect.lower()): try: optget = config.get(section, opt) if self._checkpath and os.path.isfile(optget) is False: raise OSError(errno.ENOENT, optget) if ('output' in section.lower() and 'DATE' not in optget): logger.error('No DATE placeholder in %s' % option) raise SystemExit(1) options.update({(sect+opt).lower(): optget}) except ConfigParser.NoOptionError as e: s = e.section.lower() if (s in self.optional.keys() and e.option in self.optional[s]): pass else: raise e self.options = options if not self._one_active(self.conf_general): self.logger.error('At least one of %s needs to be True' % (', '.join(self._concat_sectopt(self.conf_general)))) raise SystemExit(1) except ConfigParser.NoOptionError as e: self.logger.error(e.message) raise SystemExit(1) except ConfigParser.NoSectionError as e: self.logger.error("%s defined" % (e.args[0])) raise SystemExit(1) except OSError as e: self.logger.error('%s %s' % (os.strerror(e.args[0]), e.args[1])) raise SystemExit(1) return options
class CustomerConf: """ Class with parser for customer.conf and additional helper methods """ _custattrs = None _cust = {} _defjobattrs = {'topology-gocdb-connector.py' : ['TopoFetchType', 'TopoSelectGroupOfGroups', 'TopoSelectGroupOfEndpoints', 'TopoFeed', 'TopoFeedPaging'], 'poem-connector.py': ['PoemServerHost', 'PoemServerVO', 'PoemNamespace'], 'downtimes-gocdb-connector.py': ['DowntimesFeed'], 'weights-vapor-connector.py': ['WeightsFeed'] } _jobs, _jobattrs = {}, None _cust_optional = ['AmsHost', 'AmsProject', 'AmsToken', 'AmsTopic', 'AmsPackSingleMsg', 'AuthenticationUsePlainHttpAuth', 'AuthenticationHttpUser', 'AuthenticationHttpPass'] tenantdir = '' deftopofeed = 'https://goc.egi.eu/gocdbpi/' def __init__(self, caller, confpath, **kwargs): self.logger = Logger(str(self.__class__)) self._filename = '/etc/argo-egi-connectors/customer.conf' if not confpath else confpath if not kwargs: self._jobattrs = self._defjobattrs[os.path.basename(caller)] else: if 'jobattrs' in kwargs.keys(): self._jobattrs = kwargs['jobattrs'] if 'custattrs' in kwargs.keys(): self._custattrs = kwargs['custattrs'] def parse(self): config = ConfigParser.ConfigParser() if not os.path.exists(self._filename): self.logger.error('Could not find %s' % self._filename) raise SystemExit(1) config.read(self._filename) lower_custopt = [oo.lower() for oo in self._cust_optional] for section in config.sections(): if section.lower().startswith('CUSTOMER_'.lower()): optopts = dict() try: custjobs = config.get(section, 'Jobs').split(',') custjobs = [job.strip() for job in custjobs] custdir = config.get(section, 'OutputDir') custname = config.get(section, 'Name') for o in lower_custopt: try: code = "optopts.update(%s = config.get(section, '%s'))" % (o, o) exec code except ConfigParser.NoOptionError as e: if e.option in lower_custopt: pass else: raise e except ConfigParser.NoOptionError as e: self.logger.error(e.message) raise SystemExit(1) self._cust.update({section: {'Jobs': custjobs, 'OutputDir': custdir, 'Name': custname}}) if optopts: ams, auth = {}, {} for k, v in optopts.iteritems(): if k.startswith('ams'): ams.update({k: v}) if k.startswith('authentication'): auth.update({k: v}) self._cust[section].update(AmsOpts=ams) self._cust[section].update(AuthOpts=auth) if self._custattrs: for attr in self._custattrs: if config.has_option(section, attr): self._cust[section].update({attr: config.get(section, attr)}) for cust in self._cust: for job in self._cust[cust]['Jobs']: if config.has_section(job): try: profiles = config.get(job, 'Profiles') dirname = config.get(job, 'Dirname') except ConfigParser.NoOptionError as e: self.logger.error(e.message) raise SystemExit(1) self._jobs.update({job: {'Profiles': profiles, 'Dirname': dirname}}) if self._jobattrs: for attr in self._jobattrs: if config.has_option(job, attr): self._jobs[job].update({attr: config.get(job, attr)}) else: self.logger.error("Could not find Jobs: %s for customer: %s" % (job, cust)) raise SystemExit(1) def _sect_to_dir(self, sect): try: match = re.match('(?:^\w+?_)(\w+)', sect) assert match != None dirname = match.group(1) except (AssertionError, KeyError) as e: self.logger.error("Could not get Dirname for %s" % e) raise SystemExit(1) return dirname def _dir_from_sect(self, sect, d): dirname = '' for k, v in d.items(): if k == sect: if 'Dirname' in v.keys(): dirname = v['Dirname'] elif 'OutputDir' in v.keys(): dirname = v['OutputDir'] else: dirname = self._sect_to_dir(sect) return dirname def get_jobdir(self, job): return self._dir_from_sect(job, self._jobs) def get_amsopts(self, cust): if 'AmsOpts' in self._cust[cust]: return self._cust[cust]['AmsOpts'] else: return dict() def get_authopts(self, feed, jobcust): for job, cust in jobcust: if 'AuthOpts' in self._cust[cust]: return self._cust[cust]['AuthOpts'] else: return dict() def get_fulldir(self, cust, job): return self.get_custdir(cust) + '/' + self.get_jobdir(job) + '/' def get_fullstatedir(self, root, cust, job): return root + '/' + self.get_custname(cust) + '/' + self.get_jobdir(job) def get_custdir(self, cust): return self._dir_from_sect(cust, self._cust) def get_custname(self, cust): return self._cust[cust]['Name'] def make_dirstruct(self, root=None): dirs = [] for cust in self._cust.keys(): for job in self.get_jobs(cust): if root: dirs.append(root + '/' + self.get_custname(cust) + '/' + self.get_jobdir(job)) else: dirs.append(self.get_custdir(cust) + '/' + self.get_jobdir(job)) for d in dirs: try: os.makedirs(d) except OSError as e: if e.args[0] != errno.EEXIST: self.logger.error('%s %s %s' % (os.strerror(e.args[0]), e.args[1], d)) raise SystemExit(1) def get_jobs(self, cust): jobs = [] try: jobs = self._cust[cust]['Jobs'] except KeyError: self.logger.error("Could not get Jobs for %s" % cust) raise SystemExit(1) return jobs def get_customers(self): return self._cust.keys() def get_profiles(self, job): profiles = self._jobs[job]['Profiles'].split(',') for i, p in enumerate(profiles): profiles[i] = p.strip() return profiles def get_gocdb_fetchtype(self, job): return self._jobs[job]['TopoFetchType'] def _get_tags(self, job, option): tags = {} if option in self._jobs[job].keys(): tagstr = self._jobs[job][option] match = re.findall("(\w+)\s*:\s*(\(.*?\))", tagstr) if match is not None: for m in match: tags.update({m[0]: [e.strip('() ') for e in m[1].split(',')]}) match = re.findall('([\w]+)\s*:\s*([\w\.\-\_]+)', tagstr) if match is not None: for m in match: tags.update({m[0]: m[1]}) else: self.logger.error("Could not parse option %s: %s" % (option, tagstr)) return dict() return tags def get_gocdb_ggtags(self, job): return self._get_tags(job, 'TopoSelectGroupOfGroups') def get_gocdb_getags(self, job): return self._get_tags(job, 'TopoSelectGroupOfEndpoints') def get_vo_ggtags(self, job): return self._get_tags(job, 'TopoSelectGroupOfGroups') def _get_feed(self, job, key): try: feed = self._jobs[job][key] except KeyError: feed = '' return feed def _is_paginated(self, job): paging = False try: paging = self._jobs[job]['TopoFeedPaging'] except KeyError: pass return paging def _update_feeds(self, feeds, feedurl, job, cust): if feedurl in feeds.keys(): feeds[feedurl].append((job, cust)) elif feedurl: feeds[feedurl] = [] feeds[feedurl].append((job, cust)) def get_feedscopes(self, feed, jobcust): ggtags, getags = [], [] distinct_scopes = set() for job, cust in jobcust: gg = self._get_tags(job, 'TopoSelectGroupOfGroups') ge = self._get_tags(job, 'TopoSelectGroupOfEndpoints') for g in gg.items() + ge.items(): if 'Scope'.lower() == g[0].lower(): if isinstance(g[1], list): distinct_scopes.update(g[1]) else: distinct_scopes.update([g[1]]) return distinct_scopes def is_paginated(self, feed, jobcust): paginated = False for job, cust in jobcust: paginated = self._is_paginated(job) if paginated: break return eval(str(paginated)) def get_mapfeedjobs(self, caller, name=None, deffeed=None): feeds = {} for c in self.get_customers(): for job in self.get_jobs(c): if 'topology' in caller: feedurl = self._get_feed(job, 'TopoFeed') if feedurl: self._update_feeds(feeds, feedurl, job, c) else: feedurl = deffeed self._update_feeds(feeds, feedurl, job, c) elif 'downtimes' in caller: feedurl = self._get_feed(job, 'DowntimesFeed') if feedurl: self._update_feeds(feeds, feedurl, job, c) else: feedurl = deffeed self._update_feeds(feeds, feedurl, job, c) elif 'weights' in caller: feedurl = self._get_feed(job, 'WeightsFeed') if feedurl: self._update_feeds(feeds, feedurl, job, c) else: feedurl = deffeed self._update_feeds(feeds, feedurl, job, c) return feeds def get_poemserver_host(self, job): return self._jobs[job]['PoemServerHost'] def get_poemserver_vo(self, job): vo = self._jobs[job]['PoemServerVO'].split(',') for i, p in enumerate(vo): vo[i] = p.strip() return vo def get_namespace(self, job): return self._jobs[job]['PoemNamespace']
def by_key(key): if ArtistRelationFactory.__lib.has_key(key): return ArtistRelationFactory.__lib[key] Logger.error(u"ArtistRelationFactory.by_key(%s): key not found" % key) return None
class MarketProtocol(RPCProtocol): implements(MessageProcessor) def __init__(self, node, router, signing_key, database): self.router = router self.node = node RPCProtocol.__init__(self, node, router) self.log = Logger(system=self) self.multiplexer = None self.db = database self.signing_key = signing_key self.listeners = [] self.handled_commands = [GET_CONTRACT, GET_IMAGE, GET_PROFILE, GET_LISTINGS, GET_USER_METADATA, GET_CONTRACT_METADATA, FOLLOW, UNFOLLOW, GET_FOLLOWERS, GET_FOLLOWING, BROADCAST, MESSAGE, ORDER, ORDER_CONFIRMATION, COMPLETE_ORDER, DISPUTE_OPEN, DISPUTE_CLOSE, GET_RATINGS, REFUND] def connect_multiplexer(self, multiplexer): self.multiplexer = multiplexer def add_listener(self, listener): self.listeners.append(listener) def rpc_get_contract(self, sender, contract_hash): self.log.info("serving contract %s to %s" % (contract_hash.encode('hex'), sender)) self.router.addContact(sender) try: with open(self.db.filemap.get_file(contract_hash.encode("hex")), "r") as filename: contract = filename.read() return [contract] except Exception: self.log.warning("could not find contract %s" % contract_hash.encode('hex')) return None def rpc_get_image(self, sender, image_hash): self.router.addContact(sender) try: if len(image_hash) != 20: self.log.warning("Image hash is not 20 characters %s" % image_hash) raise Exception("Invalid image hash") self.log.info("serving image %s to %s" % (image_hash.encode('hex'), sender)) with open(self.db.filemap.get_file(image_hash.encode("hex")), "rb") as filename: image = filename.read() return [image] except Exception: self.log.warning("could not find image %s" % image_hash[:20].encode('hex')) return None def rpc_get_profile(self, sender): self.log.info("serving profile to %s" % sender) self.router.addContact(sender) try: proto = Profile(self.db).get(True) return [proto, self.signing_key.sign(proto)[:64]] except Exception: self.log.error("unable to load the profile") return None def rpc_get_user_metadata(self, sender): self.log.info("serving user metadata to %s" % sender) self.router.addContact(sender) try: proto = Profile(self.db).get(False) m = Metadata() m.name = proto.name m.handle = proto.handle m.short_description = proto.short_description m.avatar_hash = proto.avatar_hash m.nsfw = proto.nsfw return [m.SerializeToString(), self.signing_key.sign(m.SerializeToString())[:64]] except Exception: self.log.error("unable to load profile metadata") return None def rpc_get_listings(self, sender): self.log.info("serving store listings to %s" % sender) self.router.addContact(sender) try: p = Profile(self.db).get() l = Listings() l.ParseFromString(self.db.listings.get_proto()) l.handle = p.handle l.avatar_hash = p.avatar_hash return [l.SerializeToString(), self.signing_key.sign(l.SerializeToString())[:64]] except Exception: self.log.warning("could not find any listings in the database") return None def rpc_get_contract_metadata(self, sender, contract_hash): self.log.info("serving metadata for contract %s to %s" % (contract_hash.encode("hex"), sender)) self.router.addContact(sender) try: proto = self.db.listings.get_proto() p = Profile(self.db).get() l = Listings() l.ParseFromString(proto) for listing in l.listing: if listing.contract_hash == contract_hash: listing.avatar_hash = p.avatar_hash listing.handle = p.handle ser = listing.SerializeToString() return [ser, self.signing_key.sign(ser)[:64]] except Exception: self.log.warning("could not find metadata for contract %s" % contract_hash.encode("hex")) return None def rpc_follow(self, sender, proto, signature): self.log.info("received follow request from %s" % sender) self.router.addContact(sender) try: verify_key = nacl.signing.VerifyKey(sender.pubkey) verify_key.verify(proto, signature) f = Followers.Follower() f.ParseFromString(proto) if f.guid != sender.id: raise Exception('GUID does not match sending node') if f.following != self.node.id: raise Exception('Following wrong node') f.signature = signature self.db.follow.set_follower(f) proto = Profile(self.db).get(False) m = Metadata() m.name = proto.name m.handle = proto.handle m.avatar_hash = proto.avatar_hash m.short_description = proto.short_description m.nsfw = proto.nsfw for listener in self.listeners: try: verifyObject(NotificationListener, listener) listener.notify(sender.id, f.metadata.handle, "follow", "", "", f.metadata.avatar_hash) except DoesNotImplement: pass return ["True", m.SerializeToString(), self.signing_key.sign(m.SerializeToString())[:64]] except Exception: self.log.warning("failed to validate follower") return ["False"] def rpc_unfollow(self, sender, signature): self.log.info("received unfollow request from %s" % sender) self.router.addContact(sender) try: verify_key = nacl.signing.VerifyKey(sender.pubkey) verify_key.verify("unfollow:" + self.node.id, signature) f = self.db.follow f.delete_follower(sender.id) return ["True"] except Exception: self.log.warning("failed to validate signature on unfollow request") return ["False"] def rpc_get_followers(self, sender): self.log.info("serving followers list to %s" % sender) self.router.addContact(sender) ser = self.db.follow.get_followers() if ser is None: return None else: return [ser, self.signing_key.sign(ser)[:64]] def rpc_get_following(self, sender): self.log.info("serving following list to %s" % sender) self.router.addContact(sender) ser = self.db.follow.get_following() if ser is None: return None else: return [ser, self.signing_key.sign(ser)[:64]] def rpc_broadcast(self, sender, message, signature): if len(message) <= 140 and self.db.follow.is_following(sender.id): try: verify_key = nacl.signing.VerifyKey(sender.pubkey) verify_key.verify(message, signature) except Exception: self.log.warning("received invalid broadcast from %s" % sender) return ["False"] self.log.info("received a broadcast from %s" % sender) self.router.addContact(sender) for listener in self.listeners: try: verifyObject(BroadcastListener, listener) listener.notify(sender.id, message) except DoesNotImplement: pass return ["True"] else: return ["False"] def rpc_message(self, sender, pubkey, encrypted): try: box = Box(self.signing_key.to_curve25519_private_key(), PublicKey(pubkey)) plaintext = box.decrypt(encrypted) p = PlaintextMessage() p.ParseFromString(plaintext) signature = p.signature p.ClearField("signature") verify_key = nacl.signing.VerifyKey(p.pubkey) verify_key.verify(p.SerializeToString(), signature) h = nacl.hash.sha512(p.pubkey) pow_hash = h[40:] if int(pow_hash[:6], 16) >= 50 or p.sender_guid.encode("hex") != h[:40] or p.sender_guid != sender.id: raise Exception('Invalid guid') self.log.info("received a message from %s" % sender) self.router.addContact(sender) for listener in self.listeners: try: verifyObject(MessageListener, listener) listener.notify(p, signature) except DoesNotImplement: pass return ["True"] except Exception: self.log.warning("received invalid message from %s" % sender) return ["False"] def rpc_order(self, sender, pubkey, encrypted): try: box = Box(self.signing_key.to_curve25519_private_key(), PublicKey(pubkey)) order = box.decrypt(encrypted) c = Contract(self.db, contract=json.loads(order, object_pairs_hook=OrderedDict), testnet=self.multiplexer.testnet) v = c.verify(sender.pubkey) if v is True: self.router.addContact(sender) self.log.info("received an order from %s, waiting for payment..." % sender) payment_address = c.contract["buyer_order"]["order"]["payment"]["address"] chaincode = c.contract["buyer_order"]["order"]["payment"]["chaincode"] masterkey_b = c.contract["buyer_order"]["order"]["id"]["pubkeys"]["bitcoin"] buyer_key = derive_childkey(masterkey_b, chaincode) amount = c.contract["buyer_order"]["order"]["payment"]["amount"] listing_hash = c.contract["vendor_offer"]["listing"]["contract_id"] signature = self.signing_key.sign( str(payment_address) + str(amount) + str(listing_hash) + str(buyer_key))[:64] c.await_funding(self.get_notification_listener(), self.multiplexer.blockchain, signature, False) return [signature] else: self.log.warning("received invalid order from %s reason %s" % (sender, v)) return ["False"] except Exception, e: self.log.error("Exception (%s) occurred processing order from %s" % (e.message, sender)) return ["False"]
def execmd(cmd): result = {} result["type"] = cmd # 数据统一用json串发送 try: if cmd == "cpu": result["data"] = get_cpu_rate() elif cmd == "memory": result["data"] = get_mem_rate() elif cmd == "diskio": result["data"] = get_diskio_rate() elif cmd == "network": result["data"] = get_net_rate() elif cmd == "hostinfo": result["data"] = get_host_info() result["hostname"] = socket.gethostname() Logger.info("Collected %s Messages" % cmd) except Exception, e: Logger.error(e) return result if __name__ == "__main__": if sys.argv[1] and sys.argv[1] in ["cpu", "memory", "diskio", "network", "hostinfo"]: result = execmd(sys.argv[1]) data_tran(json.dumps(result)) else: Logger.error("wrong parameter")
class MarketProtocol(RPCProtocol): implements(MessageProcessor) def __init__(self, node_proto, router, signing_key): self.router = router RPCProtocol.__init__(self, node_proto, router) self.log = Logger(system=self) self.multiplexer = None self.hashmap = HashMap() self.signing_key = signing_key self.listeners = [] self.handled_commands = [GET_CONTRACT, GET_IMAGE, GET_PROFILE, GET_LISTINGS, GET_USER_METADATA, GET_CONTRACT_METADATA, FOLLOW, UNFOLLOW, GET_FOLLOWERS, GET_FOLLOWING, NOTIFY, MESSAGE] def connect_multiplexer(self, multiplexer): self.multiplexer = multiplexer def add_listener(self, listener): self.listeners.append(listener) def rpc_get_contract(self, sender, contract_hash): self.log.info("Looking up contract ID %s" % contract_hash.encode('hex')) self.router.addContact(sender) try: with open(self.hashmap.get_file(contract_hash), "r") as filename: contract = filename.read() return [contract] except Exception: self.log.warning("Could not find contract %s" % contract_hash.encode('hex')) return ["None"] def rpc_get_image(self, sender, image_hash): self.log.info("Looking up image with hash %s" % image_hash.encode('hex')) self.router.addContact(sender) try: with open(self.hashmap.get_file(image_hash), "r") as filename: image = filename.read() return [image] except Exception: self.log.warning("Could not find image %s" % image_hash.encode('hex')) return ["None"] def rpc_get_profile(self, sender): self.log.info("Fetching profile") self.router.addContact(sender) try: proto = Profile().get(True) return [proto, self.signing_key.sign(proto)[:64]] except Exception: self.log.error("Unable to load the profile") return ["None"] def rpc_get_user_metadata(self, sender): self.log.info("Fetching metadata") self.router.addContact(sender) try: proto = Profile().get(False) m = Metadata() m.name = proto.name m.handle = proto.handle m.short_description = proto.short_description m.avatar_hash = proto.avatar_hash m.nsfw = proto.nsfw return [m.SerializeToString(), self.signing_key.sign(m.SerializeToString())[:64]] except Exception: self.log.error("Unable to get the profile metadata") return ["None"] def rpc_get_listings(self, sender): self.log.info("Fetching listings") self.router.addContact(sender) try: p = Profile().get() l = Listings() l.ParseFromString(ListingsStore().get_proto()) l.handle = p.handle l.avatar_hash = p.avatar_hash return [l.SerializeToString(), self.signing_key.sign(l.SerializeToString())[:64]] except Exception: self.log.warning("Could not find any listings in the database") return ["None"] def rpc_get_contract_metadata(self, sender, contract_hash): self.log.info("Fetching metadata for contract %s" % hexlify(contract_hash)) self.router.addContact(sender) try: proto = ListingsStore().get_proto() l = Listings() l.ParseFromString(proto) for listing in l.listing: if listing.contract_hash == contract_hash: ser = listing.SerializeToString() return [ser, self.signing_key.sign(ser)[:64]] except Exception: self.log.warning("Could not find metadata for contract %s" % hexlify(contract_hash)) return ["None"] def rpc_follow(self, sender, proto, signature): self.log.info("Follow request from %s" % sender.id.encode("hex")) self.router.addContact(sender) try: verify_key = nacl.signing.VerifyKey(sender.signed_pubkey[64:]) verify_key.verify(proto, signature) f = Followers.Follower() f.ParseFromString(proto) if f.guid != sender.id: raise Exception('GUID does not match sending node') if f.following != self.proto.guid: raise Exception('Following wrong node') f.signature = signature FollowData().set_follower(f) proto = Profile().get(False) m = Metadata() m.name = proto.name m.handle = proto.handle m.avatar_hash = proto.avatar_hash m.nsfw = proto.nsfw return ["True", m.SerializeToString(), self.signing_key.sign(m.SerializeToString())[:64]] except Exception: self.log.warning("Failed to validate follower") return ["False"] def rpc_unfollow(self, sender, signature): self.log.info("Unfollow request from %s" % sender.id.encode("hex")) self.router.addContact(sender) try: verify_key = nacl.signing.VerifyKey(sender.signed_pubkey[64:]) verify_key.verify("unfollow:" + self.proto.guid, signature) f = FollowData() f.delete_follower(sender.id) return ["True"] except Exception: self.log.warning("Failed to validate follower signature") return ["False"] def rpc_get_followers(self, sender): self.log.info("Fetching followers list from db") self.router.addContact(sender) ser = FollowData().get_followers() if ser is None: return ["None"] else: return [ser, self.signing_key.sign(ser)[:64]] def rpc_get_following(self, sender): self.log.info("Fetching following list from db") self.router.addContact(sender) ser = FollowData().get_following() if ser is None: return ["None"] else: return [ser, self.signing_key.sign(ser)[:64]] def rpc_notify(self, sender, message, signature): if len(message) <= 140 and FollowData().is_following(sender.id): try: verify_key = nacl.signing.VerifyKey(sender.signed_pubkey[64:]) verify_key.verify(message, signature) except Exception: return ["False"] self.log.info("Received a notification from %s" % sender) self.router.addContact(sender) for listener in self.listeners: try: verifyObject(NotificationListener, listener) listener.notify(sender.id, message) except DoesNotImplement: pass return ["True"] else: return ["False"] def rpc_message(self, sender, pubkey, encrypted): try: box = Box(PrivateKey(self.signing_key.encode(nacl.encoding.RawEncoder)), PublicKey(pubkey)) plaintext = box.decrypt(encrypted) p = Plaintext_Message() p.ParseFromString(plaintext) signature = p.signature p.ClearField("signature") verify_key = nacl.signing.VerifyKey(p.signed_pubkey[64:]) verify_key.verify(p.SerializeToString(), signature) h = nacl.hash.sha512(p.signed_pubkey) pow_hash = h[64:128] if int(pow_hash[:6], 16) >= 50 or hexlify(p.sender_guid) != h[:40] or p.sender_guid != sender.id: raise Exception('Invalid guid') self.log.info("Received a message from %s" % sender) self.router.addContact(sender) for listener in self.listeners: try: verifyObject(MessageListener, listener) listener.notify(p, signature) except DoesNotImplement: pass return ["True"] except Exception: self.log.error("Received invalid message from %s" % sender) return ["False"] def callGetContract(self, nodeToAsk, contract_hash): address = (nodeToAsk.ip, nodeToAsk.port) d = self.get_contract(address, contract_hash) return d.addCallback(self.handleCallResponse, nodeToAsk) def callGetImage(self, nodeToAsk, image_hash): address = (nodeToAsk.ip, nodeToAsk.port) d = self.get_image(address, image_hash) return d.addCallback(self.handleCallResponse, nodeToAsk) def callGetProfile(self, nodeToAsk): address = (nodeToAsk.ip, nodeToAsk.port) d = self.get_profile(address) return d.addCallback(self.handleCallResponse, nodeToAsk) def callGetUserMetadata(self, nodeToAsk): address = (nodeToAsk.ip, nodeToAsk.port) d = self.get_user_metadata(address) return d.addCallback(self.handleCallResponse, nodeToAsk) def callGetListings(self, nodeToAsk): address = (nodeToAsk.ip, nodeToAsk.port) d = self.get_listings(address) return d.addCallback(self.handleCallResponse, nodeToAsk) def callGetContractMetadata(self, nodeToAsk, contract_hash): address = (nodeToAsk.ip, nodeToAsk.port) d = self.get_contract_metadata(address, contract_hash) return d.addCallback(self.handleCallResponse, nodeToAsk) def callFollow(self, nodeToAsk, proto, signature): address = (nodeToAsk.ip, nodeToAsk.port) d = self.follow(address, proto, signature) return d.addCallback(self.handleCallResponse, nodeToAsk) def callUnfollow(self, nodeToAsk, signature): address = (nodeToAsk.ip, nodeToAsk.port) d = self.unfollow(address, signature) return d.addCallback(self.handleCallResponse, nodeToAsk) def callGetFollowers(self, nodeToAsk): address = (nodeToAsk.ip, nodeToAsk.port) d = self.get_followers(address) return d.addCallback(self.handleCallResponse, nodeToAsk) def callGetFollowing(self, nodeToAsk): address = (nodeToAsk.ip, nodeToAsk.port) d = self.get_following(address) return d.addCallback(self.handleCallResponse, nodeToAsk) def callNotify(self, nodeToAsk, message, signature): address = (nodeToAsk.ip, nodeToAsk.port) d = self.notify(address, message, signature) return d.addCallback(self.handleCallResponse, nodeToAsk) def callMessage(self, nodeToAsk, ehemeral_pubkey, ciphertext): address = (nodeToAsk.ip, nodeToAsk.port) d = self.message(address, ehemeral_pubkey, ciphertext) return d.addCallback(self.handleCallResponse, nodeToAsk) def handleCallResponse(self, result, node): """ If we get a response, add the node to the routing table. If we get no response, make sure it's removed from the routing table. """ if result[0]: self.log.info("got response from %s, adding to router" % node) self.router.addContact(node) else: self.log.debug("no response from %s, removing from router" % node) self.router.removeContact(node) return result def __iter__(self): return iter(self.handled_commands)
class Server(object): """ High level view of a node instance. This is the object that should be created to start listening as an active node on the network. """ def __init__(self, node, db, ksize=20, alpha=3, storage=None): """ Create a server instance. This will start listening on the given port. Args: node: The node instance for this peer. It must contain (at minimum) an ID, public key, ip address, and port. ksize (int): The k parameter from the paper alpha (int): The alpha parameter from the paper storage: An instance that implements :interface:`~dht.storage.IStorage` """ self.ksize = ksize self.alpha = alpha self.log = Logger(system=self) self.storage = storage or ForgetfulStorage() self.node = node self.protocol = KademliaProtocol(self.node, self.storage, ksize, db) self.refreshLoop = LoopingCall(self.refreshTable).start(3600) def listen(self, port): """ Start listening on the given port. This is the same as calling:: reactor.listenUDP(port, server.protocol) """ return reactor.listenUDP(port, self.protocol) def refreshTable(self): """ Refresh buckets that haven't had any lookups in the last hour (per section 2.3 of the paper). """ ds = [] for rid in self.protocol.getRefreshIDs(): node = Node(rid) nearest = self.protocol.router.findNeighbors(node, self.alpha) spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha) ds.append(spider.find()) def republishKeys(_): ds = [] # Republish keys older than one hour for keyword in self.storage.iterkeys(): for k, v in self.storage.iteritems(keyword): if self.storage.get_ttl(keyword, k) < 601200: ds.append(self.set(keyword, k, v)) return defer.gatherResults(ds).addCallback(republishKeys) def querySeed(self, seed, pubkey): """ Query an HTTP seed and return a `list` if (ip, port) `tuple` pairs. Args: seed: A `string` consisting of "ip:port" or "hostname:port" pubkey: The hex encoded public key to verify the signature on the response """ try: nodes = [] c = httplib.HTTPConnection(seed) c.request("GET", "/") response = c.getresponse() self.log.info("Https response from %s: %s, %s" % (seed, response.status, response.reason)) data = response.read() reread_data = data.decode("zlib") proto = peers.PeerSeeds() proto.ParseFromString(reread_data) for peer in proto.peer_data: p = peers.PeerData() p.ParseFromString(peer) tup = (str(p.ip_address), p.port) nodes.append(tup) verify_key = nacl.signing.VerifyKey(pubkey, encoder=nacl.encoding.HexEncoder) verify_key.verify("".join(proto.peer_data), proto.signature) return nodes except Exception, e: self.log.error("Failed to query seed: %s" % str(e))
class RPCProtocol(): """ This is an abstract class for processing and sending rpc messages. A class that implements the `MessageProcessor` interface probably should extend this as it does most of the work of keeping track of messages. """ __metaclass__ = abc.ABCMeta def __init__(self, proto, router, waitTimeout=5, noisy=True): """ Args: proto: A protobuf `Node` object containing info about this node. router: A `RoutingTable` object from dht.routing. Implies a `network.Server` object must be started first. waitTimeout: Consider it a connetion failure if no response within this time window. noisy: Whether or not to log the output for this class. """ self.proto = proto self.router = router self._waitTimeout = waitTimeout self._outstanding = {} self.noisy = noisy self.log = Logger(system=self) def receive_message(self, datagram, connection): m = Message() try: m.ParseFromString(datagram) sender = node.Node(m.sender.guid, connection.dest_addr[0], connection.dest_addr[1], m.sender.signedPublicKey, m.sender.vendor) except: # If message isn't formatted property then ignore self.log.msg("Received unknown message from %s, ignoring" % str(connection.dest_addr)) return False # Check that the GUID is valid. If not, ignore if self.router.isNewNode(sender): try: pubkey = m.sender.signedPublicKey[len(m.sender.signedPublicKey) - 32:] verify_key = nacl.signing.VerifyKey(pubkey) verify_key.verify(m.sender.signedPublicKey) h = nacl.hash.sha512(m.sender.signedPublicKey) pow = h[64:128] if int(pow[:6], 16) >= 50 or hexlify(m.sender.guid) != h[:40]: raise Exception('Invalid GUID') except: self.log.msg("Received message from sender with invalid GUID, ignoring") return False msgID = m.messageID data = tuple(m.arguments) if msgID in self._outstanding: self._acceptResponse(msgID, data, sender) else: self._acceptRequest(msgID, str(Command.Name(m.command)).lower(), data, sender, connection) def _acceptResponse(self, msgID, data, sender): msgargs = (b64encode(msgID), sender) if self.noisy: self.log.msg("Received response for message id %s from %s" % msgargs) d, timeout = self._outstanding[msgID] timeout.cancel() d.callback((True, data)) del self._outstanding[msgID] def _acceptRequest(self, msgID, funcname, args, sender, connection): if self.noisy: self.log.msg("received request from %s, command %s" % (sender, funcname.upper())) f = getattr(self, "rpc_%s" % funcname, None) if f is None or not callable(f): msgargs = (self.__class__.__name__, funcname) self.log.error("%s has no callable method rpc_%s; ignoring request" % msgargs) return False d = defer.maybeDeferred(f, sender, *args) d.addCallback(self._sendResponse, funcname, msgID, sender, connection) def _sendResponse(self, response, funcname, msgID, sender, connection): if self.noisy: self.log.msg("sending response for msg id %s to %s" % (b64encode(msgID), sender)) m = Message() m.messageID = msgID m.sender.MergeFrom(self.proto) m.command = Command.Value(funcname.upper()) for arg in response: m.arguments.append(arg) data = m.SerializeToString() connection.send_message(data) def _timeout(self, msgID): args = (b64encode(msgID), self._waitTimeout) self.log.error("Did not received reply for msg id %s within %i seconds" % args) self._outstanding[msgID][0].callback((False, None)) del self._outstanding[msgID] def __getattr__(self, name): if name.startswith("_") or name.startswith("rpc_"): return object.__getattr__(self, name) try: return object.__getattr__(self, name) except AttributeError: pass def func(address, *args): msgID = sha1(str(random.getrandbits(255))).digest() m = Message() m.messageID = msgID m.sender.MergeFrom(self.proto) m.command = Command.Value(name.upper()) for arg in args: m.arguments.append(arg) data = m.SerializeToString() if self.noisy: self.log.msg("calling remote function %s on %s (msgid %s)" % (name, address, b64encode(msgID))) self.multiplexer.send_message(data, address) d = defer.Deferred() timeout = reactor.callLater(self._waitTimeout, self._timeout, msgID) self._outstanding[msgID] = (d, timeout) return d return func
class Lint: def __init__(self, options): self.log = Logger() if not options.workdir: raise RuntimeError, "Lint: No working directory specified." lintResult = self.__getLintResult(options.workdir) ignoreMessages = [] if hasattr(options, "ignoremessages"): optionType = type(options.ignoremessages).__name__ if optionType == "list": ignoreMessages = options.ignoremessages elif optionType == "str": ignoreMessages = options.ignoremessages.split(",") ignoreClasses = [] if hasattr(options, "ignoreclasses"): optionType = type(options.ignoreclasses).__name__ if optionType == "list": ignoreClasses = options.ignoreclasses elif optionType == "str": ignoreClasses = options.ignoreclasses.split(",") self.lintData = self.__parseLintResult(lintResult, ignoreMessages, ignoreClasses) def __getLintResult(self, workdir): self.log.info("Running lint in directory %s" %workdir) startdir = os.getcwd() os.chdir(workdir) ret,out,err = util.invokePiped("python generate.py lint") if (ret > 0): raise RuntimeError, "Lint run failed. " + err os.chdir(startdir) return out def __parseLintResult(self, text, ignoreMessages=[], ignoreClasses=[]): self.log.info("Lint parsing lint output") self.log.debug("Lint ignoring messages: %s" %repr(ignoreMessages)) self.log.debug("Lint ignoring classes: %s" %repr(ignoreClasses)) log = "" if (isinstance(text,str)): import string log = string.split(text,"\n") else: log = text data = {} for line in log: msgre = re.compile('.*\): (.*)$') msgma = msgre.match(line) msg = None if not msgma: continue msg = msgma.group(1) genericmsg = None member = None hint = None (genericmsg, member, hint) = self.__getMessage(msg) if not genericmsg: self.log.error("Lint.parseLintResult couldn't extract generic message from line:\n" + line) if (genericmsg[len(genericmsg)-3:] == " in"): genericmsg = genericmsg[0:len(genericmsg)-3] if genericmsg in ignoreMessages: continue msgid = genericmsg if (not msgid in data): data[msgid] = [] if (hint[0:2] == "! "): hint = hint[2:] info = {} info['member'] = member if (hint != ""): info['hint'] = hint info['path'] = '' info['line'] = '' pathre = re.compile('^.*([\\\/]source[\\\/].*) \(') pathma = pathre.match(line) if (pathma): info['path'] = pathma.group(1) linecolre = re.compile('.*(\(.*\)).*') linecolma = linecolre.match(line) if (linecolma): info['line'] = linecolma.group(1) ignoreClass = False for cls in ignoreClasses: classPath = cls.replace(".","/") clsre = re.compile("^.*" + classPath + ".*$") clsma = clsre.match(info['path']) if (clsma): ignoreClass = True if not ignoreClass: data[msgid].append(info) del_keys = [] for key, value in data.iteritems(): if (len(value) == 0): del_keys.append(key) for k in del_keys: del data[k] return data def __getMessage(self,fullmsg): genericmsg = None member = None hint = None msgre = re.compile("^([\w\- ]+)'([^\s]*)'([\w ]*)[\. ]*(.*)$") msgrma = msgre.match(fullmsg) if msgrma: genericmsg = msgrma.group(1) + msgrma.group(3) if (genericmsg[len(genericmsg)-1] == " "): genericmsg = genericmsg[:-1] member = msgrma.group(2) hint = msgrma.group(4) return (genericmsg, member, hint) def getResult(self): return self.lintData def getFlatResult(self): flatData = [] data = self.getResult() for message in data: for messageDetails in data[message]: flatMessage = { "message" : message, "member" : messageDetails["member"], "path" : messageDetails["path"], "line" : messageDetails["line"] } flatData.append(flatMessage) return flatData def getResultJson(self): return json.dumps(self.lintData, sort_keys=True, indent=2)
class RPCProtocol: """ This is an abstract class for processing and sending rpc messages. A class that implements the `MessageProcessor` interface probably should extend this as it does most of the work of keeping track of messages. """ __metaclass__ = abc.ABCMeta def __init__(self, sourceNode, router, waitTimeout=15): """ Args: sourceNode: A protobuf `Node` object containing info about this node. router: A `RoutingTable` object from dht.routing. Implies a `network.Server` object must be started first. waitTimeout: Timeout for whole messages. Note the txrudp layer has a per-packet timeout but invalid responses wont trigger it. The waitTimeout on this layer needs to be long enough to allow whole messages (ex. images) to transmit. """ self.sourceNode = sourceNode self.router = router self._waitTimeout = waitTimeout self._outstanding = {} self.log = Logger(system=self) def receive_message(self, message, sender, connection, ban_score): if message.testnet != self.multiplexer.testnet: self.log.warning("received message from %s with incorrect network parameters." % str(connection.dest_addr)) connection.shutdown() return False if message.protoVer < PROTOCOL_VERSION: self.log.warning("received message from %s with incompatible protocol version." % str(connection.dest_addr)) connection.shutdown() return False self.multiplexer.vendors[sender.id] = sender msgID = message.messageID if message.command == NOT_FOUND: data = None else: data = tuple(message.arguments) if msgID in self._outstanding: self._acceptResponse(msgID, data, sender) elif message.command != NOT_FOUND: # ban_score.process_message(message) self._acceptRequest(msgID, str(Command.Name(message.command)).lower(), data, sender, connection) def _acceptResponse(self, msgID, data, sender): if data is not None: msgargs = (b64encode(msgID), sender) self.log.debug("received response for message id %s from %s" % msgargs) else: self.log.warning("received 404 error response from %s" % sender) d = self._outstanding[msgID][0] if self._outstanding[msgID][2].active(): self._outstanding[msgID][2].cancel() d.callback((True, data)) del self._outstanding[msgID] def _acceptRequest(self, msgID, funcname, args, sender, connection): self.log.debug("received request from %s, command %s" % (sender, funcname.upper())) f = getattr(self, "rpc_%s" % funcname, None) if f is None or not callable(f): msgargs = (self.__class__.__name__, funcname) self.log.error("%s has no callable method rpc_%s; ignoring request" % msgargs) return False if funcname == "hole_punch": f(sender, *args) else: d = defer.maybeDeferred(f, sender, *args) d.addCallback(self._sendResponse, funcname, msgID, sender, connection) d.addErrback(self._sendResponse, "bad_request", msgID, sender, connection) def _sendResponse(self, response, funcname, msgID, sender, connection): self.log.debug("sending response for msg id %s to %s" % (b64encode(msgID), sender)) m = Message() m.messageID = msgID m.sender.MergeFrom(self.sourceNode.getProto()) m.protoVer = PROTOCOL_VERSION m.testnet = self.multiplexer.testnet if response is None: m.command = NOT_FOUND else: m.command = Command.Value(funcname.upper()) if not isinstance(response, list): response = [response] for arg in response: m.arguments.append(str(arg)) m.signature = self.signing_key.sign(m.SerializeToString())[:64] connection.send_message(m.SerializeToString()) def timeout(self, node): """ This timeout is called by the txrudp connection handler. We will run through the outstanding messages and callback false on any waiting on this IP address. """ address = (node.ip, node.port) for msgID, val in self._outstanding.items(): if address == val[1]: val[0].callback((False, None)) if self._outstanding[msgID][2].active(): self._outstanding[msgID][2].cancel() del self._outstanding[msgID] self.router.removeContact(node) try: self.multiplexer[address].shutdown() except Exception: pass def rpc_hole_punch(self, sender, ip, port, relay="False"): """ A method for handling an incoming HOLE_PUNCH message. Relay the message to the correct node if it's not for us. Otherwise send a datagram to allow the other node to punch through our NAT. """ if relay == "True": self.log.debug("relaying hole punch packet to %s:%s for %s:%s" % (ip, port, sender.ip, str(sender.port))) self.hole_punch(Node(digest("null"), ip, int(port), nat_type=FULL_CONE), sender.ip, sender.port) else: self.log.debug("punching through NAT for %s:%s" % (ip, port)) # pylint: disable=W0612 for i in range(20): self.multiplexer.send_datagram("", (ip, int(port))) def __getattr__(self, name): if name.startswith("_") or name.startswith("rpc_"): return object.__getattr__(self, name) try: return object.__getattr__(self, name) except AttributeError: pass def func(node, *args): msgID = sha1(str(random.getrandbits(255))).digest() m = Message() m.messageID = msgID m.sender.MergeFrom(self.sourceNode.getProto()) m.command = Command.Value(name.upper()) m.protoVer = PROTOCOL_VERSION for arg in args: m.arguments.append(str(arg)) m.testnet = self.multiplexer.testnet m.signature = self.signing_key.sign(m.SerializeToString())[:64] data = m.SerializeToString() address = (node.ip, node.port) relay_addr = None if node.nat_type == SYMMETRIC or \ (node.nat_type == RESTRICTED and self.sourceNode.nat_type == SYMMETRIC): relay_addr = node.relay_node d = defer.Deferred() if m.command != HOLE_PUNCH: timeout = reactor.callLater(self._waitTimeout, self.timeout, node) self._outstanding[msgID] = [d, address, timeout] self.log.debug("calling remote function %s on %s (msgid %s)" % (name, address, b64encode(msgID))) self.multiplexer.send_message(data, address, relay_addr) if self.multiplexer[address].state != State.CONNECTED and \ node.nat_type == RESTRICTED and \ self.sourceNode.nat_type != SYMMETRIC: self.hole_punch(Node(digest("null"), node.relay_node[0], node.relay_node[1], nat_type=FULL_CONE), address[0], address[1], "True") self.log.debug("sending hole punch message to %s" % address[0] + ":" + str(address[1])) return d return func
class RPCProtocol: """ This is an abstract class for processing and sending rpc messages. A class that implements the `MessageProcessor` interface probably should extend this as it does most of the work of keeping track of messages. """ __metaclass__ = abc.ABCMeta def __init__(self, sourceNode, router, waitTimeout=2.5): """ Args: proto: A protobuf `Node` object containing info about this node. router: A `RoutingTable` object from dht.routing. Implies a `network.Server` object must be started first. waitTimeout: Consider it a connetion failure if no response within this time window. noisy: Whether or not to log the output for this class. testnet: The network parameters to use. """ self.sourceNode = sourceNode self.router = router self._waitTimeout = waitTimeout self._outstanding = {} self.log = Logger(system=self) def receive_message(self, datagram, connection, ban_score): m = Message() try: m.ParseFromString(datagram) sender = node.Node(m.sender.guid, m.sender.ip, m.sender.port, m.sender.signedPublicKey, m.sender.vendor) except Exception: # If message isn't formatted property then ignore self.log.warning("received unknown message from %s, ignoring" % str(connection.dest_addr)) return False if m.testnet != self.multiplexer.testnet: self.log.warning("received message from %s with incorrect network parameters." % str(connection.dest_addr)) connection.shutdown() return False if m.protoVer < PROTOCOL_VERSION: self.log.warning("received message from %s with incompatible protocol version." % str(connection.dest_addr)) connection.shutdown() return False # Check that the GUID is valid. If not, ignore if self.router.isNewNode(sender): try: pubkey = m.sender.signedPublicKey[len(m.sender.signedPublicKey) - 32:] verify_key = nacl.signing.VerifyKey(pubkey) verify_key.verify(m.sender.signedPublicKey) h = nacl.hash.sha512(m.sender.signedPublicKey) pow_hash = h[64:128] if int(pow_hash[:6], 16) >= 50 or hexlify(m.sender.guid) != h[:40]: raise Exception('Invalid GUID') except Exception: self.log.warning("received message from sender with invalid GUID, ignoring") connection.shutdown() return False if m.sender.vendor: self.db.VendorStore().save_vendor(m.sender.guid.encode("hex"), m.sender.ip, m.sender.port, m.sender.signedPublicKey) msgID = m.messageID if m.command == NOT_FOUND: data = None else: data = tuple(m.arguments) if msgID in self._outstanding: self._acceptResponse(msgID, data, sender) elif m.command != NOT_FOUND: #ban_score.process_message(m) self._acceptRequest(msgID, str(Command.Name(m.command)).lower(), data, sender, connection) def _acceptResponse(self, msgID, data, sender): if data is not None: msgargs = (b64encode(msgID), sender) self.log.debug("received response for message id %s from %s" % msgargs) else: self.log.warning("received 404 error response from %s" % sender) d = self._outstanding[msgID][0] if self._outstanding[msgID][2].active(): self._outstanding[msgID][2].cancel() d.callback((True, data)) del self._outstanding[msgID] def _acceptRequest(self, msgID, funcname, args, sender, connection): self.log.debug("received request from %s, command %s" % (sender, funcname.upper())) f = getattr(self, "rpc_%s" % funcname, None) if f is None or not callable(f): msgargs = (self.__class__.__name__, funcname) self.log.error("%s has no callable method rpc_%s; ignoring request" % msgargs) return False if funcname == "hole_punch": f(sender, *args) else: d = defer.maybeDeferred(f, sender, *args) d.addCallback(self._sendResponse, funcname, msgID, sender, connection) d.addErrback(self._sendResponse, "bad_request", msgID, sender, connection) def _sendResponse(self, response, funcname, msgID, sender, connection): self.log.debug("sending response for msg id %s to %s" % (b64encode(msgID), sender)) m = Message() m.messageID = msgID m.sender.MergeFrom(self.sourceNode.getProto()) m.protoVer = PROTOCOL_VERSION m.testnet = self.multiplexer.testnet if response is None: m.command = NOT_FOUND else: m.command = Command.Value(funcname.upper()) if not isinstance(response, list): response = [response] for arg in response: m.arguments.append(str(arg)) connection.send_message(m.SerializeToString()) def timeout(self, address): """ This timeout is called by the txrudp connection handler. We will run through the outstanding messages and callback false on any waiting on this IP address. """ for msgID, val in self._outstanding.items(): if address == val[1]: val[0].callback((False, None)) del self._outstanding[msgID] try: node_to_remove = self.multiplexer[address].handler.node if node_to_remove is not None: self.router.removeContact(node_to_remove) self.multiplexer[address].shutdown() except Exception: pass def rpc_hole_punch(self, sender, ip, port, relay="False"): """ A method for handling an incoming HOLE_PUNCH message. Relay the message to the correct node if it's not for us. Otherwise send a datagram to allow the other node to punch through our NAT. """ if relay == "True": self.hole_punch((ip, int(port)), sender.ip, sender.port) else: self.log.debug("punching through NAT for %s:%s" % (ip, port)) # pylint: disable=W0612 for i in range(20): self.multiplexer.send_datagram("", (ip, int(port))) def __getattr__(self, name): if name.startswith("_") or name.startswith("rpc_"): return object.__getattr__(self, name) try: return object.__getattr__(self, name) except AttributeError: pass def func(address, *args): msgID = sha1(str(random.getrandbits(255))).digest() d = defer.Deferred() if name != "hole_punch": seed = SEED_NODE_TESTNET if self.multiplexer.testnet else SEED_NODE if address in self.multiplexer and self.multiplexer[address].state == State.CONNECTED: timeout = timeout = reactor.callLater(self._waitTimeout, self.timeout, address) else: timeout = reactor.callLater(self._waitTimeout, self.hole_punch, seed, address[0], address[1], "True", msgID) self._outstanding[msgID] = [d, address, timeout] self.log.debug("calling remote function %s on %s (msgid %s)" % (name, address, b64encode(msgID))) elif args[3] in self._outstanding: prev_msgID = args[3] args = args[:3] deferred, addr, hp = self._outstanding[prev_msgID] # pylint: disable=W0612 timeout = reactor.callLater(3, self.timeout, addr) self._outstanding[prev_msgID] = [deferred, addr, timeout] self.log.debug("sending hole punch message to %s" % args[0] + ":" + str(args[1])) m = Message() m.messageID = msgID m.sender.MergeFrom(self.sourceNode.getProto()) m.command = Command.Value(name.upper()) m.protoVer = PROTOCOL_VERSION for arg in args: m.arguments.append(str(arg)) m.testnet = self.multiplexer.testnet data = m.SerializeToString() self.multiplexer.send_message(data, address) return d return func
class Server(object): """ High level view of a node instance. This is the object that should be created to start listening as an active node on the network. """ def __init__(self, node, ksize=20, alpha=3, storage=None): """ Create a server instance. This will start listening on the given port. Args: node: The node instance for this peer. It must contain (at minimum) an ID, public key, ip address, and port. ksize (int): The k parameter from the paper alpha (int): The alpha parameter from the paper storage: An instance that implements :interface:`~dht.storage.IStorage` """ self.ksize = ksize self.alpha = alpha self.log = Logger(system=self) self.storage = storage or ForgetfulStorage() self.node = node self.protocol = KademliaProtocol(self.node, self.storage, ksize) self.refreshLoop = LoopingCall(self.refreshTable).start(3600) def listen(self, port): """ Start listening on the given port. This is the same as calling:: reactor.listenUDP(port, server.protocol) """ return reactor.listenUDP(port, self.protocol) def refreshTable(self): """ Refresh buckets that haven't had any lookups in the last hour (per section 2.3 of the paper). """ ds = [] for id in self.protocol.getRefreshIDs(): node = Node(id) nearest = self.protocol.router.findNeighbors(node, self.alpha) spider = NodeSpiderCrawl(self.protocol, node, nearest) ds.append(spider.find()) def republishKeys(_): ds = [] # Republish keys older than one hour for keyword in self.storage.iterkeys(): for k, v in self.storage.iteritems(keyword): if self.storage.get_ttl(keyword, k) < 601200: ds.append(self.set(keyword, k, v)) return defer.gatherResults(ds).addCallback(republishKeys) def querySeed(self, seed, pubkey): """ Query an HTTP seed and return a `list` if (ip, port) `tuple` pairs. Args: seed: A `string` consisting of "ip:port" or "hostname:port" pubkey: The hex encoded public key to verify the signature on the response """ nodes = [] c = httplib.HTTPConnection(seed) c.request("GET", "/") response = c.getresponse() self.log.info("Https response from %s: %s, %s" % (seed, response.status, response.reason)) data = response.read() reread_data = data.decode("zlib") seeds = peers.PeerSeeds() try: seeds.ParseFromString(reread_data) for peer in seeds.peer_data: p = peers.PeerData() p.ParseFromString(peer) tup = (str(p.ip_address), p.port) nodes.append(tup) verify_key = nacl.signing.VerifyKey(pubkey, encoder=nacl.encoding.HexEncoder) verify_key.verify(seed.signature + "".join(seeds.peer_data)) except: self.log.error("Error parsing seed response.") return nodes def bootstrappableNeighbors(self): """ Get a :class:`list` of (ip, port) :class:`tuple` pairs suitable for use as an argument to the bootstrap method. The server should have been bootstrapped already - this is just a utility for getting some neighbors and then storing them if this server is going down for a while. When it comes back up, the list of nodes can be used to bootstrap. """ neighbors = self.protocol.router.findNeighbors(self.node) return [tuple(n)[-2:] for n in neighbors] def bootstrap(self, addrs): """ Bootstrap the server by connecting to other known nodes in the network. Args: addrs: A `list` of (ip, port) `tuple` pairs. Note that only IP addresses are acceptable - hostnames will cause an error. """ # if the transport hasn't been initialized yet, wait a second if self.protocol.multiplexer.transport is None: return task.deferLater(reactor, 1, self.bootstrap, addrs) def initTable(results): nodes = [] for addr, result in results.items(): if result[0]: n = objects.Node() try: n.ParseFromString(result[1][0]) pubkey = n.signedPublicKey[len(n.signedPublicKey) - 32:] verify_key = nacl.signing.VerifyKey(pubkey) verify_key.verify(n.signedPublicKey) h = nacl.hash.sha512(n.signedPublicKey) pow = h[64:128] if int(pow[:6], 16) >= 50 or hexlify(n.guid) != h[:40]: raise Exception('Invalid GUID') nodes.append(Node(n.guid, addr[0], addr[1], n.signedPublicKey)) except: self.log.msg("Bootstrap node returned invalid GUID") spider = NodeSpiderCrawl(self.protocol, self.node, nodes, self.ksize, self.alpha) return spider.find() ds = {} for addr in addrs: ds[addr] = self.protocol.ping((addr[0], addr[1])) return deferredDict(ds).addCallback(initTable) def inetVisibleIP(self): """ Get the internet visible IP's of this node as other nodes see it. Returns: A `list` of IP's. If no one can be contacted, then the `list` will be empty. """ def handle(results): ips = [] for result in results: if result[0]: ips.append((result[1][0], int(result[1][1]))) self.log.debug("other nodes think our ip is %s" % str(ips)) return ips ds = [] for neighbor in self.bootstrappableNeighbors(): ds.append(self.protocol.stun(neighbor)) return defer.gatherResults(ds).addCallback(handle) def get(self, keyword): """ Get a key if the network has it. Returns: :class:`None` if not found, the value otherwise. """ dkey = digest(keyword) if self.storage.get(dkey) is not None: return defer.succeed(self.storage.get(dkey)) node = Node(dkey) nearest = self.protocol.router.findNeighbors(node) if len(nearest) == 0: self.log.warning("There are no known neighbors to get key %s" % keyword) return None spider = ValueSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha) return spider.find() def set(self, keyword, key, value): """ Set the given key/value tuple at the hash of the given keyword. All values stored in the DHT are stored as dictionaries of key/value pairs. If a value already exists for a given keyword, the new key/value pair will be appended to the dictionary. Args: keyword: a `string` keyword. The SHA1 hash of which will be used as the key when inserting in the DHT. key: the 20 byte hash of the data. value: a serialized `protos.objects.Node` object which serves as a pointer to the node storing the data. Return: True if at least one peer responded. False if the store rpc completely failed. """ self.log.debug("setting '%s' = '%s':'%s' on network" % (keyword, hexlify(key), hexlify(value))) dkey = digest(keyword) def store(nodes): self.log.info("setting '%s' on %s" % (keyword, map(str, nodes))) ds = [self.protocol.callStore(node, dkey, key, value) for node in nodes] keynode = Node(dkey) if self.node.distanceTo(keynode) < max([n.distanceTo(keynode) for n in nodes]): self.storage[dkey] = (key, value) self.log.debug("got a store request from %s, storing value" % str(self.node)) return defer.DeferredList(ds).addCallback(self._anyRespondSuccess) node = Node(dkey) nearest = self.protocol.router.findNeighbors(node) if len(nearest) == 0: self.log.warning("There are no known neighbors to set key %s" % key) return defer.succeed(False) spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha) return spider.find().addCallback(store) def delete(self, keyword, key, signature): """ Delete the given key/value pair from the keyword dictionary on the network. To delete you must provide a signature covering the key that you wish to delete. It will be verified against the public key stored in the value. We use our ksize as alpha to make sure we reach as many nodes storing our value as possible. Args: keyword: the `string` keyword where the data being deleted is stored. key: the 20 byte hash of the data. signature: a signature covering the key. """ self.log.debug("deleting '%s':'%s' from the network" % (keyword, hexlify(key))) dkey = digest(keyword) def delete(nodes): self.log.info("deleting '%s' on %s" % (key, map(str, nodes))) ds = [self.protocol.callDelete(node, dkey, key, signature) for node in nodes] if self.storage.getSpecific(keyword, key) is not None: self.storage.delete(keyword, key) return defer.DeferredList(ds).addCallback(self._anyRespondSuccess) node = Node(dkey) nearest = self.protocol.router.findNeighbors(node) if len(nearest) == 0: self.log.warning("There are no known neighbors to delete key %s" % key) return defer.succeed(False) spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.ksize) return spider.find().addCallback(delete) def get_node(self, guid): """ Given a guid return a `Node` object containing its ip and port or none if it's not found. Args: guid: the 20 raw bytes representing the guid. """ node_to_find = Node(guid) def check_for_node(nodes): for node in nodes: if node.id == node_to_find.id: return node return None index = self.protocol.router.getBucketFor(node_to_find) nodes = self.protocol.router.buckets[index].getNodes() for node in nodes: if node.id == node_to_find.id: return defer.succeed(node) nearest = self.protocol.router.findNeighbors(node_to_find) if len(nearest) == 0: self.log.warning("There are no known neighbors to find node %s" % node_to_find.id.encode("hex")) return defer.succeed(None) spider = NodeSpiderCrawl(self.protocol, node_to_find, nearest, self.ksize, self.alpha) return spider.find().addCallback(check_for_node) def _anyRespondSuccess(self, responses): """ Given the result of a DeferredList of calls to peers, ensure that at least one of them was contacted and responded with a Truthy result. """ for deferSuccess, result in responses: peerReached, peerResponse = result if deferSuccess and peerReached and peerResponse: return True return False def saveState(self, fname): """ Save the state of this node (the alpha/ksize/id/immediate neighbors) to a cache file with the given fname. """ data = {'ksize': self.ksize, 'alpha': self.alpha, 'id': self.node.id, 'signed_pubkey': self.node.signed_pubkey, 'neighbors': self.bootstrappableNeighbors()} if len(data['neighbors']) == 0: self.log.warning("No known neighbors, so not writing to cache.") return with open(fname, 'w') as f: pickle.dump(data, f) @classmethod def loadState(self, fname, ip_address, port, multiplexer, storage=None): """ Load the state of this node (the alpha/ksize/id/immediate neighbors) from a cache file with the given fname. """ with open(fname, 'r') as f: data = pickle.load(f) n = Node(data['id'], ip_address, port, data['signed_pubkey']) s = Server(n, data['ksize'], data['alpha'], storage=storage) s.protocol.connect_multiplexer(multiplexer) if len(data['neighbors']) > 0: s.bootstrap(data['neighbors']) return s def saveStateRegularly(self, fname, frequency=600): """ Save the state of node with a given regularity to the given filename. Args: fname: File name to save retularly to frequencey: Frequency in seconds that the state should be saved. By default, 10 minutes. """ loop = LoopingCall(self.saveState, fname) loop.start(frequency) return loop
class Server(object): """ High level view of a node instance. This is the object that should be created to start listening as an active node on the network. """ def __init__(self, node, db, signing_key, ksize=20, alpha=3, storage=None): """ Create a server instance. This will start listening on the given port. Args: node: The node instance for this peer. It must contain (at minimum) an ID, public key, ip address, and port. ksize (int): The k parameter from the paper alpha (int): The alpha parameter from the paper storage: An instance that implements :interface:`~dht.storage.IStorage` """ self.ksize = ksize self.alpha = alpha self.log = Logger(system=self) self.storage = storage or ForgetfulStorage() self.node = node self.protocol = KademliaProtocol(self.node, self.storage, ksize, db, signing_key) self.refreshLoop = LoopingCall(self.refreshTable) reactor.callLater(1800, self.refreshLoop.start, 3600) def listen(self, port): """ Start listening on the given port. This is the same as calling:: reactor.listenUDP(port, server.protocol) """ return reactor.listenUDP(port, self.protocol) def refreshTable(self): """ Refresh buckets that haven't had any lookups in the last hour (per section 2.3 of the paper). """ ds = [] refresh_ids = self.protocol.getRefreshIDs() refresh_ids.append(digest(random.getrandbits(255))) # random node so we get more diversity for rid in refresh_ids: node = Node(rid) nearest = self.protocol.router.findNeighbors(node, self.alpha) spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha) ds.append(spider.find()) def republishKeys(_): self.log.debug("Republishing key/values...") neighbors = self.protocol.router.findNeighbors(self.node, exclude=self.node) for node in neighbors: self.protocol.transferKeyValues(node) return defer.gatherResults(ds).addCallback(republishKeys) def querySeed(self, list_seed_pubkey): """ Query an HTTP seed and return a `list` if (ip, port) `tuple` pairs. Args: Receives a list of one or more tuples Example [(seed, pubkey)] seed: A `string` consisting of "ip:port" or "hostname:port" pubkey: The hex encoded public key to verify the signature on the response """ nodes = [] if not list_seed_pubkey: self.log.error('failed to query seed {0} from ob.cfg'.format(list_seed_pubkey)) return nodes else: for sp in list_seed_pubkey: seed, pubkey = sp try: self.log.info("querying %s for peers" % seed) c = httplib.HTTPConnection(seed) c.request("GET", "/") response = c.getresponse() self.log.debug("Http response from %s: %s, %s" % (seed, response.status, response.reason)) data = response.read() reread_data = data.decode("zlib") proto = peers.PeerSeeds() proto.ParseFromString(reread_data) for peer in proto.serializedNode: n = objects.Node() n.ParseFromString(peer) tup = (str(n.nodeAddress.ip), n.nodeAddress.port) nodes.append(tup) verify_key = nacl.signing.VerifyKey(pubkey, encoder=nacl.encoding.HexEncoder) verify_key.verify("".join(proto.serializedNode), proto.signature) self.log.info("%s returned %s addresses" % (seed, len(nodes))) except Exception, e: self.log.error("failed to query seed: %s" % str(e)) return nodes
class MarketProtocol(RPCProtocol): implements(MessageProcessor) def __init__(self, node, router, signing_key, database): self.router = router self.node = node RPCProtocol.__init__(self, node, router) self.log = Logger(system=self) self.multiplexer = None self.db = database self.signing_key = signing_key self.listeners = [] self.handled_commands = [GET_CONTRACT, GET_IMAGE, GET_PROFILE, GET_LISTINGS, GET_USER_METADATA, GET_CONTRACT_METADATA, FOLLOW, UNFOLLOW, GET_FOLLOWERS, GET_FOLLOWING, BROADCAST, MESSAGE, ORDER, ORDER_CONFIRMATION, COMPLETE_ORDER, DISPUTE_OPEN, DISPUTE_CLOSE] def connect_multiplexer(self, multiplexer): self.multiplexer = multiplexer def add_listener(self, listener): self.listeners.append(listener) def rpc_get_contract(self, sender, contract_hash): self.log.info("serving contract %s to %s" % (contract_hash.encode('hex'), sender)) self.router.addContact(sender) try: with open(self.db.HashMap().get_file(contract_hash.encode("hex")), "r") as filename: contract = filename.read() return [contract] except Exception: self.log.warning("could not find contract %s" % contract_hash.encode('hex')) return None def rpc_get_image(self, sender, image_hash): self.router.addContact(sender) try: if len(image_hash) != 20: raise Exception("Invalid image hash") self.log.info("serving image %s to %s" % (image_hash.encode('hex'), sender)) with open(self.db.HashMap().get_file(image_hash.encode("hex")), "rb") as filename: image = filename.read() return [image] except Exception: self.log.warning("could not find image %s" % image_hash[:20].encode('hex')) return None def rpc_get_profile(self, sender): self.log.info("serving profile to %s" % sender) self.router.addContact(sender) try: proto = Profile(self.db).get(True) return [proto, self.signing_key.sign(proto)[:64]] except Exception: self.log.error("unable to load the profile") return None def rpc_get_user_metadata(self, sender): self.log.info("serving user metadata to %s" % sender) self.router.addContact(sender) try: proto = Profile(self.db).get(False) m = Metadata() m.name = proto.name m.handle = proto.handle m.short_description = proto.short_description m.avatar_hash = proto.avatar_hash m.nsfw = proto.nsfw return [m.SerializeToString(), self.signing_key.sign(m.SerializeToString())[:64]] except Exception: self.log.error("unable to load profile metadata") return None def rpc_get_listings(self, sender): self.log.info("serving store listings to %s" % sender) self.router.addContact(sender) try: p = Profile(self.db).get() l = Listings() l.ParseFromString(self.db.ListingsStore().get_proto()) l.handle = p.handle l.avatar_hash = p.avatar_hash return [l.SerializeToString(), self.signing_key.sign(l.SerializeToString())[:64]] except Exception: self.log.warning("could not find any listings in the database") return None def rpc_get_contract_metadata(self, sender, contract_hash): self.log.info("serving metadata for contract %s to %s" % (contract_hash.encode("hex"), sender)) self.router.addContact(sender) try: proto = self.db.ListingsStore().get_proto() p = Profile(self.db).get() l = Listings() l.ParseFromString(proto) for listing in l.listing: if listing.contract_hash == contract_hash: listing.avatar_hash = p.avatar_hash listing.handle = p.handle ser = listing.SerializeToString() return [ser, self.signing_key.sign(ser)[:64]] except Exception: self.log.warning("could not find metadata for contract %s" % contract_hash.encode("hex")) return None def rpc_follow(self, sender, proto, signature): self.log.info("received follow request from %s" % sender) self.router.addContact(sender) try: verify_key = nacl.signing.VerifyKey(sender.signed_pubkey[64:]) verify_key.verify(proto, signature) f = Followers.Follower() f.ParseFromString(proto) if f.guid != sender.id: raise Exception('GUID does not match sending node') if f.following != self.node.id: raise Exception('Following wrong node') f.signature = signature self.db.FollowData().set_follower(f) proto = Profile(self.db).get(False) m = Metadata() m.name = proto.name m.handle = proto.handle m.avatar_hash = proto.avatar_hash m.short_description = proto.short_description m.nsfw = proto.nsfw for listener in self.listeners: try: verifyObject(NotificationListener, listener) listener.notify(sender.id, f.metadata.handle, "follow", "", "", f.metadata.avatar_hash) except DoesNotImplement: pass return ["True", m.SerializeToString(), self.signing_key.sign(m.SerializeToString())[:64]] except Exception: self.log.warning("failed to validate follower") return ["False"] def rpc_unfollow(self, sender, signature): self.log.info("received unfollow request from %s" % sender) self.router.addContact(sender) try: verify_key = nacl.signing.VerifyKey(sender.signed_pubkey[64:]) verify_key.verify("unfollow:" + self.node.id, signature) f = self.db.FollowData() f.delete_follower(sender.id) return ["True"] except Exception: self.log.warning("failed to validate signature on unfollow request") return ["False"] def rpc_get_followers(self, sender): self.log.info("serving followers list to %s" % sender) self.router.addContact(sender) ser = self.db.FollowData().get_followers() if ser is None: return None else: return [ser, self.signing_key.sign(ser)[:64]] def rpc_get_following(self, sender): self.log.info("serving following list to %s" % sender) self.router.addContact(sender) ser = self.db.FollowData().get_following() if ser is None: return None else: return [ser, self.signing_key.sign(ser)[:64]] def rpc_broadcast(self, sender, message, signature): if len(message) <= 140 and self.db.FollowData().is_following(sender.id): try: verify_key = nacl.signing.VerifyKey(sender.signed_pubkey[64:]) verify_key.verify(message, signature) except Exception: self.log.warning("received invalid broadcast from %s" % sender) return ["False"] self.log.info("received a broadcast from %s" % sender) self.router.addContact(sender) for listener in self.listeners: try: verifyObject(BroadcastListener, listener) listener.notify(sender.id, message) except DoesNotImplement: pass return ["True"] else: return ["False"] def rpc_message(self, sender, pubkey, encrypted): try: box = Box(PrivateKey(self.signing_key.encode(nacl.encoding.RawEncoder)), PublicKey(pubkey)) plaintext = box.decrypt(encrypted) p = PlaintextMessage() p.ParseFromString(plaintext) signature = p.signature p.ClearField("signature") verify_key = nacl.signing.VerifyKey(p.signed_pubkey[64:]) verify_key.verify(p.SerializeToString(), signature) h = nacl.hash.sha512(p.signed_pubkey) pow_hash = h[64:128] if int(pow_hash[:6], 16) >= 50 or p.sender_guid.encode("hex") != h[:40] or p.sender_guid != sender.id: raise Exception('Invalid guid') self.log.info("received a message from %s" % sender) self.router.addContact(sender) for listener in self.listeners: try: verifyObject(MessageListener, listener) listener.notify(p, signature) except DoesNotImplement: pass return ["True"] except Exception: self.log.warning("received invalid message from %s" % sender) return ["False"] def rpc_order(self, sender, pubkey, encrypted): try: box = Box(PrivateKey(self.signing_key.encode(nacl.encoding.RawEncoder)), PublicKey(pubkey)) order = box.decrypt(encrypted) c = Contract(self.db, contract=json.loads(order, object_pairs_hook=OrderedDict), testnet=self.multiplexer.testnet) if c.verify(sender.signed_pubkey[64:]): self.router.addContact(sender) self.log.info("received an order from %s, waiting for payment..." % sender) payment_address = c.contract["buyer_order"]["order"]["payment"]["address"] chaincode = c.contract["buyer_order"]["order"]["payment"]["chaincode"] masterkey_b = c.contract["buyer_order"]["order"]["id"]["pubkeys"]["bitcoin"] buyer_key = derive_childkey(masterkey_b, chaincode) amount = c.contract["buyer_order"]["order"]["payment"]["amount"] listing_hash = c.contract["buyer_order"]["order"]["ref_hash"] signature = self.signing_key.sign( str(payment_address) + str(amount) + str(listing_hash) + str(buyer_key))[:64] c.await_funding(self.get_notification_listener(), self.multiplexer.blockchain, signature, False) return [signature] else: self.log.warning("received invalid order from %s" % sender) return ["False"] except Exception: self.log.error("unable to decrypt order from %s" % sender) return ["False"] def rpc_order_confirmation(self, sender, pubkey, encrypted): try: box = Box(PrivateKey(self.signing_key.encode(nacl.encoding.RawEncoder)), PublicKey(pubkey)) order = box.decrypt(encrypted) c = Contract(self.db, contract=json.loads(order, object_pairs_hook=OrderedDict), testnet=self.multiplexer.testnet) contract_id = c.accept_order_confirmation(self.get_notification_listener()) if contract_id: self.router.addContact(sender) self.log.info("received confirmation for order %s" % contract_id) return ["True"] else: self.log.warning("received invalid order confirmation from %s" % sender) return ["False"] except Exception: self.log.error("unable to decrypt order confirmation from %s" % sender) return ["False"] def rpc_complete_order(self, sender, pubkey, encrypted): try: box = Box(PrivateKey(self.signing_key.encode(nacl.encoding.RawEncoder)), PublicKey(pubkey)) order = box.decrypt(encrypted) c = Contract(self.db, contract=json.loads(order, object_pairs_hook=OrderedDict), testnet=self.multiplexer.testnet) contract_id = c.accept_receipt(self.get_notification_listener(), self.multiplexer.blockchain) self.router.addContact(sender) self.log.info("received receipt for order %s" % contract_id) return ["True"] except Exception: self.log.error("unable to parse receipt from %s" % sender) return ["False"] def rpc_dispute_open(self, sender, pubkey, encrypted): try: box = Box(PrivateKey(self.signing_key.encode(nacl.encoding.RawEncoder)), PublicKey(pubkey)) order = box.decrypt(encrypted) contract = json.loads(order, object_pairs_hook=OrderedDict) process_dispute(contract, self.db, self.get_message_listener(), self.get_notification_listener(), self.multiplexer.testnet) self.router.addContact(sender) self.log.info("Contract dispute opened by %s" % sender) return ["True"] except Exception: self.log.error("unable to parse disputed contract from %s" % sender) return ["False"] def rpc_dispute_close(self, sender, pubkey, encrypted): try: box = Box(PrivateKey(self.signing_key.encode(nacl.encoding.RawEncoder)), PublicKey(pubkey)) order = box.decrypt(encrypted) contract = json.loads(order, object_pairs_hook=OrderedDict) close_dispute(contract, self.db, self.get_message_listener(), self.get_notification_listener(), self.multiplexer.testnet) self.router.addContact(sender) self.log.info("Contract dispute closed by %s" % sender) return ["True"] except Exception: self.log.error("unable to parse disputed close message from %s" % sender) return ["False"] def callGetContract(self, nodeToAsk, contract_hash): d = self.get_contract(nodeToAsk, contract_hash) return d.addCallback(self.handleCallResponse, nodeToAsk) def callGetImage(self, nodeToAsk, image_hash): d = self.get_image(nodeToAsk, image_hash) return d.addCallback(self.handleCallResponse, nodeToAsk) def callGetProfile(self, nodeToAsk): d = self.get_profile(nodeToAsk) return d.addCallback(self.handleCallResponse, nodeToAsk) def callGetUserMetadata(self, nodeToAsk): d = self.get_user_metadata(nodeToAsk) return d.addCallback(self.handleCallResponse, nodeToAsk) def callGetListings(self, nodeToAsk): d = self.get_listings(nodeToAsk) return d.addCallback(self.handleCallResponse, nodeToAsk) def callGetContractMetadata(self, nodeToAsk, contract_hash): d = self.get_contract_metadata(nodeToAsk, contract_hash) return d.addCallback(self.handleCallResponse, nodeToAsk) def callFollow(self, nodeToAsk, proto, signature): d = self.follow(nodeToAsk, proto, signature) return d.addCallback(self.handleCallResponse, nodeToAsk) def callUnfollow(self, nodeToAsk, signature): d = self.unfollow(nodeToAsk, signature) return d.addCallback(self.handleCallResponse, nodeToAsk) def callGetFollowers(self, nodeToAsk): d = self.get_followers(nodeToAsk) return d.addCallback(self.handleCallResponse, nodeToAsk) def callGetFollowing(self, nodeToAsk): d = self.get_following(nodeToAsk) return d.addCallback(self.handleCallResponse, nodeToAsk) def callBroadcast(self, nodeToAsk, message, signature): d = self.broadcast(nodeToAsk, message, signature) return d.addCallback(self.handleCallResponse, nodeToAsk) def callMessage(self, nodeToAsk, ehemeral_pubkey, ciphertext): d = self.message(nodeToAsk, ehemeral_pubkey, ciphertext) return d.addCallback(self.handleCallResponse, nodeToAsk) def callOrder(self, nodeToAsk, ephem_pubkey, encrypted_contract): d = self.order(nodeToAsk, ephem_pubkey, encrypted_contract) return d.addCallback(self.handleCallResponse, nodeToAsk) def callOrderConfirmation(self, nodeToAsk, ephem_pubkey, encrypted_contract): d = self.order_confirmation(nodeToAsk, ephem_pubkey, encrypted_contract) return d.addCallback(self.handleCallResponse, nodeToAsk) def callCompleteOrder(self, nodeToAsk, ephem_pubkey, encrypted_contract): d = self.complete_order(nodeToAsk, ephem_pubkey, encrypted_contract) return d.addCallback(self.handleCallResponse, nodeToAsk) def callDisputeOpen(self, nodeToAsk, ephem_pubkey, encrypted_contract): d = self.dispute_open(nodeToAsk, ephem_pubkey, encrypted_contract) return d.addCallback(self.handleCallResponse, nodeToAsk) def callDisputeClose(self, nodeToAsk, ephem_pubkey, encrypted_contract): d = self.dispute_open(nodeToAsk, ephem_pubkey, encrypted_contract) return d.addCallback(self.handleCallResponse, nodeToAsk) def handleCallResponse(self, result, node): """ If we get a response, add the node to the routing table. If we get no response, make sure it's removed from the routing table. """ if result[0]: self.router.addContact(node) else: self.log.debug("no response from %s, removing from router" % node) self.router.removeContact(node) return result def get_notification_listener(self): for listener in self.listeners: try: verifyObject(NotificationListener, listener) return listener except DoesNotImplement: pass def get_message_listener(self): for listener in self.listeners: try: verifyObject(MessageListener, listener) return listener except DoesNotImplement: pass def __iter__(self): return iter(self.handled_commands)
if status == 'success': logger.info("type:%s, target:%s, size:%d, cost:%f, speed:%f" %('upload', target, size, costTime, size/costTime/1000)) speed = 0 if totalTime > 0: speed = totalSize/totalTime/1000 logger.info("type:%s, target:%s, succ:%d, fail:%d, size:%d, cost:%f, speed:%f" %('upload', target, totalSucc, totalFail, totalSize, totalTime, speed)) def cleanUploadedPics(picDir): for file in os.listdir(picDir): os.remove(os.path.join(picDir, file)) logger.debug('Removed pic: %s' %(file)) logger.info('Removed all the pics under dir: %s' %(picDir)) files = getPicFiles(pic_dir) if len(files) <= 0 : logger.error("There's no files under dir[%s]" %(pic_dir)) else: try: markImage.markAll(files) except: logger.error("Mark image error: %s", sys.exc_info()[0]) try: uploadToWeibo(files) except: logger.error("Upload to Weibo error: %s", sys.exc_info()[0]) try: uploadToQQ(files) except: logger.error("Upload to Tencent error: %s", sys.exc_info()[0])