def main(cfg):
if cfg.unix:
if cfg.addr == "":
sys.exit("invalid listener unix domain socket: {}".format(
cfg.addr))
else:
try:
(host, port) = cfg.addr.rsplit(":", 1)
port = int(port)
except ValueError:
sys.exit("invalid listener address: {}".format(cfg.addr))
if cfg.response == "success":
handler = SuccessHandler()
elif cfg.response == "idl-exception":
handler = IDLExceptionHandler()
elif cfg.response == "exception":
# squelch traceback for the exception we throw
logging.getLogger().setLevel(logging.CRITICAL)
handler = ExceptionHandler()
else:
sys.exit("unknown server response mode {0}".format(cfg.response))
processor = Example.Processor(handler)
if cfg.service is not None:
# wrap processor with multiplexor
multi = TMultiplexedProcessor.TMultiplexedProcessor()
multi.registerProcessor(cfg.service, processor)
processor = multi
if cfg.protocol == "finagle":
# wrap processor with finagle request/response header handler
processor = TFinagleServerProcessor.TFinagleServerProcessor(processor)
if cfg.unix:
transport = TSocket.TServerSocket(unix_socket=cfg.addr)
else:
transport = TSocket.TServerSocket(host=host, port=port)
if cfg.transport == "framed":
transport_factory = TTransport.TFramedTransportFactory()
elif cfg.transport == "unframed":
transport_factory = TTransport.TBufferedTransportFactory()
elif cfg.transport == "header":
if cfg.protocol == "binary":
transport_factory = THeaderTransport.THeaderTransportFactory(
THeaderTransport.T_BINARY_PROTOCOL)
elif cfg.protocol == "compact":
transport_factory = THeaderTransport.THeaderTransportFactory(
THeaderTransport.T_COMPACT_PROTOCOL)
else:
sys.exit(
"header transport cannot be used with protocol {0}".format(
cfg.protocol))
else:
sys.exit("unknown transport {0}".format(cfg.transport))
if cfg.protocol == "binary":
protocol_factory = TBinaryProtocol.TBinaryProtocolFactory()
elif cfg.protocol == "compact":
protocol_factory = TCompactProtocol.TCompactProtocolFactory()
elif cfg.protocol == "json":
protocol_factory = TJSONProtocol.TJSONProtocolFactory()
elif cfg.protocol == "finagle":
protocol_factory = TFinagleServerProtocol.TFinagleServerProtocolFactory(
)
else:
sys.exit("unknown protocol {0}".format(cfg.protocol))
print("Thrift Server listening on {0} for {1} {2} requests".format(
cfg.addr, cfg.transport, cfg.protocol))
if cfg.service is not None:
print("Thrift Server service name {0}".format(cfg.service))
if cfg.response == "idl-exception":
print("Thrift Server will throw IDL exceptions when defined")
elif cfg.response == "exception":
print("Thrift Server will throw Thrift exceptions for all messages")
server = TServer.TThreadedServer(processor, transport, transport_factory,
protocol_factory)
try:
server.serve()
except KeyboardInterrupt:
print
class CompactProtocolTest(AbstractTest):
protocol_factory = TCompactProtocol.TCompactProtocolFactory()
logging.info(str(scores))
augf = AnalyticUUIDGeneratorFactory(communication)
aug = augf.create()
lid = LanguageIdentification(uuid=aug.next(),
languageToProbabilityMap=scores,
metadata=AnnotationMetadata(tool="valid", timestamp=int(time.time()), kBest=len(scores)),
)
communication.lidList.append(lid)
return communication
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("-p", "--port", dest="port", type=int, default=9090)
parser.add_argument("-m", "--model_path", dest="model_path")
options = parser.parse_args()
logging.basicConfig(level=logging.ERROR)
handler = CommunicationHandler(options.model_path)
processor = Annotator.Processor(handler)
transport = TSocket.TServerSocket(port=options.port)
ipfactory = TCompactProtocol.TCompactProtocolFactory()
opfactory = TCompactProtocol.TCompactProtocolFactory()
server = TNonblockingServer.TNonblockingServer(processor, transport, ipfactory, opfactory)
logging.info('Starting the server...')
server.serve()
class CompactTest(AbstractTest, unittest.TestCase):
protocol_factory = TCompactProtocol.TCompactProtocolFactory()
def echoHello(self, param):
result = "你发的字符串是：" + param
return result
def uploadFile(self, binaryData, filePath, flag):
import os
if flag == 0:
if os.path.exists(filePath):
os.remove(filePath)
# 以追加模式+二进制模式打开
fp = open(filePath, "ab")
try:
fp.write(binaryData)
finally:
fp.close()
handler = HelloHandler()
processor = Hello.Processor(handler)
socket = TSocket.TServerSocket(host="localhost", port=9000)
factory = TTransport.TFramedTransportFactory()
protocol = TCompactProtocol.TCompactProtocolFactory()
#不支持windows
#server = TServer.TForkingServer(processor, socket, factory, protocol)
server = TServer.TThreadPoolServer(processor, socket, factory, protocol)
print "python start server port: 9000"
server.serve()
def __create_server(processor, ip, port):
from ._gevent import GProcessPoolServer
transport = TSocket.TServerSocket(ip, port)
tfactory = TTransport.TFramedTransportFactory()
pfactory = TCompactProtocol.TCompactProtocolFactory()
return GProcessPoolServer(processor, transport, tfactory, pfactory)
def __create_server(processor, ip, port):
transport = TSocket.TServerSocket(ip, port)
pfactory = TCompactProtocol.TCompactProtocolFactory()
return TNonblockingServer.TNonblockingServer(processor, transport,
pfactory)
def createEntry(name, dat, tarf):
# Question Communication
qtext = data['data']['question'] + "\n\n" + data['data']['details']
comm = createComm(name, "QUORA QUESTION", qtext)
stringf = StringIO(
TSerialization.serialize(
comm, protocol_factory=TCompactProtocol.TCompactProtocolFactory()))
info = tarfile.TarInfo('{}/{}/question.comm'.format(name[2], name))
info.size = len(stringf.buf)
tarf.addfile(info, stringf)
# Question Metadata
qdata = {
"followers":
data['data']['followers'],
"topics":
data['data']['topics'],
"author":
data['log']['author'] if 'log' in data and not data['log'] is None
and 'author' in data['log'] else None,
"postTime":
data['log']['date'] if 'log' in data and not data['log'] is None
and 'date' in data['log'] else None,
"scrapeTime":
data['time'],
"url":
data['url'],
"numAnswers":
len(data['data']['answers']),
"hasDetails":
bool(data['data']['details'].strip()),
}
stringf = StringIO(json.dumps(qdata))
info = tarfile.TarInfo('{}/{}/metadata.json'.format(name[2], name))
info.size = len(stringf.buf)
tarf.addfile(info, stringf)
# Answers
for i, answer in enumerate(data['data']['answers']):
# Communication
comm = createComm(name, "QUORA ANSWER", answer['text'])
if comm is None:
continue
stringf = StringIO(
TSerialization.serialize(
comm,
protocol_factory=TCompactProtocol.TCompactProtocolFactory()))
info = tarfile.TarInfo("{}/{}/answer{}.comm".format(name[2], name, i))
info.size = len(stringf.buf)
tarf.addfile(info, stringf)
# Metadata
adata = {
"upvotes": answer['upvotes'],
"author": answer['author'],
"time": answer['time']
}
stringf = StringIO(json.dumps(adata))
info = tarfile.TarInfo('{}/{}/answer{}.json'.format(name[2], name, i))
info.size = len(stringf.buf)
tarf.addfile(info, stringf)
def testCompactProtocolEof(self):
"""Test that TCompactProtocol throws an EOFError when it reaches the end of the stream"""
self.eofTestHelper(TCompactProtocol.TCompactProtocolFactory())
self.eofTestHelperStress(TCompactProtocol.TCompactProtocolFactory())
class TestForwardCompatibilityCompact(TestForwardCompatibilityAbstract):
protocol_factory = TCompactProtocol.TCompactProtocolFactory()
from pi.ttypes import PiResponse, IllegalArgument
class PiHandler(Iface):
def calc(self, req):
if req.n < 0:
raise IllegalArgument(message='parameter must be positive')
s = 0.0
for i in range(req.n):
s += 1.0 / (2 * i + 1) / (2 * i + 1)
return PiResponse(value=math.sqrt(8 * s))
if __name__ == '__main__':
handler = PiHandler()
processor = Processor(handler)
transport = TSocket.TServerSocket(host='127.0.0.1', port=8080)
tfactory = TTransport.TBufferedTransportFactory() # 缓冲模式
pfactory = TCompactProtocol.TCompactProtocolFactory() # 紧凑模式
# 线程池服务RPC请求
server = TServer.TThreadPoolServer(processor, transport, tfactory,
pfactory)
# 设置线程池数量
server.setNumThreads(10)
# 设置线程为daemon，当进程只剩下daemon线程时会立即退出
server.daemon = True
# 启动服务
server.serve()
def ThriftToCompact(thrift_object,
protocol_factory=TCompactProtocol.TCompactProtocolFactory()):
transport = TTransport.TMemoryBuffer()
protocol = protocol_factory.getProtocol(transport)
thrift_object.write(protocol)
return transport.getvalue()
def main():
arglen = len(sys.argv)
i = 1
#默认端口号
port = 9990
#默认不持久化
store = False
#默认持久化文件
store_file = u"bloom_filter.dat"
#默认错误率
error_rate = 0.001
#默认最大元素数量
element_num = 10000000
try:
while i < arglen:
if sys.argv[i] == "-port" or sys.argv[i] == "-p":
port = int(sys.argv[i + 1])
if port < 1:
raise Exception(u"端口号格式错误")
i += 2
elif sys.argv[i] == "--store":
store = True
i += 1
elif sys.argv[i] == "-file":
store_file = sys.argv[i + 1]
i += 2
elif sys.argv[i] == "--version":
print u"bloom filter server v1.00."
quit(0)
elif sys.argv[i] == "--help":
print u"""\
Usage: python bloomfilterserver.py [options]
布隆过滤器服务器
-port, -p 设置端口号,默认9990
--store 开启持久化功能，默认不开启
-file 持久化文件名，默认bloom_filter_store.data
-error-rate 错误率，默认为0.001
-element-num 元素最大数，默认为10,000,000
--version 查看版本号
--help 查看帮助信息
退出状态：
0 正常
-1 程序出错
-2 参数错误
"""
quit(0)
elif sys.argv[i] == u"-error-rate":
temp_error_rate = float(sys.argv[i + 1])
if 0 < error_rate < 1:
error_rate = temp_error_rate
else:
raise Exception(u"错误率设置错误.")
i += 2
elif sys.argv[i] == u"-element-num":
element_num = int(sys.argv[i + 1])
if element_num < 10000:
raise Exception(u"最大元素数量设置错误。")
i += 2
else:
raise Exception(u"无效的参数:%s" % sys.argv[i])
except Exception as e:
logger.error(u"参数错误,%s." % e.message)
quit(-2)
logger.info(
u"布隆过滤器启动参数,port:%s,store:%s,store_file:%s,element_num:%s,error_rate:%s"
% (port, store, store_file, element_num, error_rate))
handler = BloomFilterServer(error_rate, element_num, store, store_file)
#启动
handler.start()
processor = BloomFilterService.Processor(handler)
hostname = socket.gethostbyname(socket.gethostname())
logger.info(u"service %s running listen on %s:%d" %
(handler.__class__.__name__, hostname, port))
transport = TSocket.TServerSocket(hostname, port)
tfactory = TTransport.TBufferedTransportFactory()
#Protocol
#Protocol用于对数据格式抽象，在rpc调用时序列化请求和响应。
#TProtocol的实现包括：TJSONProtocol，TSimpleJSONProtocol，TBinaryProtocol，TBinaryPotocolAccelerated，TCompactProtocol。
#上面每种类型，都有对应的Factory类，用于创建该类对象。
#pfactory = TBinaryProtocol.TBinaryProtocolFactory()
pfactory = TCompactProtocol.TCompactProtocolFactory()
#server = TServer.TSimpleServer(processor, transport, tfactory, pfactory)
server = TServer.TThreadPoolServer(processor, transport, tfactory,
pfactory)
server.setNumThreads(2048)
logger.info(u"BloomFilter server Starting!")
server.serve()
logger.infou(u"BloomFilter server done!")
elif len(sys.argv) == 2:
input_data = file(sys.argv[1])
else:
print >> sys.stderr, "Usage: %s [file]" % (sys.argv[0], )
sys.exit(1)
for line in input_data:
space_separated = line.split(" ")
if len(space_separated) == 3:
ts = int(space_separated[0])
print datetime.datetime.fromtimestamp(
ts / 1000.0).isoformat(), space_separated[1],
base64_encoded = space_separated[2]
elif len(space_separated) == 1:
base64_encoded = space_separated[0]
else:
raise Exception("Unexpected line: " + line)
possibly_compressed = base64.b64decode(base64_encoded)
# Handle both compressed and uncompressed Thrift profiles
try:
thrift = zlib.decompress(possibly_compressed)
except zlib.error:
thrift = possibly_compressed
tree = TRuntimeProfileTree()
deserialize(tree,
thrift,
protocol_factory=TCompactProtocol.TCompactProtocolFactory())
tree.validate()
print tree
