def run(self):
# setup communication
self._in_trans = TTransport.TFileObjectTransport(
open(self._in_pipe, 'rb'))
self._out_trans = TTransport.TFileObjectTransport(
open(self._out_pipe, 'wb'))
self._ctrl_trans = TTransport.TFileObjectTransport(
open(self._ctrl_pipe, 'wb'))
self._in_proto = TBinaryProtocol.TBinaryProtocol(self._in_trans)
self._out_proto = TBinaryProtocol.TBinaryProtocol(self._out_trans)
self._ctrl_proto = TBinaryProtocol.TBinaryProtocol(self._ctrl_trans)
self.__log_debug__('Setup communication done.')
self.__log_debug__('Start routine: {}'.format(self))
# run real implementation
self.__log_debug__('Run routine implementation.')
result = self.__run__()
# store result to map routine
self.__log_debug__('Store result to map routine.')
buffer = TSerialization.serialize(result)
send_int(self._out_proto, self._out_trans, len(buffer))
send_binary(self._out_trans, buffer)
# Shutdown routine
self.__log_debug__('Done. Shutdown routine')
send_object(self._ctrl_proto, self._ctrl_trans,
Order(command=Command.ROUTINE_DONE))
self._in_trans.close()
self._out_trans.close()
self._ctrl_trans.close()
return
def do_POST(self):
''' handling thrift messages '''
client_host, client_port = self.client_address
LOG.debug('Processing request from: ' +
client_host + ':' +
str(client_port))
# create new thrift handler
checker_md_docs = self.server.checker_md_docs
checker_md_docs_map = self.server.checker_md_docs_map
suppress_handler = self.server.suppress_handler
protocol_factory = TJSONProtocol.TJSONProtocolFactory()
input_protocol_factory = protocol_factory
output_protocol_factory = protocol_factory
itrans = TTransport.TFileObjectTransport(self.rfile)
otrans = TTransport.TFileObjectTransport(self.wfile)
itrans = TTransport.TBufferedTransport(itrans,
int(self.headers['Content-Length']))
otrans = TTransport.TMemoryBuffer()
iprot = input_protocol_factory.getProtocol(itrans)
oprot = output_protocol_factory.getProtocol(otrans)
try:
session = self.sc_session()
acc_handler = ThriftRequestHandler(session,
checker_md_docs,
checker_md_docs_map,
suppress_handler,
self.db_version_info)
processor = codeCheckerDBAccess.Processor(acc_handler)
processor.process(iprot, oprot)
result = otrans.getvalue()
self.sc_session.remove()
self.send_response(200)
self.send_header("content-type", "application/x-thrift")
self.send_header("Content-Length", len(result))
self.end_headers()
self.wfile.write(result)
return
except Exception as exn:
LOG.error(str(exn))
self.send_error(404, "Request failed.")
# self.send_header("content-type", "application/x-thrift")
# self.end_headers()
# self.wfile.write('')
return
def do_POST(self):
# Don't care about the request path.
itrans = TTransport.TFileObjectTransport(self.rfile)
otrans = TTransport.TFileObjectTransport(self.wfile)
itrans = TTransport.TBufferedTransport(itrans, int(self.headers['Content-Length']))
otrans = TTransport.TMemoryBuffer()
iprot = thttpserver.inputProtocolFactory.getProtocol(itrans)
oprot = thttpserver.outputProtocolFactory.getProtocol(otrans)
try:
thttpserver.processor.process(iprot, oprot)
except ResponseException, exn:
exn.handler(self)
def do_POST(self):
# Don't care about the request path.
self.send_response(200)
self.send_header("content-type", "application/x-thrift")
self.end_headers()
itrans = TTransport.TFileObjectTransport(self.rfile)
otrans = TTransport.TFileObjectTransport(self.wfile)
iprot = thttpserver.inputProtocolFactory.getProtocol(itrans)
oprot = thttpserver.outputProtocolFactory.getProtocol(otrans)
thttpserver.processor.process(iprot, oprot)
otrans.flush()
def read_schema(s3obj):
# get object size
object_size = s3obj.content_length
# raise error if object is too small
if object_size < 8:
raise ParquetFormatError('file is too small')
# get footer size
response = s3obj.get(Range='bytes={}-'.format(object_size - 8))
footer_size = struct.unpack('<i', response['Body'].read(4))[0]
magic_number = response['Body'].read(4)
# raise error if object is too small
if object_size < (8 + footer_size):
raise ParquetFormatError('file is too small')
# raise error if magic number is bad
if magic_number != 'PAR1':
raise ParquetFormatError('magic number is invalid')
# read footer
response = s3obj.get(Range='bytes={}-'.format(object_size - 8 -
footer_size))
footer = response['Body']
# read metadata from footer
transport = TTransport.TFileObjectTransport(footer)
protocol = TCompactProtocol.TCompactProtocol(transport)
metadata = FileMetaData()
metadata.read(protocol)
# return schema
return metadata.schema
def do_POST(self):
#ipdb.set_trace()
# Don't care about the request path.
thttpserver = self.server.thrift_server
thttpserver._replied = False
iftrans = TTransport.TFileObjectTransport(self.rfile)
itrans = TTransport.TBufferedTransport(
iftrans, int(self.headers['Content-Length']))
otrans = TTransport.TMemoryBuffer()
iprot = thttpserver.inputProtocolFactory.getProtocol(itrans)
oprot = thttpserver.outputProtocolFactory.getProtocol(otrans)
try:
#thttpserver.processor.on_message_begin(self.on_begin)
thttpserver.processor.process(iprot, oprot)
except ResponseException as exn:
exn.handler(self)
else:
if not thttpserver._replied:
# If the request was ONEWAY we would have replied already
data = otrans.getvalue()
self.send_response(200)
self.send_header("Content-Length", len(data))
self.send_header("Content-Type", "application/x-thrift")
self.end_headers()
self.wfile.write(data)
def extract(self, ops):
"""
Extracts exhibit visualizations for the provided filepattern.
Args:
filepattern: file pattern for glob of files
Yields:
Vis objects.
"""
filepattern = ops['filepattern']
for i in glob.glob(filepattern):
f = open(i)
transport = TTransport.TFileObjectTransport(f)
protocol = TBinaryProtocol.TBinaryProtocol(transport)
transport.open()
numInFile = 0
while True:
try:
v = ttypes.Visualization()
v.read(protocol)
numInFile += 1
visualizations = self.extractVisualizations(v)
for vis in visualizations:
yield vis
except EOFError:
#print "%s: %i" % (i, numInFile)
break
transport.close()
print self.typesSeen
def test_TFileObjectTransport(self):
test_dir = os.path.dirname(os.path.abspath(__file__))
datatxt_path = os.path.join(test_dir, 'data.txt')
buffer = '{"soft":"thrift","version":0.13,"1":true}'
with open(datatxt_path, "w+") as f:
buf = TTransport.TFileObjectTransport(f)
buf.write(buffer)
buf.flush()
buf.close()
with open(datatxt_path, "rb") as f:
buf = TTransport.TFileObjectTransport(f)
value = buf.read(len(buffer)).decode('utf-8')
self.assertEqual(buffer, value)
buf.close()
os.remove(datatxt_path)
def do_POST(self):
itrans = TTransport.TFileObjectTransport(self.rfile)
otrans = TTransport.TFileObjectTransport(self.wfile)
itrans = TTransport.TBufferedTransport(itrans, int(self.headers['Content-Length']))
otrans = TTransport.TMemoryBuffer()
iprot = thttpserver.inputProtocolFactory.getProtocol(itrans)
oprot = thttpserver.outputProtocolFactory.getProtocol(otrans)
try:
thttpserver.processor.process(iprot, oprot)
except ResponseException as exn:
exn.handler(self)
else:
self.send_response(200)
self.send_header('content-type', 'application/x-thrift')
self.end_headers()
self.wfile.write(otrans.getvalue())
def _read_page_header(fo):
"""Reads the page_header from the given fo"""
tin = TTransport.TFileObjectTransport(fo)
pin = TCompactProtocol.TCompactProtocol(tin)
ph = PageHeader()
ph.read(pin)
return ph
def _read_footer(fo):
"""Reads the footer from the given file object, returning a FileMetaData
object. This method assumes that the fo references a valid parquet file"""
footer_size = _get_footer_size(fo)
fo.seek(-(8 + footer_size), 2) # seek to beginning of footer
tin = TTransport.TFileObjectTransport(fo)
pin = TCompactProtocol.TCompactProtocol(tin)
fmd = FileMetaData()
fmd.read(pin)
return fmd
def do_POST(self):
# 走Http协议? 一次处理一个请求
# Don't care about the request path.
itrans = TTransport.TFileObjectTransport(self.rfile)
otrans = TTransport.TFileObjectTransport(self.wfile)
# 为输入数据添加Buffer
itrans = TTransport.TBufferedTransport(itrans, int(self.headers['Content-Length']))
otrans = TTransport.TMemoryBuffer()
iprot = thttpserver.inputProtocolFactory.getProtocol(itrans)
oprot = thttpserver.outputProtocolFactory.getProtocol(otrans)
try:
thttpserver.processor.process(iprot, oprot)
except ResponseException as exn:
exn.handler(self)
else:
self.send_response(200)
self.send_header("content-type", "application/x-thrift")
self.end_headers()
self.wfile.write(otrans.getvalue())
def main():
set_stdout_encoding()
parser = make_parser()
args = parser.parse_args()
logging.basicConfig(format='%(asctime)-15s %(levelname)s: %(message)s',
level=args.loglevel.upper())
mimetypes.init()
(ifile_type, ifile_encoding) = mimetypes.guess_type(args.input_file)
(ofile_type, ofile_encoding) = mimetypes.guess_type(args.output_file)
out_writer = None
if args.direction is None:
if args.iprotocol is None or args.oprotocol is None:
print("Either --direction, or both --iprotocol and --oprotocol,"
" must be provided")
exit(1)
else:
if (args.iprotocol is not None) or (args.oprotocol is not None):
print("Not both --direction, and either --iprotocol or"
" --oprotocol, can be provided")
exit(1)
encoding_input = KNOWN_CONVERSIONS[
args.direction][0] if args.iprotocol is None else PROTOCOLS[
args.iprotocol]
encoding_output = KNOWN_CONVERSIONS[
args.direction][1] if args.oprotocol is None else PROTOCOLS[
args.oprotocol]
if ofile_encoding == "gzip":
out_writer = gzip.GzipFile(args.output_file, 'wb')
else:
out_writer = open(args.output_file, 'w')
if ifile_encoding == 'gzip':
f = gzip.GzipFile(args.input_file)
transportIn = TTransport.TFileObjectTransport(f)
protocolIn = encoding_input().getProtocol(transportIn)
while True:
try:
comm = Communication()
comm.read(protocolIn)
output_bytes = TSerialization.serialize(
comm, protocol_factory=encoding_output())
out_writer.write(output_bytes)
except EOFError:
break
f.close()
else:
convert(input_file_path=args.input_file,
output_file_path=args.output_file,
input_protocol_factory=encoding_input,
output_protocol_factory=encoding_output)
out_writer.close()
def _read_footer(fo):
"""Reads the footer from the given file object, returning a FileMetaData
object. This method assumes that the fo references a valid parquet file"""
footer_size = _get_footer_size(fo)
if logger.isEnabledFor(logging.DEBUG):
logger.debug("Footer size in bytes: %s", footer_size)
fo.seek(-(8 + footer_size), 2) # seek to beginning of footer
tin = TTransport.TFileObjectTransport(fo)
pin = TCompactProtocol.TCompactProtocol(tin)
fmd = FileMetaData()
fmd.read(pin)
return fmd
def iter_wikilink_items_under_path(path):
for fopen in iter_fopens_at_path(os.path.join(path, '*.gz')):
with fopen() as f:
# there must be a nicer way to do this...
thrift_buffer = TTransport.TFileObjectTransport(f)
proto = TBinaryProtocol.TBinaryProtocol(thrift_buffer)
while True:
try:
w = wikilinks.WikiLinkItem()
wikilinks.WikiLinkItem.read(w, proto)
yield w
except EOFError:
break
def _open(self):
"""Open the IO stream.
Make it ready to output thrift data to local disk.
"""
try:
# Firstly, make sure close all IO stream
if self._protocol:
LOG.warning(
"Thrift protocol already exists! Need to close and reopen it."
)
self._protocol = None
if self._transport:
LOG.warning(
"Thrift transport already exists! Need to close and reopen it."
)
self._transport.close()
self._transport = None
# self._file_stream should be closed by self._transport.close()
self._file_stream = None
except IOError as ioe:
util.error_text_logging(
LOG,
'ThriftLogger for topic %s failed to open the IO stream!\n '
'I/O error (%s) : %s' % (self._topic, ioe.errno, ioe.strerror))
except:
util.error_text_logging(
LOG,
'ThriftLogger for topic %s failed to open the IO stream due to '
'unexpected error:\n %s' % (self._topic, sys.exc_info()[0]))
try:
# Secondly, open new IO stream
self._file_stream = open(self._working_file_name, 'ab')
self._transport = TTransport.TFramedTransport(
TTransport.TFileObjectTransport(self._file_stream))
self._protocol = TBinaryProtocol.TBinaryProtocol(self._transport)
self._transport.open()
except IOError as ioe:
util.error_text_logging(
LOG,
'ThriftLogger for topic %s failed to open the IO stream!\n '
'I/O error (%s) : %s' % (self._topic, ioe.errno, ioe.strerror))
except:
util.error_text_logging(
LOG,
'ThriftLogger for topic %s failed to open the IO stream due to '
'unexpected error:\n %s' % (self._topic, sys.exc_info()[0]))
def fetch_http_endpoint():
"""Thrift RPC endpoint for Concrete FetchCommunicationService
"""
itrans = TTransport.TFileObjectTransport(bottle.request.body)
itrans = TTransport.TBufferedTransport(
itrans, int(bottle.request.headers['Content-Length']))
otrans = TTransport.TMemoryBuffer()
iprot = QuicklimeServer.TSERVER.inputProtocolFactory.getProtocol(itrans)
oprot = QuicklimeServer.TSERVER.outputProtocolFactory.getProtocol(otrans)
QuicklimeServer.TSERVER.processor.process(iprot, oprot)
bytestring = otrans.getvalue()
headers = dict()
headers['Content-Length'] = len(bytestring)
headers['Content-Type'] = "application/x-thrift"
return bottle.HTTPResponse(bytestring, **headers)
def thrift_endpoint(tserver):
"""Thrift RPC endpoint
"""
itrans = TTransport.TFileObjectTransport(bottle.request.body)
itrans = TTransport.TBufferedTransport(
itrans, int(bottle.request.headers['Content-Length']))
otrans = TTransport.TMemoryBuffer()
iprot = tserver.inputProtocolFactory.getProtocol(itrans)
oprot = tserver.outputProtocolFactory.getProtocol(otrans)
tserver.processor.process(iprot, oprot)
bytestring = otrans.getvalue()
headers = dict()
headers['Content-Length'] = len(bytestring)
headers['Content-Type'] = "application/x-thrift"
return bottle.HTTPResponse(bytestring, **headers)
def make_csv(switchname):
for env in thrift_config[switchname]:
i_switch = env[2]
for cdr_file in glob.glob(os.path.join(env[1], '*sdp*.bin.*')):
try:
filename = cdr_file.split('/')[-1]
tag = filename.split('.')[0]
except:
tag = ''
filename = ''
if tag not in valid_cdr_files:
continue
print cdr_file
csv_name = '%s_%d_%s.csv' % (switchname, i_switch, filename)
csvfile = open(os.path.join('/var/data/cdrs', csv_name), "wb")
wr = csv.DictWriter(csvfile,
delimiter='`',
escapechar='\\',
lineterminator='\n',
quoting=csv.QUOTE_NONE,
fieldnames=valid_cdr_files[tag][1],
extrasaction='ignore')
fd = file(cdr_file)
t = TTransport.TFileObjectTransport(fd)
p = TBinaryProtocol.TBinaryProtocolAccelerated(t)
while True:
obj = valid_cdr_files[tag][0]()
try:
obj.read(p)
wr.writerow(align_struct(obj, i_switch))
except EOFError:
break
csvfile.close()
os.remove(cdr_file)
def do_POST(self):
"""
Handles POST queries, which are usually Thrift messages.
"""
client_host, client_port = self.client_address
self.auth_session = self.__check_session_cookie()
LOG.info("%s:%s -- [%s] POST %s", client_host, str(client_port),
self.auth_session.user if self.auth_session else "Anonymous",
self.path)
# Create new thrift handler.
checker_md_docs = self.server.checker_md_docs
checker_md_docs_map = self.server.checker_md_docs_map
version = self.server.version
protocol_factory = TJSONProtocol.TJSONProtocolFactory()
input_protocol_factory = protocol_factory
output_protocol_factory = protocol_factory
itrans = TTransport.TFileObjectTransport(self.rfile)
itrans = TTransport.TBufferedTransport(
itrans, int(self.headers['Content-Length']))
otrans = TTransport.TMemoryBuffer()
iprot = input_protocol_factory.getProtocol(itrans)
oprot = output_protocol_factory.getProtocol(otrans)
if self.server.manager.is_enabled and \
not self.path.endswith(('/Authentication',
'/Configuration')) and \
not self.auth_session:
# Bail out if the user is not authenticated...
# This response has the possibility of melting down Thrift clients,
# but the user is expected to properly authenticate first.
LOG.debug(
"%s:%s Invalid access, credentials not found "
"- session refused.", client_host, str(client_port))
self.send_thrift_exception("Error code 401: Unauthorized!", iprot,
oprot, otrans)
return
# Authentication is handled, we may now respond to the user.
try:
product_endpoint, api_ver, request_endpoint = \
routing.split_client_POST_request(self.path)
product = None
if product_endpoint:
# The current request came through a product route, and not
# to the main endpoint.
product = self.__check_prod_db(product_endpoint)
version_supported = routing.is_supported_version(api_ver)
if version_supported:
major_version, _ = version_supported
if major_version == 6:
if request_endpoint == 'Authentication':
auth_handler = AuthHandler_v6(
self.server.manager, self.auth_session,
self.server.config_session)
processor = AuthAPI_v6.Processor(auth_handler)
elif request_endpoint == 'Configuration':
conf_handler = ConfigHandler_v6(
self.auth_session, self.server.config_session)
processor = ConfigAPI_v6.Processor(conf_handler)
elif request_endpoint == 'Products':
prod_handler = ProductHandler_v6(
self.server, self.auth_session,
self.server.config_session, product, version)
processor = ProductAPI_v6.Processor(prod_handler)
elif request_endpoint == 'CodeCheckerService':
# This endpoint is a product's report_server.
if not product:
error_msg = "Requested CodeCheckerService on a " \
"nonexistent product: '{0}'." \
.format(product_endpoint)
LOG.error(error_msg)
raise ValueError(error_msg)
if product_endpoint:
# The current request came through a
# product route, and not to the main endpoint.
product = self.__check_prod_db(product_endpoint)
acc_handler = ReportHandler_v6(
self.server.manager, product.session_factory,
product, self.auth_session,
self.server.config_session, checker_md_docs,
checker_md_docs_map, version, self.server.context)
processor = ReportAPI_v6.Processor(acc_handler)
else:
LOG.debug("This API endpoint does not exist.")
error_msg = "No API endpoint named '{0}'." \
.format(self.path)
raise ValueError(error_msg)
else:
error_msg = "The API version you are using is not supported " \
"by this server (server API version: {0})!".format(
get_version_str())
self.send_thrift_exception(error_msg, iprot, oprot, otrans)
return
processor.process(iprot, oprot)
result = otrans.getvalue()
self.send_response(200)
self.send_header("content-type", "application/x-thrift")
self.send_header("Content-Length", len(result))
self.end_headers()
self.wfile.write(result)
return
except Exception as exn:
LOG.warning(str(exn))
import traceback
traceback.print_exc()
cstringio_buf = itrans.cstringio_buf.getvalue()
if cstringio_buf:
itrans = TTransport.TMemoryBuffer(cstringio_buf)
iprot = input_protocol_factory.getProtocol(itrans)
self.send_thrift_exception(str(exn), iprot, oprot, otrans)
return
print("Not both --direction, and either --iprotocol or"
" --oprotocol, can be provided")
exit(1)
encoding_input = KNOWN_CONVERSIONS[
args.direction][0] if args.iprotocol is None else PROTOCOLS[
args.iprotocol]
encoding_output = KNOWN_CONVERSIONS[
args.direction][1] if args.oprotocol is None else PROTOCOLS[
args.oprotocol]
if ofile_encoding == "gzip":
out_writer = gzip.GzipFile(args.output_file, 'wb')
else:
out_writer = open(args.output_file, 'w')
if ifile_encoding == 'gzip':
f = gzip.GzipFile(args.input_file)
transportIn = TTransport.TFileObjectTransport(f)
protocolIn = encoding_input().getProtocol(transportIn)
while True:
try:
comm = Communication()
comm.read(protocolIn)
output_bytes = TSerialization.serialize(
comm, protocol_factory=encoding_output())
out_writer.write(output_bytes)
except EOFError:
break
f.close()
else:
convert(input_file_path=args.input_file,
output_file_path=args.output_file,
input_protocol_factory=encoding_input,
# Apache Thrift JSON Protocol Struct Writer in Python
from thrift.transport import TTransport
from thrift.protocol import TJSONProtocol
from thrift import Thrift
class Trade:
def __init__(self):
symbol = ""
price = 0.0
size = 0
trans = TTransport.TFileObjectTransport(open("data", "wb"))
proto = TJSONProtocol.TJSONProtocol(trans)
trade = Trade()
trade.symbol = "F"
trade.price = 13.10
trade.size = 2500
proto.writeStructBegin("Trade")
proto.writeFieldBegin("symbol", Thrift.TType.STRING, 1)
proto.writeString(trade.symbol)
proto.writeFieldEnd()
proto.writeFieldBegin("price", Thrift.TType.DOUBLE, 2)
proto.writeDouble(trade.price)
proto.writeFieldEnd()
def do_POST(self):
"""
Handles POST queries, which are usually Thrift messages.
"""
client_host, client_port = self.client_address
auth_session = self.__check_auth_in_request()
LOG.info("{0}:{1} -- [{2}] POST {3}"
.format(client_host,
str(client_port),
auth_session.user if auth_session else "Anonymous",
self.path))
# Create new thrift handler.
checker_md_docs = self.server.checker_md_docs
checker_md_docs_map = self.server.checker_md_docs_map
suppress_handler = self.server.suppress_handler
version = self.server.version
protocol_factory = TJSONProtocol.TJSONProtocolFactory()
input_protocol_factory = protocol_factory
output_protocol_factory = protocol_factory
itrans = TTransport.TFileObjectTransport(self.rfile)
itrans = TTransport.TBufferedTransport(itrans,
int(self.headers[
'Content-Length']))
otrans = TTransport.TMemoryBuffer()
iprot = input_protocol_factory.getProtocol(itrans)
oprot = output_protocol_factory.getProtocol(otrans)
if self.server.manager.isEnabled() and \
not self.path.endswith('/Authentication') and \
not auth_session:
# Bail out if the user is not authenticated...
# This response has the possibility of melting down Thrift clients,
# but the user is expected to properly authenticate first.
LOG.debug(client_host + ":" + str(client_port) +
" Invalid access, credentials not found " +
"- session refused.")
self.send_error(401)
return
# Authentication is handled, we may now respond to the user.
try:
product_endpoint, api_ver, request_endpoint = \
routing.split_client_POST_request(self.path)
product = None
if product_endpoint:
# The current request came through a product route, and not
# to the main endpoint.
product = self.server.get_product(product_endpoint)
if product and not product.connected:
# If the product is not connected, try reconnecting...
LOG.debug("Request's product '{0}' is not connected! "
"Attempting reconnect..."
.format(product_endpoint))
product.connect()
if not product.connected:
# If the reconnection fails, send an error to the user.
LOG.debug("Product reconnection failed.")
self.send_error( # 500 Internal Server Error
500, "Product '{0}' database connection failed!"
.format(product_endpoint))
return
elif not product:
LOG.debug("Requested product does not exist.")
self.send_error(
404, "The product {0} does not exist."
.format(product_endpoint))
return
version_supported = routing.is_supported_version(api_ver)
if version_supported:
major_version, _ = version_supported
if major_version == 6:
if request_endpoint == 'Authentication':
auth_handler = AuthHandler_v6(
self.server.manager,
auth_session,
self.server.config_session)
processor = AuthAPI_v6.Processor(auth_handler)
elif request_endpoint == 'Products':
prod_handler = ProductHandler_v6(
self.server,
auth_session,
self.server.config_session,
product,
version)
processor = ProductAPI_v6.Processor(prod_handler)
elif request_endpoint == 'CodeCheckerService':
# This endpoint is a product's report_server.
if not product:
LOG.debug("Requested CodeCheckerService on a "
"nonexistent product.")
self.send_error( # 404 Not Found
404,
"The specified product '{0}' does not exist!"
.format(product_endpoint))
return
acc_handler = ReportHandler_v6(
product.session_factory,
product,
auth_session,
self.server.config_session,
checker_md_docs,
checker_md_docs_map,
suppress_handler,
version)
processor = ReportAPI_v6.Processor(acc_handler)
else:
LOG.debug("This API endpoint does not exist.")
self.send_error(404, # 404 Not Fount
"No API endpoint named '{0}'."
.format(self.path))
return
else:
if request_endpoint == 'Authentication':
# API-version checking is supported on the auth endpoint.
handler = BadAPIHandler(api_ver)
processor = AuthAPI_v6.Processor(handler)
else:
# Send a custom, but valid Thrift error message to the
# client requesting this action.
LOG.debug("API version v{0} not supported by server."
.format(api_ver))
self.send_error(400, # 400 Bad Request
"This API version 'v{0}' is not supported."
.format(api_ver))
return
processor.process(iprot, oprot)
result = otrans.getvalue()
self.send_response(200)
self.send_header("content-type", "application/x-thrift")
self.send_header("Content-Length", len(result))
self.end_headers()
self.wfile.write(result)
return
except Exception as exn:
import traceback
traceback.print_exc()
LOG.error(str(exn))
self.send_error(404, "Request failed.")
return
def do_POST(self):
"""
Handles POST queries, which are usually Thrift messages.
"""
client_host, client_port = self.client_address
auth_session = self.__check_session_cookie()
LOG.debug("%s:%s -- [%s] POST %s", client_host, str(client_port),
auth_session.user if auth_session else "Anonymous",
self.path)
# Create new thrift handler.
checker_md_docs = self.server.checker_md_docs
checker_md_docs_map = self.server.checker_md_docs_map
suppress_handler = self.server.suppress_handler
version = self.server.version
protocol_factory = TJSONProtocol.TJSONProtocolFactory()
input_protocol_factory = protocol_factory
output_protocol_factory = protocol_factory
itrans = TTransport.TFileObjectTransport(self.rfile)
itrans = TTransport.TBufferedTransport(
itrans, int(self.headers['Content-Length']))
otrans = TTransport.TMemoryBuffer()
iprot = input_protocol_factory.getProtocol(itrans)
oprot = output_protocol_factory.getProtocol(otrans)
if self.server.manager.is_enabled and \
not self.path.endswith('/Authentication') and \
not auth_session:
# Bail out if the user is not authenticated...
# This response has the possibility of melting down Thrift clients,
# but the user is expected to properly authenticate first.
LOG.debug(client_host + ":" + str(client_port) +
" Invalid access, credentials not found " +
"- session refused.")
self.send_error(401)
return
# Authentication is handled, we may now respond to the user.
try:
product_endpoint, api_ver, request_endpoint = \
routing.split_client_POST_request(self.path)
product = None
if product_endpoint:
# The current request came through a product route, and not
# to the main endpoint.
product = self.server.get_product(product_endpoint)
self.__check_prod_db(product)
version_supported = routing.is_supported_version(api_ver)
if version_supported:
major_version, _ = version_supported
if major_version == 6:
if request_endpoint == 'Authentication':
auth_handler = AuthHandler_v6(
self.server.manager, auth_session,
self.server.config_session)
processor = AuthAPI_v6.Processor(auth_handler)
elif request_endpoint == 'Products':
prod_handler = ProductHandler_v6(
self.server, auth_session,
self.server.config_session, product, version)
processor = ProductAPI_v6.Processor(prod_handler)
elif request_endpoint == 'CodeCheckerService':
# This endpoint is a product's report_server.
if not product:
error_msg = "Requested CodeCheckerService on a " \
"nonexistent product: '{0}'." \
.format(product_endpoint)
LOG.error(error_msg)
raise ValueError(error_msg)
if product_endpoint:
# The current request came through a
# product route, and not to the main endpoint.
product = self.server.get_product(product_endpoint)
self.__check_prod_db(product)
acc_handler = ReportHandler_v6(
self.server.manager, product.session_factory,
product, auth_session, self.server.config_session,
checker_md_docs, checker_md_docs_map,
suppress_handler, version)
processor = ReportAPI_v6.Processor(acc_handler)
else:
LOG.debug("This API endpoint does not exist.")
error_msg = "No API endpoint named '{0}'." \
.format(self.path)
raise ValueError(error_msg)
else:
if request_endpoint == 'Authentication':
# API-version checking is supported on the auth endpoint.
handler = BadAPIHandler(api_ver)
processor = AuthAPI_v6.Processor(handler)
else:
# Send a custom, but valid Thrift error message to the
# client requesting this action.
error_msg = "Incompatible client/server API." \
"API versions supported by this server {0}." \
.format(get_version_str())
raise ValueError(error_msg)
processor.process(iprot, oprot)
result = otrans.getvalue()
self.send_response(200)
self.send_header("content-type", "application/x-thrift")
self.send_header("Content-Length", len(result))
self.end_headers()
self.wfile.write(result)
return
except Exception as exn:
# Convert every Exception to the proper format which can be parsed
# by the Thrift clients expecting JSON responses.
LOG.error(exn.message)
import traceback
traceback.print_exc()
ex = TApplicationException(TApplicationException.INTERNAL_ERROR,
exn.message)
fname, _, seqid = iprot.readMessageBegin()
oprot.writeMessageBegin(fname, TMessageType.EXCEPTION, seqid)
ex.write(oprot)
oprot.writeMessageEnd()
oprot.trans.flush()
result = otrans.getvalue()
self.send_response(200)
self.send_header("content-type", "application/x-thrift")
self.send_header("Content-Length", len(result))
self.end_headers()
self.wfile.write(result)
return
import pickle
import sys
from thrift import Thrift
from thrift.transport import TTransport
class Trade:
def __init__(self):
symbol = ""
price = 0.0
size = 0
try:
trans = TTransport.TFileObjectTransport(open("data", "wb"))
trade = Trade()
trade.symbol = "F"
trade.price = 13.10
trade.size = 2500
trans.write(pickle.dumps(trade))
trans.close()
if len(sys.argv) == 2:
raise TTransport.TTransportException(
TTransport.TTransportException.NOT_OPEN, "cmd line ex")
trans = TTransport.TFileObjectTransport(
open("data", ("wb" if len(sys.argv) > 2 else "rb")))
bstr = trans.read(128)
trade = pickle.loads(bstr)
def __init__(self, filename, add_references=True, filetype=FileType.AUTO):
filetype = FileType.lookup(filetype)
self._add_references = add_references
self._source_filename = filename
if filetype == FileType.TAR:
self.filetype = 'tar'
self.tar = tarfile.open(filename, 'r|')
elif filetype == FileType.TAR_GZ:
self.filetype = 'tar'
self.tar = tarfile.open(filename, 'r|gz')
elif filetype == FileType.TAR_BZ2:
self.filetype = 'tar'
self.tar = tarfile.open(filename, 'r|bz2')
elif filetype == FileType.ZIP:
self.filetype = 'zip'
self.zip = zipfile.ZipFile(filename, 'r')
self.zip_infolist = self.zip.infolist()
self.zip_infolist_index = 0
elif filetype == FileType.STREAM:
self.filetype = 'stream'
f = open(filename, 'rb')
elif filetype == FileType.STREAM_GZ:
self.filetype = 'stream'
f = gzip.open(filename, 'rb')
elif filetype == FileType.STREAM_BZ2:
self.filetype = 'stream'
f = bz2.BZ2File(filename, 'r')
elif filetype == FileType.AUTO:
if tarfile.is_tarfile(filename):
self.filetype = 'tar'
self.tar = tarfile.open(filename, 'r|*')
elif zipfile.is_zipfile(filename):
self.filetype = 'zip'
self.zip = zipfile.ZipFile(filename, 'r')
self.zip_infolist = self.zip.infolist()
self.zip_infolist_index = 0
elif mimetypes.guess_type(filename)[1] == 'gzip':
# this is not a true stream---is_tarfile will have
# successfully seeked backwards on the file if we have
# reached this point
self.filetype = 'stream'
f = gzip.open(filename, 'rb')
elif mimetypes.guess_type(filename)[1] == 'bzip2':
# this is not a true stream
self.filetype = 'stream'
f = bz2.BZ2File(filename, 'r')
else:
# this is not a true stream
self.filetype = 'stream'
f = open(filename, 'rb')
else:
raise ValueError('unknown filetype %d' % filetype)
if self.filetype is 'stream':
self.transport = TTransport.TFileObjectTransport(f)
self.protocol = factory.createProtocol(self.transport)
self.transport.open()
from thrift.transport import TTransport
from thrift.protocol import TBinaryProtocol
import sys
sys.path.append("/usr/local/lib/python3.4")
sys.path.append("./thrift_stub_dir")
#from ssp import ttypes
from thrift_stub_dir.thrift.ttypes import Cdrs
from thrift_stub_dir.thrift.ttypes import CdrsConnections
cdr_type = str(sys.argv[1])
sippy_encodedfile = str(sys.argv[2])
fd = file(sippy_encodedfile)
t = TTransport.TFileObjectTransport(fd)
p = TBinaryProtocol.TBinaryProtocolAccelerated(t)
while True:
if cdr_type == 'customer':
obj = Cdrs()
else:
obj = CdrsConnections()
try:
obj.read(p)
if cdr_type == 'customer':
print(
"%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s"
%
(obj.i_cdr, obj.i_call, obj.i_account, obj.result, obj.cost,
obj.delay, obj.duration, obj.billed_duration,
obj.connect_time, obj.disconnect_time, obj.cld_in
def do_POST(self):
""" Handling thrift messages. """
client_host, client_port = self.client_address
LOG.debug('Processing request from: ' + client_host + ':' +
str(client_port))
# Create new thrift handler.
checker_md_docs = self.server.checker_md_docs
checker_md_docs_map = self.server.checker_md_docs_map
suppress_handler = self.server.suppress_handler
protocol_factory = TJSONProtocol.TJSONProtocolFactory()
input_protocol_factory = protocol_factory
output_protocol_factory = protocol_factory
itrans = TTransport.TFileObjectTransport(self.rfile)
otrans = TTransport.TFileObjectTransport(self.wfile)
itrans = TTransport.TBufferedTransport(
itrans, int(self.headers['Content-Length']))
otrans = TTransport.TMemoryBuffer()
iprot = input_protocol_factory.getProtocol(itrans)
oprot = output_protocol_factory.getProtocol(otrans)
sess_token = self.check_auth_in_request()
if self.path != '/Authentication' and not sess_token:
# Bail out if the user is not authenticated...
# This response has the possibility of melting down Thrift clients,
# but the user is expected to properly authenticate first.
LOG.debug(client_host + ":" + str(client_port) +
" Invalid access, credentials not found " +
"- session refused.")
self.send_response(401)
self.send_header("Content-type", "text/plain")
self.send_header("Content-length", str(0))
self.end_headers()
return
# Authentication is handled, we may now respond to the user.
try:
session = self.sc_session()
acc_handler = ThriftRequestHandler(session, checker_md_docs,
checker_md_docs_map,
suppress_handler,
self.db_version_info)
if self.path == '/Authentication':
# Authentication requests must be routed to a different
# handler.
auth_handler = ThriftAuthHandler(self.manager, client_host,
sess_token)
processor = codeCheckerAuthentication.Processor(auth_handler)
else:
acc_handler = ThriftRequestHandler(session, checker_md_docs,
checker_md_docs_map,
suppress_handler,
self.db_version_info)
processor = codeCheckerDBAccess.Processor(acc_handler)
processor.process(iprot, oprot)
result = otrans.getvalue()
self.sc_session.remove()
self.send_response(200)
self.send_header("content-type", "application/x-thrift")
self.send_header("Content-Length", len(result))
self.end_headers()
self.wfile.write(result)
return
except Exception as exn:
LOG.error(str(exn))
self.send_error(404, "Request failed.")
return
def __init__(self,
thrift_type,
filename,
postprocess=None,
filetype=FileType.AUTO):
"""
Args:
thrift_type: Class for Thrift type, e.g. Communication, TokenLattice
filename (str):
postprocess (function): A post-processing function that is called
with the Thrift object as argument each time a Thrift object
is read from the file
filetype (FileType): Expected type of file. Default value is
`FileType.AUTO`, where function will try to automatically
determine file type.
Raises:
ValueError: if filetype is not a known filetype name or id
"""
filetype = FileType.lookup(filetype)
self._seek_supported = True
self._thrift_type = thrift_type
if postprocess is None:
def _noop(obj):
return
self._postprocess = _noop
else:
self._postprocess = postprocess
self._source_filename = filename
if filetype == FileType.TAR:
self.filetype = 'tar'
self.tar = tarfile.open(filename, 'r|')
elif filetype == FileType.TAR_GZ:
self.filetype = 'tar'
self.tar = tarfile.open(filename, 'r|gz')
elif filetype == FileType.TAR_BZ2:
self.filetype = 'tar'
self.tar = tarfile.open(filename, 'r|bz2')
elif filetype == FileType.ZIP:
self.filetype = 'zip'
self.zip = zipfile.ZipFile(filename, 'r')
self.zip_infolist = self.zip.infolist()
self.zip_infolist_index = 0
elif filetype == FileType.STREAM:
self.filetype = 'stream'
f = open(filename, 'rb')
elif filetype == FileType.STREAM_GZ:
self.filetype = 'stream'
f = gzip_open(filename, 'rb')
elif filetype == FileType.STREAM_BZ2:
self.filetype = 'stream'
f = bz2.BZ2File(filename, 'r')
elif filetype == FileType.AUTO:
if tarfile.is_tarfile(filename):
self.filetype = 'tar'
self.tar = tarfile.open(filename, 'r|*')
elif zipfile.is_zipfile(filename):
self.filetype = 'zip'
self.zip = zipfile.ZipFile(filename, 'r')
self.zip_infolist = self.zip.infolist()
self.zip_infolist_index = 0
elif mimetypes.guess_type(filename)[1] == 'gzip':
# this is not a true stream---is_tarfile will have
# successfully seeked backwards on the file if we have
# reached this point
self.filetype = 'stream'
f = gzip_open(filename, 'rb')
elif mimetypes.guess_type(filename)[1] == 'bzip2':
# this is not a true stream
self.filetype = 'stream'
f = bz2.BZ2File(filename, 'r')
else:
# this is not a true stream
self.filetype = 'stream'
f = open(filename, 'rb')
else:
raise ValueError('unknown filetype %d' % filetype)
if self.filetype == 'stream':
self.transport = TTransport.TFileObjectTransport(f)
self.protocol = factory.createProtocol(self.transport)
self.transport.open()
# Apache Thrift File Transport in Python
import pickle
from thrift.transport import TTransport
class Trade:
def __init__(self, symbol="", price=0.0, size=0):
self.symbol = symbol
self.price = price
self.size = size
trans_out = TTransport.TFileObjectTransport(open("data", "wb"))
trade = Trade("F", 13.10, 2500)
trans_out.write(pickle.dumps(trade))
trans_out.close()
trans_in = TTransport.TFileObjectTransport(open("data", "rb"))
bstr = trans_in.read(4096)
trade = pickle.loads(bstr)
print("Trade(%d): %s %d @ %f" %
(len(bstr), trade.symbol, trade.size, trade.price))
