def main(argv):
p = argparse.ArgumentParser()
add_common_args(p)
# Since THeaderTransport acts as framed transport when detected frame, we
# cannot use --transport=framed as it would result in 2 layered frames.
p.add_argument('--override-transport')
p.add_argument('--override-protocol')
args = p.parse_args()
assert args.protocol == 'header'
assert args.transport == 'buffered'
assert not args.ssl
sock = TSocket(args.host, args.port, socket_family=socket.AF_INET)
if not args.override_transport or args.override_transport == 'buffered':
trans = TBufferedTransport(sock)
elif args.override_transport == 'framed':
print('TFRAMED')
trans = TFramedTransport(sock)
else:
raise ValueError('invalid transport')
trans.open()
if not args.override_protocol or args.override_protocol == 'binary':
proto = TBinaryProtocol(trans)
elif args.override_protocol == 'compact':
proto = TCompactProtocol(trans)
else:
raise ValueError('invalid transport')
test_void(proto)
test_void(proto)
trans.close()
class HbaseClient:
def __init__(self, host, port):
self.transport = TBufferedTransport(TSocket(host, port))
self.transport.open()
self.protocol = TBinaryProtocol.TBinaryProtocol(self.transport)
self.client = Hbase.Client(self.protocol)
self.scan = TScan()
def createTable(self, table, contents):
return self.client.createTable(table, [contents])
def mutateRow(self, table, row, mutations, st):
return self.client.mutateRow(table, row, mutations, st)
def getTable(self):
return self.client.getTableNames()
def scannerGetList(self, tableName, num):
id = self.client.scannerOpenWithScan(tableName, self.scan, None)
return self.client.scannerGetList(id, num)
def scannerGet(self, tableName):
id = self.client.scannerOpenWithScan(tableName, self.scan, None)
return self.client.scannerGet(id)
def close(self):
self.transport.close()
def main(argv):
p = argparse.ArgumentParser()
add_common_args(p)
# Since THeaderTransport acts as framed transport when detected frame, we
# cannot use --transport=framed as it would result in 2 layered frames.
p.add_argument('--override-transport')
p.add_argument('--override-protocol')
args = p.parse_args()
assert args.protocol == 'header'
assert args.transport == 'buffered'
assert not args.ssl
sock = TSocket(args.host, args.port, socket_family=socket.AF_INET)
if not args.override_transport or args.override_transport == 'buffered':
trans = TBufferedTransport(sock)
elif args.override_transport == 'framed':
print('TFRAMED')
trans = TFramedTransport(sock)
else:
raise ValueError('invalid transport')
trans.open()
if not args.override_protocol or args.override_protocol == 'binary':
proto = TBinaryProtocol(trans)
elif args.override_protocol == 'compact':
proto = TCompactProtocol(trans)
else:
raise ValueError('invalid transport')
test_void(proto)
test_void(proto)
trans.close()
def main(args):
if (len(args) < 2):
print "%s <verified file> -all" % (args[0])
sys.exit(1)
filename = args[1]
opt_all = True if len(args) > 2 and args[2] == "-all" else False
filenamearray = filename.split("_")
orgId = filenamearray[0]
subOrgId = filenamearray[1]
getConfiguration('host.properties')
transport = TBufferedTransport(TSocket(hbaseHost, 9090))
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
global client
client = Hbase.Client(protocol)
tablename = "%s_%s_master_%s" % (orgId, subOrgId, orgId)
for line in open(filename, "r"):
input = line.strip()
row = client.getRow(tablename, input)
print input
printRow(row)
print ""
if (not opt_all): break
transport.close()
def run(self):
global work_mutex
global work_numbers
err_local = 0
try:
socket = TSocket(self.server_ip, int(self.server_port))
transport = TFramedTransport(socket)
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
client = ThriftNeloEventServer.Client(protocol)
stop_flag = True
while stop_flag:
#read thrift from file
f = file(file_name, 'r')
fd_transport = TFileObjectTransport(f)
buffered_transport = TBufferedTransport(fd_transport)
binary_protocol = TBinaryProtocol.TBinaryProtocol(
buffered_transport)
fd_transport.open()
stop_flag = False
try:
evt = ThriftNeloEvent()
while True:
evt.read(binary_protocol)
#send the log to each project name
for prjName, logCnt in prj_dict.items():
try:
if logCnt_dict.has_key(prjName):
if int(logCnt_dict[prjName]) < int(logCnt):
evt.projectName = prjName
evt.sendTime = int(time.time() * 1000)
err = client.ackedAppend(evt)
tps_remember(err)
err_local += err
logCnt_dict[
prjName] = logCnt_dict[prjName] + 1
stop_flag = True
else:
evt.projectName = prjName
err = client.ackedAppend(evt)
tps_remember(err)
err_local += err
logCnt_dict[prjName] = 1
stop_flag = True
except TException, msg:
print msg, prjName
except EOFError, msg:
buffered_transport.close() #close the transport
stop_flag = True
work_mutex.acquire()
work_numbers -= 1
work_mutex.release()
socket.close()
def run(self):
global work_mutex
global work_numbers
err_local = 0
try:
socket = TSocket(self.server_ip, int(self.server_port))
transport = TFramedTransport(socket)
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
client = ThriftNeloEventServer.Client(protocol)
stop_flag = True
while stop_flag:
#read thrift from file
f = file(file_name, 'r')
fd_transport = TFileObjectTransport(f)
buffered_transport = TBufferedTransport(fd_transport)
binary_protocol = TBinaryProtocol.TBinaryProtocol(buffered_transport)
fd_transport.open()
stop_flag = False
try:
evt = ThriftNeloEvent()
while True:
evt.read(binary_protocol)
#send the log to each project name
for prjName, logCnt in prj_dict.items():
try:
if logCnt_dict.has_key(prjName):
if int(logCnt_dict[prjName]) < int(logCnt):
evt.projectName = prjName
evt.sendTime = int(time.time() * 1000)
err = client.ackedAppend(evt)
tps_remember(err)
err_local += err
logCnt_dict[prjName] = logCnt_dict[prjName] + 1
stop_flag = True
else:
evt.projectName = prjName
err = client.ackedAppend(evt)
tps_remember(err)
err_local += err
logCnt_dict[prjName] = 1
stop_flag = True
except TException, msg:
print msg, prjName
except EOFError,msg:
buffered_transport.close() #close the transport
stop_flag = True
work_mutex.acquire()
work_numbers -= 1
work_mutex.release()
socket.close()
def getMasterTables(hbaseHost):
transport = TBufferedTransport(TSocket(hbaseHost, 9090))
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
client = Hbase.Client(protocol)
for table in client.getTableNames():
if 'master' in table:
print table
transport.close()
def getMasterTables(hbaseHost):
transport = TBufferedTransport(TSocket(hbaseHost, 9090))
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
client = Hbase.Client(protocol)
for table in client.getTableNames():
if 'master' in table:
print table
transport.close()
def hs2_port_is_open(self):
"""Test if the HS2 port is open. Does not need to authenticate."""
# Impyla will try to authenticate as part of connecting, so preserve previous logic
# that uses the HS2 thrift code directly.
try:
socket = TSocket(self.hostname, self.hs2_port)
transport = TBufferedTransport(socket)
transport.open()
transport.close()
return True
except Exception, e:
LOG.info(e)
return False
def hs2_port_is_open(self):
"""Test if the HS2 port is open. Does not need to authenticate."""
# Impyla will try to authenticate as part of connecting, so preserve previous logic
# that uses the HS2 thrift code directly.
try:
socket = TSocket(self.hostname, self.hs2_port)
transport = TBufferedTransport(socket)
transport.open()
transport.close()
return True
except Exception, e:
LOG.info(e)
return False
class ThriftClient(object):
def __init__(self, port=6458, host="127.0.0.1"):
self.host = host
self.port = port
self.client = None
self.transport = None
def connect(self):
self.transport = TSocket(self.host, self.port)
self.transport = TBufferedTransport(self.transport)
protocol = TBinaryProtocol(self.transport)
self.client = Client(protocol)
self.transport.open()
def execute(self, name, *args, **kwargs):
if self.client is None:
self.connect()
result = getattr(self.client, name)(*args, **kwargs)
return result
def create(self, key, second, minute = -1, hour = -1, day = -1, month = -1, week = -1, action="shell", params={}):
return self.execute("create", key, second, minute, hour, day, month, week, action, params)
def create_timeout(self, key, second, minute = -1, hour = -1, day = -1, month = -1, week = -1, count=1, action="shell", params={}):
return self.execute("createTimeout", key, second, minute, hour, day, month, week, count, action, params)
def remove(self, key):
return self.execute("remove", key)
def get(self, key):
return self.execute("get", key)
def get_current(self):
return self.execute("getCurrent")
def get_time(self, timestamp):
return self.execute("getTime", timestamp)
def get_keys(self, prefix=''):
return self.execute("getKeys", prefix)
def info(self):
return self.execute("info")
def __del__(self):
if self.client:
self.transport.close()
self.transport = None
self.client = None
class ThriftClient(object):
def __init__(self, port=6458, host="127.0.0.1"):
self.host = host
self.port = port
self.client = None
self.transport = None
def connect(self):
self.transport = TSocket(self.host, self.port)
self.transport = TBufferedTransport(self.transport)
protocol = TBinaryProtocol(self.transport)
self.client = Client(protocol)
self.transport.open()
def execute(self, name, *args, **kwargs):
if self.client is None:
self.connect()
result = getattr(self.client, name)(*args, **kwargs)
return result
def create(self, key, second, minute = -1, hour = -1, day = -1, month = -1, week = -1, action="shell", params={}):
return self.execute("create", key, second, minute, hour, day, month, week, action, params)
def create_timeout(self, key, second, minute = -1, hour = -1, day = -1, month = -1, week = -1, count=1, action="shell", params={}):
return self.execute("createTimeout", key, second, minute, hour, day, month, week, count, action, params)
def remove(self, key):
return self.execute("remove", key)
def get(self, key):
return self.execute("get", key)
def get_current(self):
return self.execute("getCurrent")
def get_time(self, timestamp):
return self.execute("getTime", timestamp)
def get_keys(self, prefix=''):
return self.execute("getKeys", prefix)
def __del__(self):
if self.client:
self.transport.close()
self.transport = None
self.client = None
class writeThread(threading.Thread):
def __init__(self, threadname, RecordsThreadwillwrite):
threading.Thread.__init__(self, name = threadname)
bytesPerColumn = int(bytesPerRecord/columns) - 11 #suppose 3 columns
self.columnvalue = "value_" + "x"*bytesPerColumn + "_endv"
self.tbwBatch = int (RecordsThreadwillwrite / recordsPerBatch)
self.transport = TBufferedTransport(TSocket('10.1.2.230', 9090), 40960)
self.transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(self.transport)
self.client = Hbase.Client(protocol)
def run(self):
print "+%s start" % (self.getName())
global gEndT
global gWritenItems
threadWritenItem = 0
for loopidx in xrange(0, self.tbwBatch):
self.write_hbase() #write
threadWritenItem += recordsPerBatch
mylock.acquire()
gEndT = time.time()
gWritenItems += threadWritenItem
print "%s done, %s seconds past, %d reocrds saved" % (self.getName(), gEndT-gStartT, gWritenItems)
mylock.release()
self.transport.close()
def write_hbase(self): #write 50 rowkyes, and 3 column families in each rowkey
print self.getName(), "Start write"
batchmutations = []
for i in xrange(0, recordsPerBatch): # write to db, 300 items together
mutations = []
rowkey = "RK_%s_%s" % (random.random(), time.time())
for ii in xrange(0, columns):
mutations.append(Hbase.Mutation(column="f1:%s"%ii, value=self.columnvalue))
batchmutations.append(Hbase.BatchMutation(rowkey, mutations))
self.client.mutateRows("testdb1", batchmutations)
class MyHbase:
"""
the main class,use to connect to hbase,create ,select table etc.
"""
def __init__(self, netloc, port, table="diracAccounting"):
self.tableName = table
self.transport = TBufferedTransport(TSocket(netloc, port))
self.protocol = TBinaryProtocol(self.transport)
self.client = Hbase.Client(self.protocol)
self.transport.open()
tables = self.client.getTableNames()
# check if has table 'diracAccounting',if has then delete it and then recreate it
for table in tables:
if table == "diracAccounting":
if self.client.isTableEnabled(table):
print "disabling table:%s" % (table)
self.client.disableTable(table)
print "deleting table:%s" % (table)
self.client.deleteTable(table)
# if self.tableName not in tables:
self.__createTable(["groupby", "generate"])
def __del__(self):
self.transport.close()
def __createTable(self, columnfamilyList):
"""argument:
columnfamilyList is a list
the columnfanilyList=['gruopby','generate']
"""
columns = []
for name in columnfamilyList:
col = ColumnDescriptor(name)
columns.append(col)
print "creating tables:%s" % (self.tableName)
try:
self.client.createTable(self.tableName, columns)
except AlreadyExists, ae:
print "WARN: " + ae.message
def main(args):
if (len(args) < 3):
print "%s hbasehost tablename " % args[0]
sys.exit(1)
hbaseHost = args[1]
table_name = args[2]
transport = TBufferedTransport(TSocket(hbaseHost, 9090))
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
global client
client = Hbase.Client(protocol)
updateColumn(table_name)
transport.close()
def main(args):
if (len(args)<3):
print "%s hbasehost tablename " % args[0]
sys.exit(1)
hbaseHost = args[1]
table_name= args[2]
transport = TBufferedTransport(TSocket(hbaseHost, 9090))
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
global client
client = Hbase.Client(protocol)
updateColumn(table_name)
transport.close()
class ServidorArchivos:
host = "localhost"
puerto = 42778
def __init__(self):
socket = TSocket(self.host, self.puerto)
self.transport = TBufferedTransport(socket)
protocolo = TBinaryProtocol(self.transport)
self.conexion = Client(protocolo)
def guardar_archivo(self, archivo: FileStorage, ruta: str):
imagen = Imagen(archivo.read(), ruta)
self.transport.open()
resultado = self.conexion.guardarArchivo(imagen)
self.transport.close()
return resultado
def eliminar_archivo(self, ruta: str):
self.transport.open()
resultado = self.conexion.eliminarArchivo(ruta)
self.transport.close()
return resultado
def obtener_archivos(self, ruta: str) -> list:
print(ruta)
self.transport.open()
resultado = self.conexion.obtenerArchivos(ruta)
self.transport.close()
return resultado
def main(args):
if (len(args) != 2):
print "%s tableName" % (args[0])
sys.exit(1)
tablename = args[1]
getConfiguration('host.properties')
transport = TBufferedTransport(TSocket(hbaseHost, 9090))
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
global client
client = Hbase.Client(protocol)
# tablename = "%s_%s_master_%s"%(orgId,subOrgId,orgId);
client.disableTable(tablename)
client.deleteTable(tablename)
transport.close()
def main(argv):
p = argparse.ArgumentParser()
add_common_args(p)
# Since THeaderTransport acts as framed transport when detected frame, we
# cannot use --transport=framed as it would result in 2 layered frames.
p.add_argument('--override-transport')
args = p.parse_args()
assert args.protocol == 'header'
assert args.transport == 'buffered'
assert not args.ssl
sock = TSocket(args.host, args.port, socket_family=socket.AF_INET)
if not args.override_transport or args.override_transport == 'buffered':
trans = TBufferedTransport(sock)
elif args.override_transport == 'framed':
trans = TFramedTransport(sock)
else:
raise ValueError('invalid transport')
trans.open()
proto = TBinaryProtocol(trans)
proto.writeMessageBegin('testVoid', TMessageType.CALL, 3)
proto.writeStructBegin('testVoid_args')
proto.writeFieldStop()
proto.writeStructEnd()
proto.writeMessageEnd()
trans.flush()
_, mtype, _ = proto.readMessageBegin()
assert mtype == TMessageType.REPLY
proto.readStructBegin()
_, ftype, _ = proto.readFieldBegin()
assert ftype == TType.STOP
proto.readFieldEnd()
proto.readStructEnd()
proto.readMessageEnd()
trans.close()
class MyHbase():
def __init__(self,netloc,port,tablename):
self.tablename = tablename
self.transport = TBufferedTransport(TSocket(netloc,port))
self.protocol = TBinaryProtocol(self.transport)
self.client = Hbase.Client(self.protocol)
self.transport.open()
if self.tablename in self.client.getTableNames():
if not self.client.isTableEnabled(self.tablename):
print 'table off_line, start bring it on_line now'
self.client.enableTable(self.tablename)
print "successfully connect table: %s"%(self.tablename)
print
else:
print "wrong table name"
def __del__(self):
self.transport.close()
def printRow(self,entry):
print "row: " + entry.row + ", cols:",
for k in sorted(entry.columns):
print k + " => " + entry.columns[k].value
print
def showTableInfo(self):
'''
show some information of the hbase table
'''
#print self.client.getColumnDescriptors(self.tablename)
temp = self.client.getColumnDescriptors(self.tablename)
for (col,desc) in temp.items():
print "column with name:"+col
print desc
def generatePlot(self,groupbyname,generatename):
'''
def main(args):
if(len(args) < 2):
print "%s <verified file> -all"%(args[0])
sys.exit(1)
filename = args[1]
opt_all = True if len(args)>2 and args[2] == "-all" else False
filenamearray = filename.split("_")
orgId = filenamearray[0]
subOrgId = filenamearray[1]
getConfiguration('host.properties')
transport = TBufferedTransport(TSocket(hbaseHost, 9090))
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
global client
client = Hbase.Client(protocol)
tablename = "%s_%s_master_%s"%(orgId,subOrgId,orgId);
for line in open(filename,"r"):
input=line.strip()
row=client.getRow(tablename,input)
print input
printRow(row)
print ""
if (not opt_all): break
transport.close()
def main(args):
if(len(args)!=2):
print "%s tableName"%(args[0])
sys.exit(1)
tablename = args[1]
getConfiguration('host.properties')
transport = TBufferedTransport(TSocket(hbaseHost, 9090))
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
global client
client = Hbase.Client(protocol)
# tablename = "%s_%s_master_%s"%(orgId,subOrgId,orgId);
client.disableTable(tablename)
client.deleteTable(tablename)
transport.close()
class HBaseOperator():
def __init__(self):
self.host = "193.169.100.33"
self.port = 2181
self.transport = TBufferedTransport(TSocket(self.host, self.port))
self.transport.open()
self.protocol = TBinaryProtocol.TBinaryProtocol(self.transport)
self.client = Hbase.Client(self.protocol)
def __del__(self):
self.transport.close()
def getAllTablesInfo(self):
#get table info
listTables = self.client.getTableNames()
print("=" * 40)
print("Show all tables information....")
for tableName in listTables:
print("TableName:" + tableName)
listColumns = self.client.getColumnDescriptors(tableName)
print(listColumns)
listTableRegions = self.client.getTableRegions(tableName)
print(listTableRegions)
print("+" * 40)
def main(args):
if len(args) != 3:
print "%s orgId subOrgId" % (args[0])
sys.exit(1)
orgId = args[1]
subOrgId = args[2]
getConfiguration("host.properties")
transport = TBufferedTransport(TSocket(hbaseHost, 9090))
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
global client
client = Hbase.Client(protocol)
tablename = "%s_%s_master_%s" % (orgId, subOrgId, orgId)
connectSolrServer(orgId, subOrgId)
verifyData(tablename)
transport.close()
def main(args):
if (len(args) != 3):
print "%s orgId subOrgId" % (args[0])
sys.exit(1)
orgId = args[1]
subOrgId = args[2]
getConfiguration('host.properties')
transport = TBufferedTransport(TSocket(hbaseHost, 9090))
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
global client
client = Hbase.Client(protocol)
tablename = "%s_%s_master_%s" % (orgId, subOrgId, orgId)
connectSolrServer(orgId, subOrgId)
verifyData(tablename)
transport.close()
class TedUI:
def __init__(self):
self.connected = False
readline.parse_and_bind("tab: complete")
readline.set_completer(self.tab_complete)
readline.parse_and_bind("set bell-style none")
readline.set_completer_delims(" ")
# Get all non-private functions
self.funs = [x for x in Iface.__dict__.keys() if not x[0] == "_"]
self.funs.sort()
self.builtins = {}
self.builtins["connect"] = self.connect
self.builtins["help"] = self.help
def tab_complete(self, text, index):
matches = [word for word in self.funs if word.startswith(text)]
try:
return matches[index]
except IndexError:
return None
def _show_commands(self):
print "Commands"
print "--------"
keys = [k for k in self.builtins if not k[0] == "_"]
keys.sort()
for key in keys:
print key
print "--------"
def connect(self, *args):
self.transport = TBufferedTransport(TSocket("localhost", 9030))
self.transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(self.transport)
self.client = ted.TedService.Client(protocol)
self.connected = True
for fun in self.funs:
self.builtins[fun] = getattr(self.client, fun)
del self.builtins["connect"]
self.builtins["disconnect"] = self.disconnect
def disconnect(self, *args):
for fun in self.funs:
del self.builtins[fun]
self.client.logout()
self.transport.close()
self.client = None
self.connected = False
del self.builtins["disconnect"]
self.builtins["connect"] = self.connect
def help(self, *args):
self._show_commands()
def main(self):
logger = logging.getLogger("ted")
logger.addHandler(logging.StreamHandler())
shell = code.InteractiveConsole(self.builtins)
shell.interact("Welcome to ted. Type 'help()' for a list of commands")
if self.connected:
self.disconnect()
class HBaseClient(object):
def __init__(self, ip, port=9090):
"""
建立与thrift server端的连接
"""
# server端地址和端口设定
self.__transport = TBufferedTransport(TSocket.TSocket(ip, port))
# 设置传输协议
protocol = TBinaryProtocol.TBinaryProtocol(self.__transport)
# 客户端
self.__client = Hbase.Client(protocol)
# 打开连接
self.__transport.open()
def __del__(self):
self.__transport.close()
def get_tables(self):
"""
获得所有表
:return:表名列表
"""
return self.__client.getTableNames()
def create_table(self, table, *columns):
"""
创建表格
:param table:表名
:param columns:列族名
"""
func = lambda col: ColumnDescriptor(col)
column_families = map(func, columns)
self.__client.createTable(table, column_families)
def put(self, table, row, columns):
"""
添加记录
:param table:表名
:param row:行键
:param columns:列名
:return:
"""
func = lambda k, v: Mutation(column=k, value=v)
mutations = map(func, columns.items())
self.__client.mutateRow(table, row, mutations)
def delete(self, table, row, column):
"""
删除记录
:param table:表名
:param row:行键
"""
self.__client.deleteAll(table, row, column)
def scan(self, table, start_row="", columns=None):
"""
获得记录
:param table: 表名
:param start_row: 起始行
:param columns: 列族
:param attributes:
"""
scanner = self.__client.scannerOpen(table, start_row, columns)
func = lambda k, v: (k, v.value)
while True:
r = self.__client.scannerGet(scanner)
if not r:
break
yield dict(map(func, r[0].columns.items()))
class ThriftClientMixin(object):
"""Mixin for all Thrift clients.
Provides the following features:
- Simplified instantiation using host and port
- Opaque singleton instantiation
- Lazy connection
- Ensures client is connected on each call
"""
__metaclass__ = ThriftClientConnectionEnsured
def connect(self, conn_timeout_ms, req_timeout_ms):
"""Connect to the endpoint specified in self.host and self.port.
.. note:: It makes connection only if it's not already connected.
"""
if self.connected:
self._socket.setTimeout(req_timeout_ms)
return
# Socket.
if self.is_ssl:
self._socket = TSSLSocket(self.host, self.port, self.validate, self.ca_certs)
else:
self._socket = TNoDelaySocket(self.host, self.port)
# Set socket timeout
self._socket.setTimeout(conn_timeout_ms)
# Transport.
self._transport = TBufferedTransport(self._socket)
# open() may throw TTransportException() if fail to connect.
self._transport.open()
# Protocol.
self._protocol = self.protocol_factory.getProtocol(self._transport)
# Client.
# Need to get the parent class of the client class
# Ex: <class
# 'data_clients.follower_service_client.FollowerServiceClient'>
# => <class services.follower.thrift_libs.FollowerService.Client>.
self._client = self.__class__.__bases__[0](self._protocol)
self.connected = True
self.connected_at = time.time()
self._socket.setTimeout(req_timeout_ms)
def teardown_connection(self, select_new_host=False):
"""Tear down the connection.
Args:
select_new_host: Boolean, whether to reset self.host and
self.port to a random one from host_provider specified in
the constructor.
"""
if self.connected:
# Close transport, which closes its socket.
self._transport.close()
# Set connected to false such that next connect() call will
# re-establish connection.
self.connected = False
self.connected_at = 0
self._transport = None
self._socket = None
self.requests_served = 0
if select_new_host:
new_host, new_port = self._host_selector.get_host().split(":")
self.host = new_host
self.port = int(new_port)
def _is_connection_old(self):
"""Check whether the current connection age is beyond configured time.
"""
if not self.connected:
return False
return time.time() - self.connected_at > SECS_FOR_CONNECTION_TEARDOWN
def refresh_connection_if_needed(
self, num_requests=NUM_REQUESTS_FOR_CONNECTION_TEARDOWN):
"""Track whether we need connect to a new endpoint.
:param num_requests: number of requests needed for connection to be
torn down and a new endpoint chosen. New endpoint is lazily
established in ``ensure_connection()`` call.
"""
self.requests_served += 1
if (self._is_connection_old() or self.requests_served ==
NUM_REQUESTS_FOR_CONNECTION_TEARDOWN):
self.teardown_connection(select_new_host=True)
class Client (object):
'''
This class handles all communication to and from the backend pyLoad server
instance
Events:
on_connected: Raised when a connection to the backend server is made
on_disconnected: Raised when the connection has been severed
'''
on_connected = None
on_disconnected = None
def __init__ (self):
'''
Constructor
'''
self.__connected = False
self.on_connected = Event()
self.on_disconnected = Event()
# set all live properties in this class for polling
self.properties_to_poll = [
self.links_active,
self.links_waiting,
self.links_total,
self.speed,
]
self.tasks = {}
def connect (self, host, port, username, password):
'''
Connect to the specified backend
'''
self.__connected = False
self.host = host
self.port = port
self.username = username
self.password = password
self.socket = TSocket (host, port)
self.transport = TBufferedTransport (self.socket)
self.protocol = TBinaryProtocol (self.transport)
self.client = Pyload.Client (self.protocol)
try:
self.transport.open()
self.__connected = True
logging.info ("Connected to {0}:{1}".format(host, port))
except:
logging.warn ("Failed to connect to {0}:{1}".format(host, port))
raise ConnectionFailed
if self.client.login (username, password):
logging.info ("Server version: {0}".format(self.version))
self.on_connected()
return True
return False
@login_required
def disconnect (self):
'''
Disconnect and release the resources from the backend
'''
self.transport.close()
self.__connected = False
self.client = None
self.protocol = None
self.transport = None
self.socket = None
self.host = None
self.port = None
self.username = None
self.password = None
logging.info ("Disconnected from {0}:{1}".format(self.host, self.port))
self.on_disconnected()
@property
def connected (self):
'''
Whether or not the client is connected to a server
'''
return self.__connected
@property
@login_required
def version (self):
'''
Get the version of the server
'''
return self.client.getServerVersion()
@property
@login_required
def free_space (self):
'''
The amount of free space in the download directory
'''
return self.client.freeSpace()
@live_property
@login_required
def links_active (self):
'''
The amount of links currently active
'''
status = self.client.statusServer()
return status.active
@live_property
@login_required
def links_waiting (self):
'''
The amount of links currently waiting
'''
status = self.client.statusServer()
return status.queue
@live_property
@login_required
def links_total (self):
'''
The total amount of links in the queue
'''
status = self.client.statusServer()
return status.total
@live_property
@login_required
def speed (self):
'''
The current download speed in bytes
'''
status = self.client.statusServer()
return status.speed
@property
@login_required
def paused (self):
'''
Whether or not the queue is currently paused
'''
status = self.client.statusServer()
return status.pause
@property
@login_required
def download_enabled (self):
'''
Whether or not downloading is enabled
'''
status = self.client.statusServer()
return status.download
@property
@login_required
def reconnect_enabled (self):
'''
Whether or not the downloader will reconnect when needed
'''
status = self.client.statusServer()
return status.reconnect
@login_required
def get_user_details (self):
'''
Get the details of the currently logged in user
'''
data = self.client.getUserData (self.username, self.password)
return {'name': data.name, 'email': data.email, 'role': data.role, 'permission': data.permission}
@login_required
def pause (self):
'''
Pause the current queue. This will only stop processing of the queue,
any links downloading will continue to do so
'''
self.client.pauseServer()
@login_required
def resume (self):
'''
Resume the current queue
'''
self.client.unpauseServer()
@live_property
@login_required
def collector (self):
'''
A dict of packages currently in the collector
'''
packages = {}
for item in self.client.getCollectorData():
packages[item.pid] = Package (item)
return packages
@live_property
@login_required
def queue (self):
'''
A dict of packages currently in the queue
'''
packages = {}
for item in self.client.getQueueData():
packages[item.pid] = Package (item)
return packages
@live_property
@login_required
def downloads (self):
'''
A dict of all active downloads
'''
downloads = {}
for item in self.client.statusDownloads():
downloads[item.fid] = Download (item)
return downloads
@live_property
@login_required
def captchas (self):
'''
The current captcha tasks waiting to be actioned
'''
captchas = {}
if self.client.isCaptchaWaiting():
task = self.client.getCaptchaTask (False)
captchas[task.tid] = task
return captchas
def restart_link (self, link):
'''
Restarts the link so that it can be downloaded again
'''
self.client.restartFile (link.id)
def abort_link (self, link):
'''
Aborts the link
'''
self.client.stopDownloads ([link.id])
def queue_package (self, package):
'''
Adds the specified package to the queue
'''
self.client.pushToQueue (package.id)
def check_online_status (self, links):
'''
Queues a task to check the online status of the specified link items.
The results will automatically be set on the link item
'''
# add the link to the results data cache
items = {}
for link in links:
items[link.url] = link
# send the check request
urls = items.keys()
results = self.client.checkOnlineStatus (urls)
self.tasks[results.rid] = items
def clear_finished (self):
'''
Clears all the finished packages and removes them from the list.
'''
ids = self.client.deleteFinished()
def answer_captcha (self, captcha, answer):
'''
Set the answer to a captcha challenge
'''
self.client.setCaptchaResult (captcha.tid, answer)
def poll_tasks (self):
'''
Poll the backend for results on a previously queued online check task
'''
finished = []
for rid, links in self.tasks.iteritems():
results = self.client.pollResults (rid)
if results.rid == -1:
finished.append (rid)
for url, result in results.data.iteritems():
link = links[url]
link.name = result.name
link.size = result.size
link.plugin = result.plugin
link.status = Link.Status (result.status)
for rid in finished:
del self.tasks[rid]
def poll (self):
'''
Poll the backend server. This will check all the 'live' properties for
any changes and raise the property_changed event
'''
for prop in self.properties_to_poll:
prop.update()
self.collector.update()
self.queue.update()
self.downloads.update()
self.captchas.update()
self.poll_tasks()
return True
class ESIndexCreator:
def __init__(self):
reComments = re.compile('<!--[^>]*-->')
reHtml = re.compile('</?\w+[^>]*>')
self.host = "172.20.6.61"
self.port = 9090
self.transport = TBufferedTransport(TSocket(self.host, self.port))
self.transport.open()
self.protocol = TBinaryProtocol.TBinaryProtocol(self.transport)
self.client = Hbase.Client(self.protocol)
self.es = Elasticsearch(
['172.20.8.162'],
sniff_on_start = True,
sniff_on_connection_fail = True,
sniffer_timeout = 60
)
def __del__(self):
self.transport.close()
def filterTags(self,htmlstr):
#先过滤CDATA
re_cdata=re.compile('//<!\[CDATA\[[^>]*//\]\]>',re.I) #匹配CDATA
re_script=re.compile('<\s*script[^>]*>[^<]*<\s*/\s*script\s*>',re.I)#Script
re_style=re.compile('<\s*style[^>]*>[^<]*<\s*/\s*style\s*>',re.I)#style
re_br=re.compile('<br\s*?/?>')#处理换行
re_h=re.compile('</?\w+[^>]*>')#HTML标签
re_comment=re.compile('<!--[^>]*-->')#HTML注释
s=re_cdata.sub('',htmlstr)#去掉CDATA
s=re_script.sub('',s) #去掉SCRIPT
s=re_style.sub('',s)#去掉style
s=re_br.sub('',s)#将br转换为换行
s=re_h.sub('',s) #去掉HTML 标签
s=re_comment.sub('',s)#去掉HTML注释
#去掉多余的空行
blank_line=re.compile('\n+')
s=blank_line.sub('',s)
return s
def getAllTablesInfo(self):
#get table info
listTables = self.client.getTableNames()
print "="*40
print "Show all tables information...."
for tableName in listTables:
print "TableName:" + tableName
print " "
listColumns = self.client.getColumnDescriptors(tableName)
print listColumns
print " "
listTableRegions = self.client.getTableRegions(tableName)
print listTableRegions
print "+"*40
def deleteIndex(self,indexName):
print "Remove index...."
self.es.indices.delete(index=indexName,ignore=[400,404])
def createIndex(self,indexName):
print "Create index...."
#create indexs
self.es.indices.create(index=indexName)
print "Define mapping...."
#define mapping
self.es.indices.put_mapping(
index=indexName,
doc_type="article",
ignore_conflicts='true',
body={
"article":{
"properties":{
"sitename":{ "type":"string", "store":"true", "index":"not_analyzed" },
"addtime":{"type":"date", "store":'true' },
"publishtime":{"type":"date", "store":'true' },
"keywords":{"type":"string", "store":'true',"analyzer":"ik" },
"content":{"type":"string", "store":'true',"analyzer":"ik" },
"url":{"type":"string", "store":'true', "index":"not_analyzed"},
"title":{"type":"string","store":'true', "analyzer":"ik"}
}
}
}
)
self.es.indices.put_mapping(
index=indexName,
doc_type="baidu",
ignore_conflicts='true',
body={
"baidu":{
"properties":{
"sitename":{ "type":"string", "store":"true", "index":"not_analyzed" },
"addtime":{"type":"date", "store":'true' },
"publishtime":{"type":"date", "store":'true' },
"keywords":{"type":"string", "store":'true',"analyzer":"ik" },
"content":{"type":"string", "store":'true',"analyzer":"ik" },
"url":{"type":"string", "store":'true', "index":"not_analyzed"},
"title":{"type":"string","store":'true', "analyzer":"ik"}
}
}
}
)
self.es.indices.put_mapping(
index=indexName,
doc_type="report",
ignore_conflicts='true',
body={
"report":{
"properties":{
"sitename":{ "type":"string", "store":"true", "index":"not_analyzed" },
"addtime":{"type":"date", "store":'true' },
"publishtime":{"type":"date", "store":'true' },
"infSource":{"type":"string", "store":'true',"index":"not_analyzed" },
"url":{"type":"string", "store":'true', "index":"not_analyzed"},
"title":{"type":"string","store":'true', "analyzer":"ik"}
}
}
}
)
self.es.indices.put_mapping(
index=indexName,
doc_type="blog",
ignore_conflicts='true',
body={
"blog":{
"properties":{
"sitename":{ "type":"string", "store":"true", "index":"not_analyzed" },
"addtime":{"type":"date", "store":'true' },
"author":{"type":"string", "store":'true', "index":"not_analyzed" },
"content":{"type":"string", "store":'true',"analyzer":"ik" },
"url":{"type":"string", "store":'true', "index":"not_analyzed"},
"title":{"type":"string","store":'true', "analyzer":"ik"}
}
}
}
)
self.es.indices.put_mapping(
index=indexName,
doc_type="weibo",
ignore_conflicts='true',
body={
"weibo":{
"properties":{
"user_id":{ "type":"string", "store":"true"},
"screen_name":{"type":"string", "store":'true', "index":"not_analyzed" },
"content":{"type":"string", "store":'true',"analyzer":"ik" },
"publishtime":{"type":"date", "store":'true' },
"comments_count":{"type":"integer", "store":'true'},
"reposts_count":{"type":"integer","store":'true'}
}
}
}
)
def getHBaseList(self,tableName,listColumns):
scanner = self.client.scannerOpen(tableName,'',listColumns,None)
res = self.client.scannerGetList(scanner,100000)
self.client.scannerClose(scanner)
return res
'''
def createDataIndex(self,tableName,tableColumnFamily,tableColumnList,indexName,indexTypeName,indexColumnList):
listColumns = []
for key in range(len(tableColumnList)):
listColumns.append(tableColumnFamily + ':' + tableColumnList[key])
res = self.getHBaseList(tableName,listColumns)
for i in res:
body = {}
for key in range(len(listColumns)):
if indexColumnList[key] == 'addtime':
body[indexColumnList[key]] = datetime.datetime.strptime(i.columns.get(listColumns[key]).value,'%Y-%m-%d %H:%M:%S')
elif indexColumnList[key] == 'publishtime':
if i.columns.get(listColumns[key]).value == '':
body[indexColumnList[key]] = datetime.datetime.now()
elif len((i.columns.get(listColumns[key]).value).strip()) <= 10:
body[indexColumnList[key]] = datetime.datetime.strptime((i.columns.get(listColumns[key]).value).strip() + ' 00:00:00','%Y-%m-%d %H:%M:%S')
else:
body[indexColumnList[key]] = datetime.datetime.strptime(i.columns.get(listColumns[key]).value,'%Y-%m-%d %H:%M:%S')
elif indexColumnList[key] == 'content':
#body[indexColumnList[key]] = self.filterTags(i.columns.get(listColumns[key]).value).strip()
body[indexColumnList[key]] = ''
else:
body[indexColumnList[key]] = i.columns.get(listColumns[key]).value
self.es.index(index=indexName,doc_type=indexTypeName, body=body)
'''
def createOtherArticlesIndex(self,indexName):
print "Create other articles index...."
tableColumnList = ['siteName','publishTime','url','title','keyWords','content','addTime']
indexColumnList = ['sitename','publishtime','url','title','keywords','content','addtime']
self.createDataIndex('info_public_monitor','other_articles',tableColumnList,indexName,'article',indexColumnList)
def createBaiduArticlesIndex(self,indexName):
print "Create baidu articles index...."
tableColumnList = ['siteName','publishTime','url','title','keyWords','content','addTime']
indexColumnList = ['sitename','publishtime','url','title','keywords','content','addtime']
self.createDataIndex('info_public_monitor','baidu_articles',tableColumnList,indexName,'baidu',indexColumnList)
def createReportIndex(self,indexName):
print "Create report index...."
tableColumnList = ['siteName','publishTime','url','title','source','addTime']
indexColumnList = ['sitename','publishtime','url','title','infsource','addtime']
self.createDataIndex('info_public_monitor','report',tableColumnList,indexName,'report',indexColumnList)
def createBlogIndex(self,indexName):
print "Create blog index...."
tableColumnList = ['siteName','author','url','title','content','addTime']
indexColumnList = ['sitename','author','url','title','content','addtime']
self.createDataIndex('info_public_monitor','blog',tableColumnList,indexName,'blog',indexColumnList)
def createWeiboIndex(self,indexName):
print "Create weibo index...."
tableColumnList = ['user_id','created_at','content','screen_name','comments_count','reposts_count']
indexColumnList = ['user_id','publishtime','content','screen_name','comments_count','reposts_count']
self.createDataIndex('info_public_monitor','weibo',tableColumnList,indexName,'weibo',indexColumnList)
def createActivityIndex(self,indexName):
print "Create activity index...."
tableColumnList = ['activityID','addTime','keyWords','location','siteName','time','title','trad','url']
indexColumnList = ['activityid','addtime','keywords','location','sitename','time','title','trad','url']
self.createDataIndex('info_public_monitor','activity',tableColumnList,indexName,'activity',indexColumnList)
def createAllIndex(self,indexName):
self.deleteIndex(indexName)
self.createIndex(indexName)
#self.createBaiduArticlesIndex(indexName)
self.createOtherArticlesIndex(indexName)
#self.createReportIndex(indexName)
#self.createBlogIndex(indexName)
#self.createWeiboIndex(indexName)
#self.createActivityIndex(indexName)
def createDataIndex(self,tableName,tableColumnFamily,tableColumnList,indexName,indexTypeName,indexColumnList):
listColumns = []
for key in range(len(tableColumnList)):
listColumns.append(tableColumnFamily + ':' + tableColumnList[key])
scannerId = self.client.scannerOpen(tableName, '', listColumns, None)
actions = []
while True:
try:
results = self.client.scannerGet(scannerId)
i = results[0]
except:
break
body = {}
for key in range(len(listColumns)):
if indexColumnList[key] == 'addtime':
body[indexColumnList[key]] = datetime.datetime.strptime(i.columns.get(listColumns[key]).value,'%Y-%m-%d %H:%M:%S')
elif indexColumnList[key] == 'publishtime':
if i.columns.get(listColumns[key]).value == '':
body[indexColumnList[key]] = datetime.datetime.now()
elif len((i.columns.get(listColumns[key]).value).strip()) <= 10:
body[indexColumnList[key]] = datetime.datetime.strptime((i.columns.get(listColumns[key]).value).strip() + ' 00:00:00','%Y-%m-%d %H:%M:%S')
else:
body[indexColumnList[key]] = datetime.datetime.strptime(i.columns.get(listColumns[key]).value,'%Y-%m-%d %H:%M:%S')
elif indexColumnList[key] == 'content':
body[indexColumnList[key]] = self.filterTags(i.columns.get(listColumns[key]).value).strip()
else:
body[indexColumnList[key]] = i.columns.get(listColumns[key]).value
#self.es.index(index=indexName,doc_type=indexTypeName, body=body)
action = {
'_index':indexName,
'_type':indexTypeName,
'_source':body
}
actions.append(action)
if(len(actions) == 500):
print 'creating indexes....'
helpers.bulk(self.es,actions)
del actions[0:len(actions)]
if(len(actions) > 0):
helpers.bulk(self.es, actions)
del actions[0:len(actions)]
self.client.scannerClose(scannerId)
class ThriftClientMixin(object):
"""Mixin for all Thrift clients.
Provides the following features:
- Simplified instantiation using host and port
- Opaque singleton instantiation
- Lazy connection
- Ensures client is connected on each call
"""
__metaclass__ = ThriftClientConnectionEnsured
def connect(self, conn_timeout_ms, req_timeout_ms):
"""Connect to the endpoint specified in self.host and self.port.
.. note:: It makes connection only if it's not already connected.
"""
if self.connected:
self._socket.setTimeout(req_timeout_ms)
return
# Socket.
if self.is_ssl:
self._socket = TSSLSocket(self.host, self.port, self.validate,
self.ca_certs)
else:
self._socket = TNoDelaySocket(self.host, self.port)
# Set socket timeout
self._socket.setTimeout(conn_timeout_ms)
# Transport.
self._transport = TBufferedTransport(self._socket)
# open() may throw TTransportException() if fail to connect.
self._transport.open()
# Protocol.
self._protocol = self.protocol_factory.getProtocol(self._transport)
# Client.
# Need to get the parent class of the client class
# Ex: <class
# 'data_clients.follower_service_client.FollowerServiceClient'>
# => <class services.follower.thrift_libs.FollowerService.Client>.
self._client = self.__class__.__bases__[0](self._protocol)
self.connected = True
self.connected_at = time.time()
self._socket.setTimeout(req_timeout_ms)
def teardown_connection(self, select_new_host=False):
"""Tear down the connection.
Args:
select_new_host: Boolean, whether to reset self.host and
self.port to a random one from host_provider specified in
the constructor.
"""
if self.connected:
# Close transport, which closes its socket.
self._transport.close()
# Set connected to false such that next connect() call will
# re-establish connection.
self.connected = False
self.connected_at = 0
self._transport = None
self._socket = None
self.requests_served = 0
if select_new_host:
new_host, new_port = self._host_selector.get_host().split(":")
self.host = new_host
self.port = int(new_port)
def _is_connection_old(self):
"""Check whether the current connection age is beyond configured time.
"""
if not self.connected:
return False
return time.time() - self.connected_at > SECS_FOR_CONNECTION_TEARDOWN
def refresh_connection_if_needed(
self, num_requests=NUM_REQUESTS_FOR_CONNECTION_TEARDOWN):
"""Track whether we need connect to a new endpoint.
:param num_requests: number of requests needed for connection to be
torn down and a new endpoint chosen. New endpoint is lazily
established in ``ensure_connection()`` call.
"""
self.requests_served += 1
if (self._is_connection_old() or self.requests_served
== NUM_REQUESTS_FOR_CONNECTION_TEARDOWN):
self.teardown_connection(select_new_host=True)
client.mutateRow(table_name,newrowid,mutatelist)
break
def main(args):
rows = getRowsLimit('lyris_uptilt_master_lyris',100)
for row in rows:
keys = row.row.split('|')
#print keys[1]
uuids.append(keys[1])
table_name='email_by_hour'
# updateUUID(table_name)
# client.mutateRow(table_name,'[email protected]|079678-13795157-000',
# [Hbase.Mutation(column='master_info_cf:webSiteType',value='Commercial')])
if __name__ == "__main__":
main(sys.argv)
transport.close()
class Connection(object):
"""Connection to an HBase Thrift server.
The `host` and `port` parameters specify the host name and TCP port of the
HBase Thrift server to connect to. If omitted or ``None``, a connection to
the default port on ``localhost`` is made.
If `autoconnect` is `True` (the default) the connection is made directly,
otherwise :py:meth:`Connection.open` must be called explicitly before first
use.
The optional `table_prefix` and `table_prefix_separator` arguments specify
a prefix and a separator string to be prepended to all table names, e.g.
when :py:meth:`Connection.table` is invoked. For example, if `table_prefix`
is ``myproject``, all tables tables will have names like ``myproject_XYZ``.
The optional `compat` parameter sets the compatibility level for this
connection. Older HBase versions have slightly different Thrift interfaces,
and using the wrong protocol can lead to crashes caused by communication
errors, so make sure to use the correct one. This value can be either
'0.92' (the default) for use with HBase 0.92.x and later versions, or
'0.90' for use with HBase 0.90.x.
:param str host: The host to connect to
:param int port: The port to connect to
:param bool autoconnect: Whether the connection should be opened directly.
:param str compat: Compatibility mode (optional)
:param str table_prefix: Prefix used to construct table names (optional)
:param str table_prefix_separator: Separator used for `table_prefix`
"""
def __init__(self, host=DEFAULT_HOST, port=DEFAULT_PORT, autoconnect=True,
compat='0.92', table_prefix=None, table_prefix_separator='_'):
# Allow host and port to be None, which may be easier for
# applications wrapping a Connection instance.
self.host = host or DEFAULT_HOST
self.port = port or DEFAULT_PORT
if compat not in COMPAT_MODES:
raise ValueError("'compat' must be one of %s"
% ", ".join(COMPAT_MODES))
if table_prefix is not None and not isinstance(table_prefix, basestring):
raise TypeError("'table_prefix' must be a string")
if not isinstance(table_prefix_separator, basestring):
raise TypeError("'table_prefix_separator' must be a string")
self.compat = compat
self.table_prefix = table_prefix
self.table_prefix_separator = table_prefix_separator
socket = TSocket(self.host, self.port)
framed_transport = False # TODO: make parameter, add docs
if framed_transport:
self.transport = TFramedTransport(socket)
else:
self.transport = TBufferedTransport(socket)
protocol = TBinaryProtocol.TBinaryProtocolAccelerated(self.transport)
self.client = Hbase.Client(protocol)
if autoconnect:
self.open()
self._initialized = True
def _table_name(self, name):
"""Constructs a table name by optionally adding a table name prefix."""
if self.table_prefix is None:
return name
return self.table_prefix + self.table_prefix_separator + name
def open(self):
"""Opens the underlying transport to the HBase instance.
This method opens the underlying Thrift transport (TCP connection).
"""
logger.debug("Opening Thrift transport to %s:%d", self.host, self.port)
self.transport.open()
def close(self):
"""Closes the underyling transport to the HBase instance.
This method closes the underlying Thrift transport (TCP connection).
"""
logger.debug("Closing Thrift transport to %s:%d", self.host, self.port)
self.transport.close()
def __del__(self):
try:
self._initialized
except AttributeError:
# Failure from constructor
return
else:
self.close()
def table(self, name, use_prefix=True):
"""Returns a table object.
Returns a :py:class:`happybase.Table` instance for the table named
`name`. This does not result in a round-trip to the server, and the
table is not checked for existence.
The optional `use_prefix` parameter specifies whether the table prefix
(if any) is prepended to the specified `name`. Set this to `False` if
you want to use a table that resides in another ‘prefix namespace’,
e.g. a table from a ‘friendly’ application co-hosted on the same HBase
instance. See the `table_prefix` parameter to the
:py:class:`Connection` constructor for more information.
:param str name: the name of the table
:param bool use_prefix: whether to use the table prefix (if any)
:return: Table instance
:rtype: :py:class:`Table`
"""
if use_prefix:
name = self._table_name(name)
return Table(name, self)
#
# Table administration and maintenance
#
def tables(self):
"""Returns a list of table names available in this HBase instance.
If a `table_prefix` was set for this :py:class:`Connection`, only
tables that have the specified prefix will be listed.
:return: The table names
:rtype: List of strings
"""
names = self.client.getTableNames()
# Filter using prefix, and strip prefix from names
if self.table_prefix is not None:
prefix = self._table_name('')
offset = len(prefix)
names = [n[offset:] for n in names if n.startswith(prefix)]
return names
def create_table(self, name, families):
"""Creates a table.
:param str name: The table name
:param dict families: The name and options for each column family
The `families` parameter is a dictionary mapping column family names to
a dictionary containing the options for this column family. See
ColumnDescriptor in the Thrift API for the supported options, but note
that the names should be provided in Python style, and not in camel
case notation, for example `time_to_live` (not `timeToLive`) and
`max_versions` (not `maxVersions`).
"""
name = self._table_name(name)
if not isinstance(families, dict):
raise TypeError("'families' arg must be a dictionary")
if not families:
raise ValueError(
"Cannot create table %r (no column families specified)"
% name)
column_descriptors = []
for cf_name, options in families.iteritems():
if options is None:
options = dict()
kwargs = dict()
for option_name, value in options.iteritems():
kwargs[pep8_to_camel_case(option_name)] = value
if not cf_name.endswith(':'):
cf_name += ':'
kwargs['name'] = cf_name
column_descriptors.append(ColumnDescriptor(**kwargs))
self.client.createTable(name, column_descriptors)
def delete_table(self, name):
"""Deletes the specified table.
:param str name: The table name
"""
name = self._table_name(name)
self.client.deleteTable(name)
def enable_table(self, name):
"""Enables the specified table.
:param str name: The table name
"""
name = self._table_name(name)
self.client.enableTable(name)
def disable_table(self, name):
"""Disables the specified table.
:param str name: The table name
"""
name = self._table_name(name)
self.client.disableTable(name)
def is_table_enabled(self, name):
"""Returns whether the specified table is enabled.
:param str name: The table name
:return: whether the table is enabled
:rtype: bool
"""
name = self._table_name(name)
return self.client.isTableEnabled(name)
def compact_table(self, name, major=False):
"""Compacts the specified table.
:param str name: The table name
:param bool major: Whether to perform a major compaction.
"""
name = self._table_name(name)
if major:
self.client.majorCompact(name)
else:
self.client.compact(name)
class TedUI():
def __init__(self):
self.connected = False
readline.parse_and_bind('tab: complete')
readline.set_completer(self.tab_complete)
readline.parse_and_bind('set bell-style none')
readline.set_completer_delims(' ')
# Get all non-private functions
self.funs = [x for x in Iface.__dict__.keys() if not x[0] == '_']
self.funs.sort()
self.builtins = {}
self.builtins['connect'] = self.connect
self.builtins['help'] = self.help
def tab_complete(self, text, index):
matches = [word for word in self.funs if word.startswith(text)]
try:
return matches[index]
except IndexError:
return None
def _show_commands(self):
print "Commands"
print "--------"
keys = [k for k in self.builtins if not k[0] == '_']
keys.sort()
for key in keys:
print key
print "--------"
def connect(self, *args):
self.transport = TBufferedTransport(TSocket('localhost', 9030))
self.transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(self.transport)
self.client = ted.TedService.Client(protocol)
self.connected = True
for fun in self.funs:
self.builtins[fun] = getattr(self.client, fun)
del self.builtins['connect']
self.builtins['disconnect'] = self.disconnect
def disconnect(self, *args):
for fun in self.funs:
del self.builtins[fun]
self.client.logout()
self.transport.close()
self.client = None
self.connected = False
del self.builtins['disconnect']
self.builtins['connect'] = self.connect
def help(self, *args):
self._show_commands()
def main(self):
logger = logging.getLogger('ted')
logger.addHandler(logging.StreamHandler())
shell = code.InteractiveConsole(self.builtins)
shell.interact("Welcome to ted. Type 'help()' for a list of commands")
if self.connected:
self.disconnect()
protocol = TBinaryProtocol(transporte)
conexion = Client(protocol)
continuarMenu = True
while continuarMenu:
print("Digite el primer numero\n")
numero1 = int(input())
print("Digite el segundo numero\n")
numero2 = int(input())
print("Que operacion desea realizar?\n" "[+, -, *, /]")
operador = str(input())
if operador == "+":
transporte.open()
resultado = conexion.sumar(numero1, numero2)
transporte.close()
elif operador == "-":
transporte.open()
resultado = conexion.restar(numero1, numero2)
transporte.close()
elif operador == "*":
transporte.open()
resultado = conexion.multiplicar(numero1, numero2)
transporte.close()
elif operador == "/":
transporte.open()
resultado = conexion.dividir(numero1, numero2)
transporte.close()
else:
print("No es un operador valido")
