def write(self, buf):
if len(self._uid) != UID_LEN:
show_error(self, 'invalid uid')
return
if self._key:
flg = FLG_SEC
else:
flg = 0
cnt = 0
length = len(buf)
total = (length + PACKET_LEN - 1) / PACKET_LEN
head = self._uid + struct.pack('I', flg) + struct.pack('I', total)
self._sock.sendall(head)
while cnt < total:
start = cnt * PACKET_LEN
end = min(start + PACKET_LEN, length)
if self._key:
body = rsa.encrypt(buf[start:end], self._key)
else:
body = buf[start:end]
head = struct.pack('H', len(body))
self._sock.sendall(head)
self._sock.sendall(body)
cnt += 1
self._sock.recv(1)
def download(self, package, version):
self._print('start to download')
try:
if SHOW_TIME:
start_time = datetime.utcnow()
addr = self._get_repo(package)
rpcclient = RPCClient(addr, REPOSITORY_PORT)
if not version:
version = rpcclient.request('version', package=package)
if not version:
show_error(
self,
'failed to download, invalid version, package=%s, ' %
str(package))
return
ret = rpcclient.request('download',
package=package,
version=version)
if ret:
if SHOW_TIME:
self._print('download, time=%d sec' %
(datetime.utcnow() - start_time).seconds)
if DEBUG:
self._download_cnt += 1
self._print('download, count=%d' % self._download_cnt)
return ret
except:
show_error(self, 'failed to download')
def download(self, package, version):
self._print('start to download, package=%s, version=%s' %
(str(package), str(version)))
try:
if SHOW_TIME:
start_time = datetime.utcnow()
addr = self._get_addr(package)
uid, ver = self._db.get_version(package)
if not version:
version = ver
if not self._db.has_package(uid, package, version):
show_error(
self,
'failed to download, invalid version, package=%s, version=%s'
% (str(package), str(version)))
return
if SHOW_TIME:
self._print('download, time=%d sec' %
(datetime.utcnow() - start_time).seconds)
res = self._client.download(addr, self._port, package, version)
if res:
if DEBUG:
self._download_cnt += 1
self._print('download, count=%d' % self._download_cnt)
return res
except:
show_error(self, 'failed to download')
def upload(self, buf, uid, package, version, typ, key):
ret = upload_package(buf, uid, package, version, typ, key)
if ret:
return '%s upload %s-%s successfully' % (str(uid), str(package), str(version))
else:
show_error(self, 'failed to upload, uid=%s, package=%s, version=%s, typ=%s' % (str(uid), str(package), str(version), str(typ)))
return False
def __init__(self, addr, port):
RPCServer.__init__(self, addr, port)
len_up = len(SERVER_UPLOADER)
len_repo = len(SERVER_REPOSITORY)
if len_up < len_repo or len_up % len_repo != 0:
show_error(self, 'failed to initialize')
raise Exception('failed to initialize')
addr = localhost()
if addr not in SERVER_REPOSITORY:
show_error(self, 'failed to initialize')
raise Exception('failed to initialize ')
for i in range(len(SERVER_REPOSITORY)):
if addr == SERVER_REPOSITORY[i]:
break
total = len_up / len_repo
self._upload_servers = SERVER_UPLOADER[i * total:(i + 1) * total]
self._print('upload_servers=%s' % str(self._upload_servers))
if HDFS:
self._port = HDFS_PORT
self._client = HDFSClient()
else:
self._port = FTP_PORT
self._client = FTPClient()
self._server = FTPServer()
if SERVER_REPODB:
self._db = Database(addr=SERVER_REPODB[0])
else:
self._db = Database(addr=addr)
locks = []
for _ in range(LOCK_MAX):
locks.append(Lock())
self._locks = HashRing(locks)
if DEBUG:
self._upload_cnt = 0
self._download_cnt = 0
def register(self, user, pwd, email):
self._print('register starts')
lock = self._get_lock(user)
lock.acquire()
try:
if SHOW_TIME:
start_time = datetime.utcnow()
uid = self.user.add(user, pwd, email)
if not uid:
show_error(self, 'failed to register, invalid register table')
return False
info = self._alloc_installer(uid)
if SHOW_TIME:
self._print('register, time=%d sec' %
(datetime.utcnow() - start_time).seconds)
if info:
if DEBUG:
self._register_cnt += 1
self._print('register, count=%d' % self._register_cnt)
return True
else:
self.user.remove(user)
show_error(
self, 'failed to register, invalid alloc installer table')
return False
finally:
lock.release()
def get_description(self, package):
self._print('get_descripton starts' )
try:
ret = self._recorder.get_description(package)
if ret:
return ret
except:
show_error(self, 'get_descripton failed')
def install(self, uid, package, version, typ, content):
if typ == APP:
return self._app.install(uid, package, version, content)
elif typ == DRIVER:
return self._driver.install(uid, package, version)
else:
show_error(self,
'failed to install, invalid type, typ=%s' % str(typ))
def check_installer(self, addr):
self._print('add_installer starts!')
try:
coll = self._get_collection(TABLE_INST_ADDR)
info = coll.find_one({'addr':addr})
if info:
return True
except:
show_error(self, 'have not added installer %s' % addr)
def get_name(self, uid):
self._print('get_name->uid=%s' % str(uid))
try:
coll = self._get_collection(uid, TABLE_USERINFO)
res = coll.find_one({'uid': uid}, {'user': 1, '_id': 0})
if res:
return res.get('user')
except:
show_error(self, 'failed to get name')
def has_version(self, uid, package, version, table):
self._print('has_version->uid=%s, package=%s, version=%s' % (str(uid), str(package), str(version)))
if not version:
show_error(self, 'failed to has version, the %s has no %s version' % (str(package), str(version)))
return
coll = self._get_collection(table)
res = coll.find_one({'package': package}, {'uid': 1, 'version': 1, '_id': 0})
if res and res.get('uid') == uid and res.get('version') == version:
return True
def _alloc_installer(self, uid):
self._print('alloc_installer->uid=%s' % str(uid))
addr = self._get_allocator(uid)
rpcclient = RPCClient(addr, ALLOCATOR_PORT)
if rpcclient.request('alloc_installer', uid=uid):
return True
else:
show_error(self, 'failed to allocate installer')
return False
def install(self, uid, package, version):
addr = self._get_repo(package)
rpcclient = RPCClient(addr, REPOSITORY_PORT)
if not version:
version = rpcclient.request('version', package=package)
if not version:
show_error(self, 'failed to install, invalid version, uid=%s, package=%s' % (uid, package))
return
ret = rpcclient.request('download', package=package, version=version)
return ret
def get_top(self, category):
self._print('get_top starts, category=%s' %str(category))
try:
ret = self._recorder.get_top(category)
if ret:
return ret
else:
return ''
except:
show_error(self, 'get_top failed')
def get_password(self, user):
self._print('get_password->user=%s' % str(user))
try:
uid = get_uid(user)
coll = self._get_collection(uid, TABLE_USERINFO)
res = coll.find_one({'user': user}, {'password': 1, '_id': 0})
if res:
return res.get('password')
except:
show_error(self, 'failed to get password')
def remove(self, user):
self._print('remove starts')
try:
uid = get_uid(user)
coll = self._get_collection(uid, TABLE_USERINFO)
res = coll.find_one({'user': user}, {'uid': 1})
if res:
coll.remove(res['_id'])
return True
except:
show_error(self, 'failed to get public key')
def get_package_detail(self, package):
self._print('get_package_detail starts')
try:
inst, title = self._recorder.get_package_detail(package)
auth = self.get_author(package)
if not auth:
show_error(self, 'get_package_detail failed, no author')
return
return {'inst':inst, 'auth':auth, 'title':title}
except:
show_error(self, 'get_package_detail failed')
def has_version(self, uid, package, version, table):
self._print('has_version->uid=%s, package=%s, version=%s' % (str(uid), str(package), str(version)))
path = self._get_path(table)
info = shelve.open(path)
try:
if not info:
show_error(self, 'failed to has version, invalid information, package=%s, version=%s' % (str(package), str(version)))
return
if info.has_key(uid) and info[uid].has_key(package) and info[uid][package] == version:
return True
finally:
info.close()
def get_author(self, package):
self._print('get_author starts, package=%s' % str(package))
try:
uid = self._recorder.get_uid(package)
if uid:
addr = self._get_backend()
rpcclient = RPCClient(addr, BACKEND_PORT)
name = rpcclient.request('get_name', uid=uid)
if name:
return str(name)
except:
show_error(self, 'get_author failed')
def readall(self):
uid = self._recv(UID_LEN)
if len(uid) != UID_LEN:
show_error(self, 'failed to receive uid')
return (None, None, '')
buf = self._recv(FLG_LEN)
if len(buf) != FLG_LEN:
show_error(self, 'failed to receive flag')
return (None, None, '')
flg = struct.unpack('I', buf)[0]
buf = self.read()
return (uid, flg, buf)
def uninstall(self, uid, package, typ):
self._print('start to uninstall')
addr = self._get_installer(uid)
rpcclient = RPCClient(addr, INSTALLER_PORT)
res = rpcclient.request('uninstall', uid=uid, package=package, typ=typ)
if not res:
show_error(self, 'failed to uninstall')
return
if DEBUG:
self._uninstall_cnt += 1
self._print('uninstall, count=%d' % self._uninstall_cnt)
return res
def uninstall(self, uid, package):
self._lock.acquire()
try:
if not self._db.has_package(uid, package, None):
show_error(self, 'failed to uninstall %s' % package)
return
version, info = self._db.get_package(uid, package, None)
if info:
self._uninstall(uid, package, info)
self._db.rm_package(uid, package, version)
return True
finally:
self._lock.release()
def add_installer(self, addr):
self._print('add_installer starts!')
try:
coll = self._get_collection(TABLE_INST_ADDR)
info = coll.find_one({'addr':addr})
if info:
show_error(self, 'failed to add installer, the address has been added')
return
if len(addr.split('.')) != 4:
show_error(self, 'failed to add installer, invalid installer address')
return
coll = self._get_collection(TABLE_INST_TOTAL)
coll.find_and_modify({}, {'$inc':{'cnt':1}}, upsert=True)
res = coll.find_one({})
if not res:
show_error(self, 'failed to add installer, invalid installer total number table')
return
inst = res.get('cnt')
coll = self._get_collection(TABLE_INST_ALLOC)
coll.save({'_id':inst, 'count':0})
coll = self._get_collection(TABLE_INST_ADDR)
coll.save({'_id':inst,'addr':addr})
return True
except:
show_error(self, 'failed to add installer')
def handle(self):
if SHOW_TIME:
start_time = datetime.utcnow()
uid = self.request.recv(UID_LEN)
if len(uid) != UID_LEN:
show_error(self, 'failed to handle, invalid head')
return
buf = self.request.recv(FLG_LEN)
if len(buf) != FLG_LEN:
show_error(self, 'failed to handle, invalid head')
return
flg, = struct.unpack('I', buf)
if flg == FLG_SEC:
if not self.server.rpcserver.user:
show_error(self, 'user is not initialized')
raise Exception('user is not initialized')
key = self.server.cache_get(uid)
if not key:
key = self.server.rpcserver.user.get_private_key(uid)
if not key:
show_error(self, 'failed to handle, invalid private key')
return
key = rsa.PrivateKey.load_pkcs1(key)
self.server.cache_update(uid, key)
stream = Stream(self.request, uid=uid, key=key)
else:
stream = Stream(self.request)
buf = stream.read()
if buf:
res = self.server.rpcserver.proc(buf)
if flg == FLG_SEC:
stream = Stream(self.request)
stream.write(res)
if SHOW_TIME:
self._print('handle, time=%d sec' % (datetime.utcnow() - start_time).seconds)
def uninstall(self, uid, package):
try:
addr = self._get_backend()
rpcclient = RPCClient(addr, BACKEND_PORT)
res = rpcclient.request('uninstall', uid=uid, package=package, typ=APP)
if not res:
show_error(self, 'failed to uninstall, invalid return res')
return
if DEBUG:
self._uninstall_cnt += 1
self._print('uninstall, count=%d' % self._uninstall_cnt)
return res
except:
show_error(self, 'failed to uninstall')
def get_installed_packages(self, uid):
self._print('get_installed_packages starts')
try:
addr = self._get_backend()
rpcclient = RPCClient(addr, BACKEND_PORT)
res = rpcclient.request('get_installed_packages', uid=uid, typ=APP)
if res:
result = []
for i in res:
result.append(str(i))
return result
else:
return ''
except:
show_error(self, 'failed to get installed packages')
def download(self, package, version):
self._print('download, package=%s, version=%s' %(str(package), str(version)))
try:
addr = self._get_backend()
rpcclient = RPCClient(addr, BACKEND_PORT)
info = rpcclient.request('download', package=package, version=version)
if not info:
show_error(self, 'failed to download, invalid return info')
return
if DEBUG:
self._download_cnt += 1
self._print('download, count=%d' % self._install_cnt)
return zlib.decompress(info)
except:
show_error(self, 'failed to download')
def get_private_key(self, uid):
self._print('get_private_key->uid=%s' % str(uid))
try:
key = self._private_keys.get(uid)
if not key:
coll = self._get_collection(uid, TABLE_USERINFO)
res = coll.find_one({'uid': uid}, {'privkey': 1, '_id': 0})
key = res.get('privkey')
if key:
if len(self._private_keys) >= CACHE_MAX:
self._private_keys.popitem()
self._private_keys.update({uid: key})
return key
except:
show_error(self, 'failed to get private key')
def handle(self):
uid = self.request.recv(UID_LEN)
if len(uid) != UID_LEN:
show_error(self, 'failed to handle, invalid head')
return
addr = self._get_backend(uid)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((addr, BACKEND_PORT))
try:
self._forward(uid, self.request, sock)
stream = Stream(sock)
_, _, res = stream.readall()
stream = Stream(self.request)
stream.write(res)
finally:
sock.close()
def set_package(self, uid, package, version, output, table):
self._print('set_package->uid=%s, package=%s, version=%s' % (str(uid), str(package), str(version)))
coll = self._get_collection(table)
res = coll.find_one({'uid': uid}, {'uid': 0, '_id': 0})
if res and res.has_key('package') and res['package'].has_key(package):
versions = res['package'][package]
for item in versions:
if item['version'] == version:
show_error(self, 'failed to set package, invalid version, uid=%s, package=%s, version=%s' % (str(uid), str(package), str(version)))
return
coll.update({'uid':uid}, {'$addToSet': {'package.%s' % package: {'version':version, 'output':output}}}, upsert=True)
else:
if not res:
coll.save({'uid':uid, 'package':{package: [{'version':version, 'output':output}]}})
else:
coll.update({'uid':uid}, {'$set': {'package.%s' % package:[{'version':version, 'output':output}]}}, upsert=True)
