def write(self, buf): if len(self._uid) != UID_LEN: show_error(self, 'invalid uid') return if self._key: flg = FLG_SEC else: flg = 0 cnt = 0 length = len(buf) total = (length + PACKET_LEN - 1) / PACKET_LEN head = self._uid + struct.pack('I', flg) + struct.pack('I', total) self._sock.sendall(head) while cnt < total: start = cnt * PACKET_LEN end = min(start + PACKET_LEN, length) if self._key: body = rsa.encrypt(buf[start:end], self._key) else: body = buf[start:end] head = struct.pack('H', len(body)) self._sock.sendall(head) self._sock.sendall(body) cnt += 1 self._sock.recv(1)
def download(self, package, version): self._print('start to download') try: if SHOW_TIME: start_time = datetime.utcnow() addr = self._get_repo(package) rpcclient = RPCClient(addr, REPOSITORY_PORT) if not version: version = rpcclient.request('version', package=package) if not version: show_error( self, 'failed to download, invalid version, package=%s, ' % str(package)) return ret = rpcclient.request('download', package=package, version=version) if ret: if SHOW_TIME: self._print('download, time=%d sec' % (datetime.utcnow() - start_time).seconds) if DEBUG: self._download_cnt += 1 self._print('download, count=%d' % self._download_cnt) return ret except: show_error(self, 'failed to download')
def download(self, package, version): self._print('start to download, package=%s, version=%s' % (str(package), str(version))) try: if SHOW_TIME: start_time = datetime.utcnow() addr = self._get_addr(package) uid, ver = self._db.get_version(package) if not version: version = ver if not self._db.has_package(uid, package, version): show_error( self, 'failed to download, invalid version, package=%s, version=%s' % (str(package), str(version))) return if SHOW_TIME: self._print('download, time=%d sec' % (datetime.utcnow() - start_time).seconds) res = self._client.download(addr, self._port, package, version) if res: if DEBUG: self._download_cnt += 1 self._print('download, count=%d' % self._download_cnt) return res except: show_error(self, 'failed to download')
def upload(self, buf, uid, package, version, typ, key): ret = upload_package(buf, uid, package, version, typ, key) if ret: return '%s upload %s-%s successfully' % (str(uid), str(package), str(version)) else: show_error(self, 'failed to upload, uid=%s, package=%s, version=%s, typ=%s' % (str(uid), str(package), str(version), str(typ))) return False
def __init__(self, addr, port): RPCServer.__init__(self, addr, port) len_up = len(SERVER_UPLOADER) len_repo = len(SERVER_REPOSITORY) if len_up < len_repo or len_up % len_repo != 0: show_error(self, 'failed to initialize') raise Exception('failed to initialize') addr = localhost() if addr not in SERVER_REPOSITORY: show_error(self, 'failed to initialize') raise Exception('failed to initialize ') for i in range(len(SERVER_REPOSITORY)): if addr == SERVER_REPOSITORY[i]: break total = len_up / len_repo self._upload_servers = SERVER_UPLOADER[i * total:(i + 1) * total] self._print('upload_servers=%s' % str(self._upload_servers)) if HDFS: self._port = HDFS_PORT self._client = HDFSClient() else: self._port = FTP_PORT self._client = FTPClient() self._server = FTPServer() if SERVER_REPODB: self._db = Database(addr=SERVER_REPODB[0]) else: self._db = Database(addr=addr) locks = [] for _ in range(LOCK_MAX): locks.append(Lock()) self._locks = HashRing(locks) if DEBUG: self._upload_cnt = 0 self._download_cnt = 0
def register(self, user, pwd, email): self._print('register starts') lock = self._get_lock(user) lock.acquire() try: if SHOW_TIME: start_time = datetime.utcnow() uid = self.user.add(user, pwd, email) if not uid: show_error(self, 'failed to register, invalid register table') return False info = self._alloc_installer(uid) if SHOW_TIME: self._print('register, time=%d sec' % (datetime.utcnow() - start_time).seconds) if info: if DEBUG: self._register_cnt += 1 self._print('register, count=%d' % self._register_cnt) return True else: self.user.remove(user) show_error( self, 'failed to register, invalid alloc installer table') return False finally: lock.release()
def get_description(self, package): self._print('get_descripton starts' ) try: ret = self._recorder.get_description(package) if ret: return ret except: show_error(self, 'get_descripton failed')
def install(self, uid, package, version, typ, content): if typ == APP: return self._app.install(uid, package, version, content) elif typ == DRIVER: return self._driver.install(uid, package, version) else: show_error(self, 'failed to install, invalid type, typ=%s' % str(typ))
def check_installer(self, addr): self._print('add_installer starts!') try: coll = self._get_collection(TABLE_INST_ADDR) info = coll.find_one({'addr':addr}) if info: return True except: show_error(self, 'have not added installer %s' % addr)
def get_name(self, uid): self._print('get_name->uid=%s' % str(uid)) try: coll = self._get_collection(uid, TABLE_USERINFO) res = coll.find_one({'uid': uid}, {'user': 1, '_id': 0}) if res: return res.get('user') except: show_error(self, 'failed to get name')
def has_version(self, uid, package, version, table): self._print('has_version->uid=%s, package=%s, version=%s' % (str(uid), str(package), str(version))) if not version: show_error(self, 'failed to has version, the %s has no %s version' % (str(package), str(version))) return coll = self._get_collection(table) res = coll.find_one({'package': package}, {'uid': 1, 'version': 1, '_id': 0}) if res and res.get('uid') == uid and res.get('version') == version: return True
def _alloc_installer(self, uid): self._print('alloc_installer->uid=%s' % str(uid)) addr = self._get_allocator(uid) rpcclient = RPCClient(addr, ALLOCATOR_PORT) if rpcclient.request('alloc_installer', uid=uid): return True else: show_error(self, 'failed to allocate installer') return False
def install(self, uid, package, version): addr = self._get_repo(package) rpcclient = RPCClient(addr, REPOSITORY_PORT) if not version: version = rpcclient.request('version', package=package) if not version: show_error(self, 'failed to install, invalid version, uid=%s, package=%s' % (uid, package)) return ret = rpcclient.request('download', package=package, version=version) return ret
def get_top(self, category): self._print('get_top starts, category=%s' %str(category)) try: ret = self._recorder.get_top(category) if ret: return ret else: return '' except: show_error(self, 'get_top failed')
def get_password(self, user): self._print('get_password->user=%s' % str(user)) try: uid = get_uid(user) coll = self._get_collection(uid, TABLE_USERINFO) res = coll.find_one({'user': user}, {'password': 1, '_id': 0}) if res: return res.get('password') except: show_error(self, 'failed to get password')
def remove(self, user): self._print('remove starts') try: uid = get_uid(user) coll = self._get_collection(uid, TABLE_USERINFO) res = coll.find_one({'user': user}, {'uid': 1}) if res: coll.remove(res['_id']) return True except: show_error(self, 'failed to get public key')
def get_package_detail(self, package): self._print('get_package_detail starts') try: inst, title = self._recorder.get_package_detail(package) auth = self.get_author(package) if not auth: show_error(self, 'get_package_detail failed, no author') return return {'inst':inst, 'auth':auth, 'title':title} except: show_error(self, 'get_package_detail failed')
def has_version(self, uid, package, version, table): self._print('has_version->uid=%s, package=%s, version=%s' % (str(uid), str(package), str(version))) path = self._get_path(table) info = shelve.open(path) try: if not info: show_error(self, 'failed to has version, invalid information, package=%s, version=%s' % (str(package), str(version))) return if info.has_key(uid) and info[uid].has_key(package) and info[uid][package] == version: return True finally: info.close()
def get_author(self, package): self._print('get_author starts, package=%s' % str(package)) try: uid = self._recorder.get_uid(package) if uid: addr = self._get_backend() rpcclient = RPCClient(addr, BACKEND_PORT) name = rpcclient.request('get_name', uid=uid) if name: return str(name) except: show_error(self, 'get_author failed')
def readall(self): uid = self._recv(UID_LEN) if len(uid) != UID_LEN: show_error(self, 'failed to receive uid') return (None, None, '') buf = self._recv(FLG_LEN) if len(buf) != FLG_LEN: show_error(self, 'failed to receive flag') return (None, None, '') flg = struct.unpack('I', buf)[0] buf = self.read() return (uid, flg, buf)
def uninstall(self, uid, package, typ): self._print('start to uninstall') addr = self._get_installer(uid) rpcclient = RPCClient(addr, INSTALLER_PORT) res = rpcclient.request('uninstall', uid=uid, package=package, typ=typ) if not res: show_error(self, 'failed to uninstall') return if DEBUG: self._uninstall_cnt += 1 self._print('uninstall, count=%d' % self._uninstall_cnt) return res
def uninstall(self, uid, package): self._lock.acquire() try: if not self._db.has_package(uid, package, None): show_error(self, 'failed to uninstall %s' % package) return version, info = self._db.get_package(uid, package, None) if info: self._uninstall(uid, package, info) self._db.rm_package(uid, package, version) return True finally: self._lock.release()
def add_installer(self, addr): self._print('add_installer starts!') try: coll = self._get_collection(TABLE_INST_ADDR) info = coll.find_one({'addr':addr}) if info: show_error(self, 'failed to add installer, the address has been added') return if len(addr.split('.')) != 4: show_error(self, 'failed to add installer, invalid installer address') return coll = self._get_collection(TABLE_INST_TOTAL) coll.find_and_modify({}, {'$inc':{'cnt':1}}, upsert=True) res = coll.find_one({}) if not res: show_error(self, 'failed to add installer, invalid installer total number table') return inst = res.get('cnt') coll = self._get_collection(TABLE_INST_ALLOC) coll.save({'_id':inst, 'count':0}) coll = self._get_collection(TABLE_INST_ADDR) coll.save({'_id':inst,'addr':addr}) return True except: show_error(self, 'failed to add installer')
def handle(self): if SHOW_TIME: start_time = datetime.utcnow() uid = self.request.recv(UID_LEN) if len(uid) != UID_LEN: show_error(self, 'failed to handle, invalid head') return buf = self.request.recv(FLG_LEN) if len(buf) != FLG_LEN: show_error(self, 'failed to handle, invalid head') return flg, = struct.unpack('I', buf) if flg == FLG_SEC: if not self.server.rpcserver.user: show_error(self, 'user is not initialized') raise Exception('user is not initialized') key = self.server.cache_get(uid) if not key: key = self.server.rpcserver.user.get_private_key(uid) if not key: show_error(self, 'failed to handle, invalid private key') return key = rsa.PrivateKey.load_pkcs1(key) self.server.cache_update(uid, key) stream = Stream(self.request, uid=uid, key=key) else: stream = Stream(self.request) buf = stream.read() if buf: res = self.server.rpcserver.proc(buf) if flg == FLG_SEC: stream = Stream(self.request) stream.write(res) if SHOW_TIME: self._print('handle, time=%d sec' % (datetime.utcnow() - start_time).seconds)
def uninstall(self, uid, package): try: addr = self._get_backend() rpcclient = RPCClient(addr, BACKEND_PORT) res = rpcclient.request('uninstall', uid=uid, package=package, typ=APP) if not res: show_error(self, 'failed to uninstall, invalid return res') return if DEBUG: self._uninstall_cnt += 1 self._print('uninstall, count=%d' % self._uninstall_cnt) return res except: show_error(self, 'failed to uninstall')
def get_installed_packages(self, uid): self._print('get_installed_packages starts') try: addr = self._get_backend() rpcclient = RPCClient(addr, BACKEND_PORT) res = rpcclient.request('get_installed_packages', uid=uid, typ=APP) if res: result = [] for i in res: result.append(str(i)) return result else: return '' except: show_error(self, 'failed to get installed packages')
def download(self, package, version): self._print('download, package=%s, version=%s' %(str(package), str(version))) try: addr = self._get_backend() rpcclient = RPCClient(addr, BACKEND_PORT) info = rpcclient.request('download', package=package, version=version) if not info: show_error(self, 'failed to download, invalid return info') return if DEBUG: self._download_cnt += 1 self._print('download, count=%d' % self._install_cnt) return zlib.decompress(info) except: show_error(self, 'failed to download')
def get_private_key(self, uid): self._print('get_private_key->uid=%s' % str(uid)) try: key = self._private_keys.get(uid) if not key: coll = self._get_collection(uid, TABLE_USERINFO) res = coll.find_one({'uid': uid}, {'privkey': 1, '_id': 0}) key = res.get('privkey') if key: if len(self._private_keys) >= CACHE_MAX: self._private_keys.popitem() self._private_keys.update({uid: key}) return key except: show_error(self, 'failed to get private key')
def handle(self): uid = self.request.recv(UID_LEN) if len(uid) != UID_LEN: show_error(self, 'failed to handle, invalid head') return addr = self._get_backend(uid) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect((addr, BACKEND_PORT)) try: self._forward(uid, self.request, sock) stream = Stream(sock) _, _, res = stream.readall() stream = Stream(self.request) stream.write(res) finally: sock.close()
def set_package(self, uid, package, version, output, table): self._print('set_package->uid=%s, package=%s, version=%s' % (str(uid), str(package), str(version))) coll = self._get_collection(table) res = coll.find_one({'uid': uid}, {'uid': 0, '_id': 0}) if res and res.has_key('package') and res['package'].has_key(package): versions = res['package'][package] for item in versions: if item['version'] == version: show_error(self, 'failed to set package, invalid version, uid=%s, package=%s, version=%s' % (str(uid), str(package), str(version))) return coll.update({'uid':uid}, {'$addToSet': {'package.%s' % package: {'version':version, 'output':output}}}, upsert=True) else: if not res: coll.save({'uid':uid, 'package':{package: [{'version':version, 'output':output}]}}) else: coll.update({'uid':uid}, {'$set': {'package.%s' % package:[{'version':version, 'output':output}]}}, upsert=True)