def write(self, buf): if len(self._uid) != UID_LEN: show_error(self, 'invalid uid') return if self._key: flg = FLG_SEC else: flg = 0 cnt = 0 length = len(buf) total = (length + PACKET_LEN - 1) / PACKET_LEN head = self._uid + struct.pack('I', flg) + struct.pack('I', total) self._sock.sendall(head) while cnt < total: start = cnt * PACKET_LEN end = min(start + PACKET_LEN, length) if self._key: body = rsa.encrypt(buf[start:end], self._key) else: body = buf[start:end] head = struct.pack('H', len(body)) self._sock.sendall(head) self._sock.sendall(body) cnt += 1 self._sock.recv(1)
def get_package_detail(self, package): self._print('get_package_detail starts, package=%s' % str(package)) try: category = self._get_category(package) if not category: self._print( 'get_package_detail, cannot find category, package=%s' % str(package)) return coll = self.get_collection(TABLE_DESCRIPTION, category=category) res = coll.find_one({'pkg': package}, { 'inst': 1, 'title': 1, '_id': 0 }) if res: inst = res['inst'] title = res['title'] if not inst or not title: show_error( self, 'get_package_detail failed, cannot find valid message') return return (inst, title) except: show_error(self, 'get_package_detail failed')
def register(self, user, password, email): self._print('register starts') self._lock.acquire() try: if SHOW_TIME: start_time = datetime.utcnow() pwd = get_md5(password) addr = self._get_user_backend(user) rpcclient = RPCClient(addr, BACKEND_PORT) res = rpcclient.request('register', user=user, pwd=pwd, email=email) if SHOW_TIME: self._print('register , time=%d sec' % (datetime.utcnow() - start_time).seconds) if res: if DEBUG: self._register_cnt += 1 self._print('register, count=%d' % self._register_cnt) return True else: show_error(self, 'failed to register %s' % str(user)) return False finally: self._lock.release()
def register(self, user, pwd, email): self._print('register starts') lock = self._get_lock(user) lock.acquire() try: if SHOW_TIME: start_time = datetime.utcnow() uid = self.user.add(user, pwd, email) if not uid: show_error(self, 'failed to register, invalid register table') return False info= self._alloc_installer(uid) if SHOW_TIME: self._print('register, time=%d sec' % (datetime.utcnow() - start_time).seconds) if info: if DEBUG: self._register_cnt += 1 self._print('register, count=%d' % self._register_cnt) return True else: self.user.remove(user) show_error(self, 'failed to register, invalid alloc installer table') return False finally: lock.release()
def __init__(self, addr, port): RPCServer.__init__(self, addr, port) len_up = len(UPLOAD_SERVERS) len_repo = len(REPOSITORY_SERVERS) if len_up < len_repo or len_up % len_repo != 0: show_error(self, 'failed to initialize') raise Exception('failed to initialize') addr = localhost() if addr not in REPOSITORY_SERVERS: show_error(self, 'failed to initialize') raise Exception('failed to initialize REPOSITORY_SERVERS') for i in range(len(REPOSITORY_SERVERS)): if addr == REPOSITORY_SERVERS[i]: break total = len_up / len_repo self._upload_servers = UPLOAD_SERVERS[i * total:(i + 1) * total] self._print('upload_servers=%s' % str(self._upload_servers)) if HDFS: self._port = HDFS_PORT self._client = HDFSClient() else: self._port = FTP_PORT self._client = FTPClient() self._server = FTPServer() if REPO_DB: self._db = Database(addr=REPO_DB) else: self._db = Database(addr=addr) locks = [] for _ in range(LOCK_MAX): locks.append(Lock()) self._locks = HashRing(locks) if DEBUG: self._upload_cnt = 0 self._download_cnt = 0
def install(self, uid, package, version, typ): if typ == APP: return self._app.install(uid, package, version) elif typ == DRIVER: return self._driver.install(uid, package, version) else: show_error(self, 'failed to install, invalid type, typ=%s' % str(typ))
def install(self, package): self._print('install->package=%s' % str(package)) try: if SHOW_TIME: start_time = datetime.utcnow() category = self._get_category(package) if not category: self._print('install, cannot find category, package=%s' % str(package)) return coll = self.get_collection(TABLE_DESCRIPTION, category=category) res = coll.find_and_modify({'pkg': package}, {'$inc': { 'inst': 1 }}, upsert=True) if not res: cnt = 1 else: cnt = res.get('inst') if cnt != None: cnt += 1 if not cnt: show_error(self, 'failed to install, invalid counter') return coll = self.get_collection(TABLE_TOP, category=category) res = coll.find_one({'name': TOP_NAME}, {'name': 0, '_id': 0}) if not res or len(res) < TOP: coll.update({'name': TOP_NAME}, {'$set': { package: cnt }}, upsert=True) return True else: if package not in res: for i in res: if res[i] < cnt: coll.update({'name': TOP_NAME}, {'$unset': { i: '' }}) coll.update({'name': TOP_NAME}, {'$set': { package: cnt }}, upsert=True) break if SHOW_TIME: self._print('install, time=%d sec' % (datetime.utcnow() - start_time).seconds) return True else: coll.update({'name': TOP_NAME}, {'$set': {package: cnt}}) if SHOW_TIME: self._print('install, time=%d sec' % (datetime.utcnow() - start_time).seconds) return True except: show_error(self, 'failed to install')
def register(self, user, pwd, email): self._print('register starts') lock = self._get_lock(user) lock.acquire() try: if SHOW_TIME: start_time = datetime.utcnow() uid = self.user.add(user, pwd, email) if not uid: show_error(self, 'failed to register, invalid register table') return False info = self._alloc_installer(uid) if SHOW_TIME: self._print('register, time=%d sec' % (datetime.utcnow() - start_time).seconds) if info: if DEBUG: self._register_cnt += 1 self._print('register, count=%d' % self._register_cnt) return True else: self.user.remove(user) show_error( self, 'failed to register, invalid alloc installer table') return False finally: lock.release()
def download(self, package, version): self._print('start to download, package=%s, version=%s' % (str(package), str(version))) try: if SHOW_TIME: start_time = datetime.utcnow() addr = self._get_addr(package) uid, ver = self._db.get_version(package) if not version: version = ver if not self._db.has_package(uid, package, version): show_error( self, 'failed to download, invalid version, package=%s, version=%s' % (str(package), str(version))) return if DEBUG: self._download_cnt += 1 self._print('download, count=%d' % self._download_cnt) if SHOW_TIME: self._print('download, time=%d sec' % (datetime.utcnow() - start_time).seconds) return self._client.download(addr, self._port, package, version) except: show_error(self, 'failed to download')
def get_packages_details(self, category, rank): self._print('get_packages_details starts') try: result = [] if SHOW_TIME: start_time = datetime.utcnow() coll = self.get_collection(TABLE_DESCRIPTION, category=category) res = coll.find({'rank': rank}, { 'pkg': 1, 'title': 1, 'inst': 1, '_id': 0 }) if res: for item in res: ret = { 'pkg': item['pkg'], 'title': item['title'], 'inst': item['inst'] } result.append(ret) if SHOW_TIME: self._print('get_packages_details , time=%d sec' % (datetime.utcnow() - start_time).seconds) return result except: show_error(self, 'get_packages_details failed')
def upload(self, uid, category, package, title, description): self._print('upload->category=%s, package=%s' % (str(category), str(package))) try: if SHOW_TIME: start_time = datetime.utcnow() if not category or not package or not description: show_error(self, 'failed to upload, invalid arguments') return coll = self.get_collection(TABLE_CATEGORY, package=package) coll.update({'pkg':package}, {'cat':category, 'pkg':package}, upsert=True) t = str(datetime.utcnow()) cnt = self._update_counter(category) rank = cnt / PAGE_SIZE coll = self.get_collection(TABLE_DESCRIPTION, category=category) coll.update({'pkg':package}, {'rank':rank, 'pkg':package,'title':title, 'des':description, 'inst':0, 't':t}, upsert=True) coll = self.get_collection(TABLE_AUTHOR, package=package) coll.update({'uid':uid}, {'$set':{'pkg':package}}, upsert=True) if SHOW_TIME: self._print('upload, time=%d sec' % (datetime.utcnow() - start_time).seconds) if DEBUG: self._upload_cnt += 1 self._print('upload, count=%d' % self._upload_cnt) return True except: show_error(self, 'failed to upload')
def get_description(self, package): self._print('get_descripton starts') try: ret = self._recorder.get_description(package) if ret: return ret except: show_error(self, 'get_descripton failed')
def get_description(self, package): self._print('get_descripton starts' ) try: ret = self._recorder.get_description(package) if ret: return ret except: show_error(self, 'get_descripton failed')
def _alloc_installer(self, uid): self._print('alloc_installer->uid=%s' % str(uid)) addr = self._get_allocator(uid) rpcclient = RPCClient(addr, ALLOCATOR_PORT) if rpcclient.request('alloc_installer', uid=uid): return True else: show_error(self, 'failed to allocate installer') return False
def get_uid(self, package): self._print('get_uid starts, package=%s' % str(package)) try: coll = self.get_collection(TABLE_AUTHOR, package=package) res = coll.find_one({'pkg': package}, {'uid': 1, '_id': 0}) if res: return res['uid'] except: show_error(self, 'get_uid failed')
def get_name(self, uid): self._print('get_name->uid=%s' % str(uid)) try: coll = self._get_collection(uid, TABLE_USERINFO) res = coll.find_one({'uid': uid}, {'user': 1, '_id': 0}) if res: return res.get('user') except: show_error(self, 'failed to get name')
def get_uid(self, package): self._print('get_uid starts, package=%s' %str(package)) try: coll = self.get_collection(TABLE_AUTHOR, package=package) res = coll.find_one({'pkg':package}, {'uid':1, '_id':0}) if res: return res['uid'] except: show_error(self, 'get_uid failed')
def has_version(self, uid, package, version, table): self._print('has_version->uid=%s, package=%s, version=%s' % (str(uid), str(package), str(version))) if not version: show_error(self, 'failed to has version, the %s has no %s version' % (str(package), str(version))) return coll = self._get_collection(table) res = coll.find_one({'package': package}, {'uid': 1, 'version': 1, '_id': 0}) if res and res.get('uid') == uid and res.get('version') == version: return True
def get_name(self, uid): self._print('get_name->uid=%s' % str(uid)) try: coll = self._get_collection(uid, TABLE_USERINFO) res = coll.find_one({'uid': uid}, {'user':1, '_id':0}) if res: return res.get('user') except: show_error(self, 'failed to get name')
def get_top(self, category): self._print('get_top starts, category=%s' % str(category)) try: ret = self._recorder.get_top(category) if ret: return ret else: return '' except: show_error(self, 'get_top failed')
def get_top(self, category): self._print('get_top starts, category=%s' %str(category)) try: ret = self._recorder.get_top(category) if ret: return ret else: return '' except: show_error(self, 'get_top failed')
def get_password(self, user): self._print('get_password->user=%s' % str(user)) try: uid = get_uid(user) coll = self._get_collection(uid, TABLE_USERINFO) res = coll.find_one({'user': user}, {'password': 1, '_id': 0}) if res: return res.get('password') except: show_error(self, 'failed to get password')
def get_password(self, user): self._print('get_password->user=%s' % str(user)) try: uid = get_uid(user) coll = self._get_collection(uid, TABLE_USERINFO) res = coll.find_one({'user': user}, {'password':1, '_id':0}) if res: return res.get('password') except: show_error(self, 'failed to get password')
def get_package_detail(self, package): self._print('get_package_detail starts') try: inst, title = self._recorder.get_package_detail(package) auth = self.get_author(package) if not auth: show_error(self, 'get_package_detail failed, no author') return return {'inst':inst, 'auth':auth, 'title':title} except: show_error(self, 'get_package_detail failed')
def remove(self, user): self._print('remove starts') try: uid = get_uid(user) coll = self._get_collection(uid, TABLE_USERINFO) res = coll.find_one({'user':user}, {'uid':1}) if res: coll.remove(res['_id']) return True except: show_error(self, 'failed to get public key')
def install(self, uid, package, version): addr = self._get_repo(package) rpcclient = RPCClient(addr, REPOSITORY_PORT) if not version: version = rpcclient.request("version", package=package) if not version: show_error(self, "failed to install, invalid version, uid=%s, package=%s" % (uid, package)) return ret = rpcclient.request("download", package=package, version=version) self._print("finished installing driver %s, version=%s" % (package, version)) return ret
def get_counter(self, category): self._print('get_counter starts, category=%s' % str(category)) try: coll = self.get_collection(TABLE_COUNTER, category=category) res = coll.find_one({'cat': category}, {'cnt': 1, '_id': 0}) if not res: return str(0) else: return str(res.get('cnt')) except: show_error(self, 'get_counter failed')
def get_package_detail(self, package): self._print('get_package_detail starts') try: inst, title = self._recorder.get_package_detail(package) auth = self.get_author(package) if not auth: show_error(self, 'get_package_detail failed, no author') return return {'inst': inst, 'auth': auth, 'title': title} except: show_error(self, 'get_package_detail failed')
def remove(self, user): self._print('remove starts') try: uid = get_uid(user) coll = self._get_collection(uid, TABLE_USERINFO) res = coll.find_one({'user': user}, {'uid': 1}) if res: coll.remove(res['_id']) return True except: show_error(self, 'failed to get public key')
def get_counter(self, category): self._print('get_counter starts, category=%s' % str(category)) try: coll = self.get_collection(TABLE_COUNTER, category=category) res = coll.find_one({'cat': category}, {'cnt':1, '_id':0}) if not res: return str(0) else: return str(res.get('cnt')) except: show_error(self, 'get_counter failed')
def _update_counter(self, category): self._print('update_counter->category=%s' % str(category)) try: coll = self.get_collection(TABLE_COUNTER, category=category) res = coll.find_and_modify({'cat': category}, {'$inc':{'cnt':1}}, upsert=True) if not res: return 0 else: return res.get('cnt') except: show_error(self, 'failed to update counter')
def readall(self): uid = self._recv(UID_LEN) if len(uid) != UID_LEN: show_error(self, 'failed to receive uid') return (None, None, '') buf = self._recv(FLG_LEN) if len(buf) != FLG_LEN: show_error(self, 'failed to receive flag') return (None, None, '') flg = struct.unpack('I', buf)[0] buf = self.read() return (uid, flg, buf)
def get_author(self, package): self._print('get_author starts, package=%s' % str(package)) try: uid = self._recorder.get_uid(package) if uid: addr = self._get_backend() rpcclient = RPCClient(addr, BACKEND_PORT) name = rpcclient.request('get_name', uid=uid) if name: return str(name) except: show_error(self, 'get_author failed')
def uninstall(self, uid, package, typ): self._print('start to uninstall') addr = self._get_installer(uid) rpcclient = RPCClient(addr, INSTALLER_PORT) res = rpcclient.request('uninstall', uid=uid, package=package, typ=typ) if not res: show_error(self, 'failed to uninstall') return if DEBUG: self._uninstall_cnt += 1 self._print('uninstall, count=%d' % self._uninstall_cnt) return res
def has_version(self, uid, package, version, table): self._print('has_version->uid=%s, package=%s, version=%s' % (str(uid), str(package), str(version))) path = self._get_path(table) info = shelve.open(path) try: if not info: show_error(self, 'failed to has version, invalid information, package=%s, version=%s' % (str(package), str(version))) return if info.has_key(uid) and info[uid].has_key(package) and info[uid][package] == version: return True finally: info.close()
def uninstall(self, uid, package): self._lock.acquire() try: if not self._db.has_package(uid, package, None): show_error(self, 'failed to uninstall %s ' % package) return version, info = self._db.get_package(uid, package, None) if info: self._uninstall(uid, package, info) self._db.rm_package(uid, package, version) return True finally: self._lock.release()
def get_top_details(self, category): self._print('get_top_details starts, category=%s' % str(category)) try: info = self._get_top(category) res = [] for i in info: pkg = i.keys()[0] inst, title = self.get_package_detail(pkg) item = {'pkg':pkg, 'title':title, 'inst':inst} res.append(item) return res except: show_error(self, 'get_top_details failed')
def get_top_details(self, category): self._print('get_top_details starts, category=%s' % str(category)) try: info = self._get_top(category) res = [] for i in info: pkg = i.keys()[0] inst, title = self.get_package_detail(pkg) item = {'pkg': pkg, 'title': title, 'inst': inst} res.append(item) return res except: show_error(self, 'get_top_details failed')
def add_installer(self, addr): self._print('add_installer starts!') try: coll = self._get_collection(TABLE_INST_ADDR) info = coll.find_one({'addr':addr}) if info: show_error(self, 'failed to add installer, the address has been added') return if len(addr.split('.')) != 4: show_error(self, 'failed to add installer, invalid installer address') return coll = self._get_collection(TABLE_INST_TOTAL) coll.find_and_modify({}, {'$inc':{'cnt':1}}, upsert=True) res = coll.find_one({}) if not res: show_error(self, 'failed to add installer, invalid installer total number table') return inst = res.get('cnt') coll = self._get_collection(TABLE_INST_ALLOC) coll.save({'_id':inst, 'count':0}) coll = self._get_collection(TABLE_INST_ADDR) coll.save({'_id':inst,'addr':addr}) return True except: show_error(self, 'failed to add installer')
def _update_counter(self, category): self._print('update_counter->category=%s' % str(category)) try: coll = self.get_collection(TABLE_COUNTER, category=category) res = coll.find_and_modify({'cat': category}, {'$inc': { 'cnt': 1 }}, upsert=True) if not res: return 0 else: return res.get('cnt') except: show_error(self, 'failed to update counter')
def add_installer(self, addr): self._print('add_installer starts!') try: coll = self._get_collection(TABLE_INST_ADDR) info = coll.find_one({'addr': addr}) if info: show_error( self, 'failed to add installer, the address has been added') return if len(addr.split('.')) != 4: show_error( self, 'failed to add installer, invalid installer address') return coll = self._get_collection(TABLE_INST_TOTAL) coll.find_and_modify({}, {'$inc': {'cnt': 1}}, upsert=True) res = coll.find_one({}) if not res: show_error( self, 'failed to add installer, invalid installer total number table' ) return inst = res.get('cnt') coll = self._get_collection(TABLE_INST_ALLOC) coll.save({'_id': inst, 'count': 0}) coll = self._get_collection(TABLE_INST_ADDR) coll.save({'_id': inst, 'addr': addr}) return True except: show_error(self, 'failed to add installer')
def handle(self): if SHOW_TIME: start_time = datetime.utcnow() uid = self.request.recv(UID_LEN) if len(uid) != UID_LEN: show_error(self, 'failed to handle, invalid head') return buf = self.request.recv(FLG_LEN) if len(buf) != FLG_LEN: show_error(self, 'failed to handle, invalid head') return flg, = struct.unpack('I', buf) if flg == FLG_SEC: if not self.server.rpcserver.user: show_error(self, 'user is not initialized') raise Exception('user is not initialized') key = self.server.cache_get(uid) if not key: key = self.server.rpcserver.user.get_private_key(uid) if not key: show_error(self, 'failed to handle, invalid private key') return key = rsa.PrivateKey.load_pkcs1(key) self.server.cache_update(uid, key) stream = Stream(self.request, uid=uid, key=key) else: stream = Stream(self.request) buf = stream.read() if buf: res = self.server.rpcserver.proc(buf) if flg == FLG_SEC: stream = Stream(self.request) stream.write(res) if SHOW_TIME: self._print('handle, time=%d sec' % (datetime.utcnow() - start_time).seconds)
def uninstall(self, uid, package): try: addr = self._get_backend() rpcclient = RPCClient(addr, BACKEND_PORT) res = rpcclient.request('uninstall', uid=uid, package=package, typ=APP) if not res: show_error(self, 'failed to uninstall, invalid return res') return if DEBUG: self._uninstall_cnt += 1 self._print('uninstall, count=%d' % self._uninstall_cnt) return res except: show_error(self, 'failed to uninstall')
def get_inst(self, package): self._print('get_inst start, package=%s' %str(package)) try: category = self._get_category(package) if not category: self._print('get_inst, cannot find category, package=%s' % str(package)) return coll = self.get_collection(TABLE_DESCRIPTION, category=category) res = coll.find_one({'pkg':package}, {'inst':1, '_id':0}) if res: return str(res['inst']) else: return str(0) except: show_error(self, 'get_inst failed')
def get_description(self, package): self._print('get_description starts, package=%s' %str(package)) try: category = self._get_category(package) if not category: self._print('get_description, cannot find category, package=%s' % str(package)) return coll = self.get_collection(TABLE_DESCRIPTION, category=category) res = coll.find_one({'pkg':package}, {'title':1, 'des':1, '_id':0}) if res: return (str(res['title']), str(res['des'])) else: self._print('get_description, cannot find description, package=%s' % str(package)) except: show_error(self, 'get_description failed')
def get_private_key(self, uid): self._print('get_private_key->uid=%s' % str(uid)) try: key = self._private_keys.get(uid) if not key: coll = self._get_collection(uid, TABLE_USERINFO) res = coll.find_one({'uid': uid}, {'privkey': 1, '_id': 0}) key = res.get('privkey') if key: if len(self._private_keys) >= CACHE_MAX: self._private_keys.popitem() self._private_keys.update({uid: key}) return key except: show_error(self, 'failed to get private key')
def get_installed_packages(self, uid): self._print('get_installed_packages starts') try: addr = self._get_backend() rpcclient = RPCClient(addr, BACKEND_PORT) res = rpcclient.request('get_installed_packages', uid=uid, typ=APP) if res: result = [] for i in res: result.append(str(i)) return result else: return '' except: show_error(self, 'failed to get installed packages')
def get_private_key(self, uid): self._print('get_private_key->uid=%s' % str(uid)) try: key = self._private_keys.get(uid) if not key: coll = self._get_collection(uid, TABLE_USERINFO) res = coll.find_one({'uid': uid}, {'privkey':1, '_id':0}) key = res.get('privkey') if key: if len(self._private_keys) >= CACHE_MAX: self._private_keys.popitem() self._private_keys.update({uid:key}) return key except: show_error(self, 'failed to get private key')
def install(self, uid, package, version): addr = self._get_repo(package) rpcclient = RPCClient(addr, REPOSITORY_PORT) if not version: version = rpcclient.request('version', package=package) if not version: show_error( self, 'failed to install, invalid version, uid=%s, package=%s' % (uid, package)) return ret = rpcclient.request('download', package=package, version=version) self._print('finished installing driver %s, version=%s' % (package, version)) return ret