def _update_finders(self):
while True:
try:
time.sleep(WAIT_TIME)
self._do_update_finders()
except:
log_err(self, 'failed to update finders')
def test():
log_debug('install', 'start testing ')
login_cnt = 0
install_cnt = 0
for i in range(TEST_ROUNDS):
if SHOW_TIME:
start_time = datetime.utcnow()
username = USER + str(USER_START + i)
uid, key = login(username, PASSWORD)
if not uid or not key:
log_err('install->login', 'failed to login %s.' % str(username))
return False
login_cnt += 1
log_debug('install->login', 'login_cnt=%d' % login_cnt)
if SHOW_TIME:
log_debug('install->login',
'time=%d sec' % (datetime.utcnow() - start_time).seconds)
start_time = datetime.utcnow()
if RANDOM_DRIVER:
driver_num = randint(40, DRIVER_NUM)
package = DRIVER_PACKAGE + str(driver_num)
else:
package = DRIVER_PACKAGE + str(DRIVER_START + i)
if not install_driver(uid, package, VERSION):
log_err('instll', 'failed to install driver %s' % str(package))
return False
if SHOW_TIME:
log_debug('install',
'time=%d sec' % (datetime.utcnow() - start_time).seconds)
install_cnt += 1
log_debug('install', 'install_cnt=%d' % install_cnt)
def put(self, uid, name, buf):
self._log('device->put, name=%s' % str(name))
if len(buf) >= RECORD_LEN:
log_err(self, 'failed to put, name=%s' % str(name))
return
try:
fields = ast.literal_eval(buf)
if type(fields) != dict:
log_err(self, 'failed to put, name=%s' % str(name))
return
except:
log_err(self, 'failed to put, name=%s' % str(name))
return
record = json.dumps(fields)
if len(record) >= RECORD_LEN:
log_err(self, 'failed to put, name=%s' % str(name))
return
path = get_mnt_path(uid, name)
fd = os.open(path, os.O_CREAT | os.O_WRONLY | os.O_TRUNC)
try:
os.write(fd, str(record))
except:
log_err(self, 'failed to put, cannot write, name=%s' % str(name))
return
finally:
os.close(fd)
if HISTORY:
self._query.history.put(uid, name, **fields)
self._query.event.put(uid, name)
self._log('device->put, name=%s, finished' % str(name))
return True
def __init__(self, domain):
if domain != DOMAIN_USR and domain != DOMAIN_DEV:
log_err(self, 'invalid domain')
raise Exception(log_get(self, 'invalid domain'))
self._mapper = MapperServer(domain)
self._domain = domain
Thread.__init__(self)
def delete(self, key):
addr = self._get_addr(key)
try:
cli = memcache.Client([addr], debug=0)
cli.delete(key)
except:
log_err(self, 'failed to delete, key=%s' % key)
def _request(self, op, args, event=None, timeout=None):
if not args:
log_debug(self, 'failed to request, no arguments')
return
ev = None
if event:
ev = self._channel_event.get(event['ev_type'], event['ev_name'])
if not ev:
log_err(self, 'failed to get event')
return
ret = None
try:
cli = zerorpc.Client()
cli.connect(zmqaddr(ADAPTER_ADDR, ADAPTER_PORT))
try:
cli.request(op, args);
finally:
cli.close()
if event:
if not timeout:
timeout = EV_TIMEOUT[event['ev_type']]
ret = self._channel_event.wait(ev, timeout)
finally:
if event:
err = self._channel_event.put(event['ev_type'], event['ev_name'])
if err != None:
ret = err
return ret
def initialize(self, uid, name, attr, val):
if attr not in ATTRIBUTES:
log_err(self, 'failed to initialize, invalid attribute %s, name=%s' % (str(attr), str(name)))
raise Exception(log_get(self, 'failed to initialize'))
if attr == ATTR_PROFILE:
val = json.dumps(val)
self._create_attr(uid, name, attr, val)
def install_driver(uid, package, version=None):
driver_path = os.path.join(PATH_DRIVER, driver)
if os.path.exists(driver_path):
shutil.rmtree(driver_path)
ret = install(uid, package, version, DRIVER)
if not ret:
log_err(
'util', 'failed to install driver, uid=%s, driver=%s, version=%s' %
(str(uid), str(driver), str(version)))
return False
dirname = tempfile.mkdtemp()
try:
buf = json.loads(zlib.decompress(ret))
dep = buf['dep']
if not _check_dep(dep):
log_err(
'util',
'failed to install driver, invalid dependency, uid=%s, driver=%s, version=%s'
% (str(uid), str(driver), str(version)))
return False
driver = buf['driver']
if driver:
src = os.path.join(dirname, 'driver')
os.mkdir(src)
filenames = driver.keys()
for name in filenames:
filepath = os.path.join(src, name)
with open(filepath, 'wb') as f:
f.write(driver[name])
shutil.copytree(src, driver_path)
finally:
shutil.rmtree(dirname)
return True
def test():
log_debug('get_author', 'start testing ')
auth_cnt = 0
for i in range(TEST_ROUNDS):
if SHOW_TIME:
start_time = datetime.utcnow()
if RANDOM_PKG:
cat_num = randint(0, len(CATEGORIES.keys()) - 1)
cat_name = CATEGORIES.keys()[cat_num]
pkg_num = randint(50, PKG_NUM)
package = cat_name + '_' + PKG + str(pkg_num)
else:
package = PACKAGE + str(PKG_START + i)
message = json.dumps({'op':'get_author', 'package':package})
ws = create_connection("ws://%s:%d/ws" % (get_manager(), get_port()))
ws.send(message)
ret = ws.recv()
ws.close()
result = json.loads(ret)
if not result or 'get_author' != result['op'] or package != result['package'] or not result['data']:
log_err('get_author', 'failed to get the author of %s ' % str(package))
return False
auth_cnt += 1
log_debug('get_author', 'auth_cnt=%d' % auth_cnt)
log_debug('get_author', 'the author of %s is %s' % (str(package), str(result['data'])))
if SHOW_TIME:
log_debug('get_author', 'time=%d sec' % (datetime.utcnow() - start_time).seconds)
def install_driver(uid, driver, version=None):
driver_path = os.path.join(PATH_DRIVER, driver)
if os.path.exists(driver_path):
shutil.rmtree(driver_path)
ret = install(uid, driver, version, DRIVER)
if not ret:
log_err(
'util', 'failed to install driver, uid=%s, driver=%s, version=%s' %
(str(uid), str(driver), str(version)))
return False
dirname = tempfile.mkdtemp()
try:
src = os.path.join(dirname, driver) + '.zip'
with open(src, 'wb') as f:
f.write(ret)
dest = os.path.join(dirname, driver)
unzip_file(src, dest)
dep_path = os.path.join(dest, 'dep')
if not _check_dep(dep_path):
log_err(
'util',
'failed to install driver, invalid dependency, uid=%s, driver=%s, version=%s'
% (str(uid), str(driver), str(version)))
return False
os.remove(dep_path)
shutil.copytree(dest, driver_path)
finally:
shutil.rmtree(dirname)
return True
def run(self):
while True:
try:
sock = self._srv.accept()[0]
Thread(target=self._proc, args=(sock, )).start()
except:
log_err(self, 'failed')
def _proc(self, cmd, buf):
result = ''
force = False
if cmd == CMD_OPEN:
result = self._driver.open()
elif cmd == CMD_CLOSE:
result = self._driver.close()
elif cmd == CMD_GET:
force = True
ret = self._driver.get()
if ret:
result = ret
elif cmd == CMD_PUT:
force = True
args, kwargs = self._check_args(buf)
if args != None and kwargs != None:
ret = self._driver.put(*args, **kwargs)
if ret:
result = ret
else:
log_err(self, 'failed to process, invalid command, driver=%s, cmd=%s' % (str(self._driver), str(cmd)))
if result or force:
if result:
self._log(cmd=cmd)
self._put(result)
def put(self, uid, name):
try:
addr = self._router.get(uid, DOMAIN_USR)
if addr:
self._put(addr, uid, name)
except:
log_err(self, 'failed to put')
def test():
log_debug('install', 'start testing ')
login_cnt = 0
install_cnt = 0
for i in range(TEST_ROUNDS):
if SHOW_TIME:
start_time = datetime.utcnow()
username = USER + str(USER_START + i)
uid, key = login(username, PASSWORD)
if not uid or not key:
log_err('install->login', 'failed to login %s.' % str(username))
return False
login_cnt += 1
log_debug('install->login', 'login_cnt=%d' % login_cnt)
if SHOW_TIME:
log_debug('install->login', 'time=%d sec' % (datetime.utcnow() - start_time).seconds)
start_time = datetime.utcnow()
if RANDOM_DRIVER:
driver_num = randint(0, DRIVER_NUM - 1)
package = DRIVER_PACKAGE + str(driver_num)
else:
package = DRIVER_PACKAGE + str(DRIVER_START + i)
if not install_driver(uid, package, VERSION):
log_err('instll', 'failed to install driver %s' % str(package))
return False
if SHOW_TIME:
log_debug('install', 'time=%d sec' % (datetime.utcnow() - start_time).seconds)
install_cnt += 1
log_debug('install', 'install_cnt=%d' % install_cnt)
def test():
log_debug('login', 'start testing ')
login_cnt = 0
for i in range(TEST_ROUNDS):
if SHOW_TIME:
start_time = datetime.utcnow()
username = USER + str(USER_START + i)
message = json.dumps({
'op': 'login',
'user': username,
'password': PASSWORD
})
ws = create_connection("ws://%s:%d/ws" % (get_manager(), get_port()))
ws.send(message)
ret = ws.recv()
ws.close()
result = json.loads(ret)
if not result or 'login' != result['op'] or username != result[
'user'] or not result['data']:
log_err('login', 'failed to login %s.' % str(username))
return False
uid = result['data'][0]
key = result['data'][1]
if not uid or not key:
log_err('login',
'failed to login %s, invalid uid or key.' % str(username))
return False
if SHOW_TIME:
log_debug('login',
'time=%d sec' % (datetime.utcnow() - start_time).seconds)
login_cnt += 1
log_debug('login', 'login_cnt=%d' % login_cnt)
def get_md5(text):
if type(text) == str or type(text) == unicode:
tmp = hashlib.md5()
tmp.update(text)
return tmp.hexdigest()
else:
log_err('util', 'failed to get md5')
def _try_connect(self, addr, timeout=None):
if not self._alloc.has_key(addr):
log_err(self, 'failed to connect, no channel, addr=%s' % str(addr))
raise Exception(log_get(self, 'failed to connect'))
key = self._alloc[addr]['key']
bridge = self._alloc[addr]['bridge']
self._connect(addr, key, bridge, timeout=timeout, release=False)
def put(self, key, value):
addr = self._get_addr(key)
try:
cli = memcache.Client([addr], debug=0)
cli.set(key, value)
except:
log_err(self, 'failed to put, key=%s' % key)
def test():
log_debug('get_author', 'start testing ')
auth_cnt = 0
for i in range(TEST_ROUNDS):
if SHOW_TIME:
start_time = datetime.utcnow()
if RANDOM_PKG:
cat_num = randint(0, len(CAT) - 1)
cat_name = CAT[cat_num]
pkg_num = randint(0, PKG_NUM - 1)
package = cat_name + '_' + PKG + str(pkg_num)
else:
package = PACKAGE + str(PKG_START + i)
message = json.dumps({'operator':'get_author', 'package':package})
ws = create_connection("ws://%s:%d/" % (get_manager(), get_port()))
ws.send(message)
ret = ws.recv()
ws.close()
result = json.loads(ret)
if not result or 'get_author' != result['operator'] or package != result['package'] or not result['data']:
log_err('get_author', 'failed to get the author of %s ' % str(package))
return False
auth_cnt += 1
log_debug('get_author', 'auth_cnt=%d' % auth_cnt)
log_debug('get_author', 'the author of %s is %s' % (str(package), str(result['data'])))
if SHOW_TIME:
log_debug('get_author', 'time=%d sec' % (datetime.utcnow() - start_time).seconds)
def test():
log_debug('login', 'start testing ')
login_cnt = 0
for i in range(TEST_ROUNDS):
if SHOW_TIME:
start_time = datetime.utcnow()
username = USER + str(USER_START + i)
message = json.dumps({'operator':'login', 'user':username, 'password':PASSWORD})
ws = create_connection("ws://%s:%d/" % (get_manager(), get_port()))
ws.send(message)
ret = ws.recv()
ws.close()
result = json.loads(ret)
if not result or 'login' != result['operator'] or username != result['user'] or not result['data']:
log_err('login', 'failed to login %s.' % str(username))
return False
uid = result['data'][0]
key = result['data'][1]
if not uid or not key:
log_err('login', 'failed to login %s, invalid uid or key.' % str(username))
return False
if SHOW_TIME:
log_debug('login', 'time=%d sec' % (datetime.utcnow() - start_time).seconds)
login_cnt += 1
log_debug('login', 'login_cnt=%d' % login_cnt)
def _check_dep(buf):
if not buf:
return
dep = yaml.load(buf)
if not dep:
return
cmds = []
for i in dep:
installer = dep[i].get('installer')
if installer not in INSTALLERS:
log_err('util', 'failed to check dependency, invalid installer')
return
cmd = '%s install %s' % (installer, i)
version = dep[i].get('version')
if version:
if installer == APT:
cmd += '=%s' % version
elif installer == PIP:
cmd += '==%s' % version
cmds.append(cmd)
for cmd in cmds:
status, output = commands.getstatusoutput(cmd)
if status != 0:
log_err('util', 'failed to check dependency')
return
return True
def get_md5(text):
if type(text) == str or type(text) == unicode:
tmp = hashlib.md5()
tmp.update(text)
return tmp.hexdigest()
else:
log_err('util', 'failed to get md5')
def release(self, path, fh):
obj, uid, name = self._parse(path)
if not obj:
log_err(self, 'failed to release, no object, name=%s' % str(name))
raise FuseOSError(EINVAL)
self._do_release(path, fh, obj, uid, name)
self._log('release, path=%s' % path)
def unlink(self, path):
obj, uid, name = self._parse(path)
if not obj:
log_err(self, 'failed to unlink, no object, name=%s' % str(name))
raise FuseOSError(EINVAL)
self._do_unlink(obj, uid, name)
obj.unlink(uid, name)
def initialize(self, manager):
key = manager.key
uid = manager.uid
addr = manager.addr
token = manager.token
if not key or not uid or not addr or not token:
log_err(self, 'faild to initialize')
raise Exception(log_get(self, 'failed to initialize'))
self._keys = {}
self._tokens = {}
self._devices = {}
self._event = Event()
self._op = Operation(manager)
if ASYNC:
for _ in range(POOL_SIZE):
self.add(ConductorQueue(self))
self.uid = uid
self.addr = addr
self.token = token
self.user = manager.user
self.devices = manager.devices
self.request = Request(uid, token)
channel.create(uid, addr, key)
self._ready = True
def test():
log_debug('register', 'start testing ')
get_top_details_cnt = 0
for i in range(TEST_ROUNDS):
if SHOW_TIME:
start_time = datetime.utcnow()
if RANDOM_CAT:
cat_num = randint(0, len(CAT) - 1)
cat_name = CAT[cat_num]
if not CATEGORIES.has_key(cat_name):
log_err('get_top_details', 'failed to get the top details, invalid category is %s' % str(cat_name))
return False
cate = CATEGORIES[cat_name]
else:
cate = CATEGORY
message = json.dumps({'operator':'get_top_details', 'category':cate})
ws = create_connection("ws://%s:%d/" % (get_manager(), get_port()))
ws.send(message)
ret = ws.recv()
ws.close()
result = json.loads(ret)
if not result or 'get_top_details' != result['operator'] or cate != result['category']:
log_err('get_top_details', 'failed to get the top packages details of %s' % str(cate))
return False
if SHOW_TIME:
log_debug('register', 'time=%d sec' % (datetime.utcnow() - start_time).seconds)
ret = result['data']
get_top_details_cnt += 1
log_debug('get_top_details', 'get_top_details_cnt=%d' % get_top_details_cnt)
log_debug('get_top_details', 'top details packages=%s' % str(ret))
def run(self):
self._log('start ...')
liveness = PPP_HEARTBEAT_LIVENESS
timeout = time.time() + PPP_HEARTBEAT_INTERVAL
while True:
sockets = dict(self._poller.poll(PPP_HEARTBEAT_INTERVAL * 1000))
if sockets.get(self._socket) == POLLIN:
frames = self._socket.recv_multipart()
if not frames:
log_err(self, 'invalid request')
break
if len(frames) == PPP_NR_FRAMES:
self._pool.apply_async(self._proc, args=(frames[PPP_FRAME_ID], frames[PPP_FRAME_SEQ], frames[PPP_FRAME_BUF]))
elif len(frames) == 1 and frames[0] == PPP_HEARTBEAT:
liveness = PPP_HEARTBEAT_LIVENESS
else:
log_err(self, "invalid request")
else:
liveness -= 1
if liveness == 0:
time.sleep(random.randint(SLEEP_TIME / 2, SLEEP_TIME))
self._reset_sock()
liveness = PPP_HEARTBEAT_LIVENESS
if time.time() > timeout:
timeout = time.time() + PPP_HEARTBEAT_INTERVAL
self._socket.send(PPP_HEARTBEAT)
def __init__(self, router, domain=None):
if not router:
log_err(self, 'failed to initialize')
raise Exception(log_get(self, 'failed to initialize'))
self._lock = Lock()
self._router = router
self._domain = domain
self._collections = {}
def open(self, path, flags):
obj, uid, name = self._parse(path)
if not obj:
log_err(self, 'failed to open, no object, name=%s' % str(name))
raise FuseOSError(EINVAL)
self._check_file(obj, name, flags=flags)
self._log('open, path=%s' % path)
return self._do_open(path, flags, obj, uid, name)
def create(self, path, mode):
obj, uid, name = self._parse(path)
if not obj:
log_err(self, 'failed to create, no object, name=%s' % str(name))
raise FuseOSError(EINVAL)
self._check_file(obj, name, create=True)
self._log('create, path=%s' % path)
return self._do_create(path, obj, uid, name)
def getattr(self, path, fh=None):
obj, uid, name = self._parse(path)
if not name:
return STAT_DIR
if not obj:
log_err(self, 'failed to getattr, no object, name=%s' % str(name))
raise FuseOSError(EINVAL)
return obj.getattr(uid, name)
def _proc(self, identity, seq, buf):
uid, token = self._get_token(buf)
if not uid or not token:
log_err(self, 'failed to process, cannot get token')
return
addr = self._get_worker(uid)
ret = self._request(addr, uid=uid, token=token, buf=buf)
self._reply(identity, seq, ret)
def _proc(self, name, buf):
try:
if not self.has_handler(name):
return self._handle(name, buf)
else:
return self.handle(name, buf)
except:
log_err(self, 'failed to process, name=%s' % name)
def _accept(self, path, target):
if not self._shadow:
return
obj, _, name = self._parse(path)
if not obj:
log_err(self, 'failed to accept, no object, name=%s' % str(name))
raise FuseOSError(EINVAL)
self._manager.guest.accept(name, target)
def symlink(self, uid, name):
child = self.child(name)
parent = self.parent(name)
if child == parent:
log_err(self, 'failed to create symlink')
raise FuseOSError(EINVAL)
path = os.path.join(self.get_path(uid, parent), child)
self._fs.touch(uid, path)
def initialize(self, uid, name, vrtx):
if type(vrtx) != list:
log_err(self, 'failed to initialize')
raise FuseOSError(EINVAL)
for i in vrtx:
v = os.path.join(name, i)
self.create(uid, v)
def allocate(self, addr, key, bridge):
try:
if self._alloc.has_key(addr):
self._free(addr)
self._alloc[addr] = {'key':key, 'bridge':bridge}
except:
log_err(self, 'failed to allocate')
raise Exception(log_get(self, 'failed to allocate'))
def _create(self, cmd, buf):
try:
if cmd == CMD_MOUNT:
result = self._driver.get_info()
self._put(result, index=False)
self._active = True
except:
log_err(self, 'failed to create, driver=%s, cmd=%s' % (str(self._driver), str(cmd)))
def uninstall(uid, package, typ):
addr = _get_frontend()
rpcclient = RPCClient(addr, FRONTEND_PORT)
ret = rpcclient.request('uninstall', uid=uid, package=package, typ=typ)
if not ret:
log_err(
'util', 'failed to uninstall, uid=%s, package=%s, typ=%s' %
(str(uid), str(package), str(typ)))
return
return ret
def unpack(buf):
tmp = loads(buf)
if type(tmp) != dict or not tmp.has_key('op') or not tmp.has_key(
'args') or not tmp.has_key('kwargs'):
log_err('package', 'failed to unpack, invalid type')
return
op = tmp['op']
args = tmp['args']
kwargs = tmp['kwargs']
if type(op) != str or type(args) != list or type(kwargs) != dict:
log_err('package', 'failed to unpack, invalid arguments')
return
return (op, args, kwargs)
def _proc(self, sock):
try:
req = recv_pkt(sock)
if not req or type(req) != dict:
log_err(self, 'failed to process, invalid request')
return
op = req.get('op')
args = req.get('args')
name = req.get('name')
timeout = req.get('timeout')
if not op or not name:
log_err(self, 'failed to process, invalid request')
return
service = self._services.get(name)
if not service:
log_err(
self,
'failed to process, cannot find service %s' % str(name))
return
pool = ThreadPool(processes=1)
result = pool.apply_async(service.proc, (op, args))
pool.close()
try:
send_pkt(sock, result.get(timeout))
except TimeoutError:
log_err(self, 'timeout')
finally:
pool.join()
finally:
sock.close()
def _request(self, *args):
try:
self._open()
cmd = {
'name': self._name,
'op': self._op,
'args': args,
'timeout': self._timeout
}
self._send(cmd)
return self._recv()
except:
log_err(self, 'failed')
finally:
self._close()
def upload_package(buf, uid, package, version, typ, key):
addr = _get_frontend()
rpcclient = RPCClient(addr, FRONTEND_PORT, uid, key)
ret = rpcclient.request('upload',
uid=uid,
package=package,
version=version,
buf=buf,
typ=typ)
if ret:
return True
else:
log_err(
'util',
'failed to upload, uid=%s, package=%s, version=%s, typ=%s' %
(str(uid), str(package), str(version), str(typ)))
return False
def install(uid, package, version, typ):
addr = _get_frontend()
rpcclient = RPCClient(addr, FRONTEND_PORT)
if typ == DRIVER:
content = None
ret = rpcclient.request('install',
uid=uid,
package=package,
version=version,
typ=typ,
content=content)
if not ret:
log_err(
'util',
'failed to install, uid=%s, package=%s, version=%s, typ=%s' %
(str(uid), str(package), str(version), str(typ)))
return
return ret
def test():
log_debug('register', 'start testing ')
register_cnt = 0
for i in range(TEST_ROUNDS):
if SHOW_TIME:
start_time = datetime.utcnow()
username = USER + str(USER_START + i)
message = json.dumps({'op':'register', 'user':username, 'password':PASSWORD, 'email':EMAIL})
ws = create_connection("ws://%s:%d/ws" % (get_manager(), get_port()))
ws.send(message)
ret = ws.recv()
ws.close()
result = json.loads(ret)
if not result or 'register' != result['op'] or username != result['user'] or not result['data']:
log_err('register', 'failed to register %s.' % str(username))
return False
register_cnt += 1
log_debug('register', 'register_cnt=%d' % register_cnt)
if SHOW_TIME:
log_debug('register', 'test, time=%d sec' % (datetime.utcnow() - start_time).seconds)
def test():
log_debug('get_categories', 'start testing ')
cat_cnt = 0
for _ in range(TEST_ROUNDS):
if SHOW_TIME:
start_time = datetime.utcnow()
message = json.dumps({'op': 'get_categories'})
ws = create_connection("ws://%s:%d/ws" % (get_manager(), get_port()))
ws.send(message)
ret = ws.recv()
ws.close()
result = json.loads(ret)
if not result or 'get_categories' != result['op'] or not result['data']:
log_err('get_categories', 'failed to get categories')
return False
cat_cnt += 1
log_debug('get_categories', 'cat_cnt=%d' % cat_cnt)
log_debug('get_categories', 'categories=%s' % str(result['data']))
if SHOW_TIME:
log_debug('get_categories',
'time=%d sec' % (datetime.utcnow() - start_time).seconds)
def test():
log_debug('get_top_details', 'start testing ')
get_top_details_cnt = 0
for i in range(TEST_ROUNDS):
if SHOW_TIME:
start_time = datetime.utcnow()
if RANDOM_CAT:
cat_num = randint(0, len(CATEGORIES.keys()) - 1)
cat_name = CATEGORIES.keys()[cat_num]
if not CATEGORIES.has_key(cat_name):
log_err(
'get_top_details',
'failed to get the top details, invalid category is %s' %
str(cat_name))
return False
cate = CATEGORIES[cat_name]
else:
cate = CATEGORY
message = json.dumps({'op': 'get_top_details', 'category': cate})
ws = create_connection("ws://%s:%d/ws" % (get_manager(), get_port()))
ws.send(message)
ret = ws.recv()
ws.close()
result = json.loads(ret)
if not result or 'get_top_details' != result['op'] or cate != result[
'category']:
log_err('get_top_details',
'failed to get the top packages details of %s' % str(cate))
return False
if SHOW_TIME:
log_debug('get_top_details',
'time=%d sec' % (datetime.utcnow() - start_time).seconds)
ret = result['data']
get_top_details_cnt += 1
log_debug('get_top_details',
'get_top_details_cnt=%d' % get_top_details_cnt)
log_debug('get_top_details', 'top details packages=%s' % str(ret))
def upload(path, uid, package, version, typ, key):
zipfilename = get_filename(package, version)
zipfilepath = os.path.join('/tmp', zipfilename)
zip_dir(path, zipfilepath)
with open(zipfilepath) as f:
buf = f.read()
os.remove(zipfilepath)
addr = _get_frontend()
rpcclient = RPCClient(addr, FRONTEND_PORT, uid, key)
ret = rpcclient.request('upload',
uid=uid,
package=package,
version=version,
buf=buf,
typ=typ)
if ret:
return True
else:
log_err(
'util',
'failed to upload, uid=%s, package=%s, version=%s, typ=%s' %
(str(uid), str(package), str(version), str(typ)))
return False
def test():
log_debug('upload', 'start testing ')
login_cnt = 0
upload_cnt = 0
for i in range(TEST_ROUNDS):
if SHOW_TIME:
start_time = datetime.utcnow()
username = USER + str(USER_START + i)
uid, key = login(username, PASSWORD)
if not uid or not key:
log_err('upload->login', 'failed to login %s.' % str(username))
return False
login_cnt += 1
log_debug('upload->login', 'login_cnt=%d' % login_cnt)
package = DRIVER_PACKAGE + str(DRIVER_START + i)
if not upload(DRIVER_PATH, uid, package, VERSION, DRIVER, key):
log_err('upload', 'failed to upload driver %s' % str(package))
return False
if SHOW_TIME:
log_debug('upload',
'time=%d sec' % (datetime.utcnow() - start_time).seconds)
upload_cnt += 1
log_debug('upload', 'upload_cnt=%d' % upload_cnt)
def proc(self, op, args):
try:
if op[0] == '_':
log_err(self, 'invalid operation %s' % op)
return
func = getattr(self, op)
if not func:
log_err(self, 'invalid operation %s' % op)
return
return func(*args)
except:
log_err(self, 'failed to process %s' % op)
def test():
log_debug('get_counter', 'start testing ')
get_count = 0
for i in range(TEST_ROUNDS):
if SHOW_TIME:
start_time = datetime.utcnow()
if RANDOM_CAT:
cat_num = randint(0, len(CATEGORIES.keys()) - 1)
cat_name = CATEGORIES.keys()[cat_num]
if not CATEGORIES.has_key(cat_name):
log_err(
'get_counter',
'failed to get the category, invalid category is %s' %
str(cat_name))
return False
cate = CATEGORIES[cat_name]
else:
cate = CATEGORY
message = json.dumps({'op': 'get_counter', 'category': cate})
ws = create_connection("ws://%s:%d/ws" % (get_manager(), get_port()))
ws.send(message)
ret = ws.recv()
ws.close()
result = json.loads(ret)
if not result or 'get_counter' != result['op'] or cate != result[
'category'] or not result['data']:
log_err('get_counter', 'failed to get counter')
return False
counter = result['data']
if not counter:
log_err('get_counter',
'failed to get the total number of %s ' % str(cate))
return False
get_count += 1
log_debug('get_inst', 'get_count=%d' % get_count)
if SHOW_TIME:
log_debug('get_counter',
'time=%d sec' % (datetime.utcnow() - start_time).seconds)
log_debug('get_counter', 'counter=%s' % str(counter))
if int(counter) < 1:
rank = 0
else:
rank = randint(0, (int(counter) + PAGE_SIZE - 1) / PAGE_SIZE - 1)
log_debug('get_counter', 'rank=%d' % rank)
def test():
log_debug('get_installed_packages', 'start testing ')
login_cnt = 0
get_inst_pkgs_cnt = 0
for i in range(TEST_ROUNDS):
if SHOW_TIME:
start_time = datetime.utcnow()
username = USER + str(USER_START + i)
message = json.dumps({'op':'login', 'user':username, 'password':PASSWORD})
ws = create_connection("ws://%s:%d/ws" % (get_manager(), get_port()))
ws.send(message)
ret = ws.recv()
ws.close()
result = json.loads(ret)
if not result or 'login' != result['op'] or username != result['user'] or not result['data']:
log_err('get_installed_packages->login', 'failed to login %s.' % str(username))
return False
uid = result['data'][0]
if not uid:
log_err('get_installed_packages->login', 'failed to login %s, invalid uid.' % str(username))
return False
if SHOW_TIME:
log_debug('get_installed_packages->login', 'time=%d sec' % (datetime.utcnow() - start_time).seconds)
login_cnt += 1
log_debug('get_installed_packages->login', 'login_cnt=%d' % login_cnt)
if SHOW_TIME:
start_time = datetime.utcnow()
message = json.dumps({'op':'get_installed_packages', 'uid':uid})
ws = create_connection("ws://%s:%d/ws" % (get_manager(), get_port()))
ws.send(message)
ret = ws.recv()
ws.close()
result = json.loads(ret)
if not result or 'get_installed_packages' != result['op'] or uid != result['uid']:
log_err('get_installed_packages', 'failed to get installed packages')
return False
get_inst_pkgs_cnt += 1
log_debug('get_installed_packages', 'get_inst_pkgs_cnt=%d' % get_inst_pkgs_cnt)
log_debug('get_installer_packages', 'the packages of %s has installed are %s' % (str(username), str(result['data'])))
if SHOW_TIME:
log_debug('get_installed_packages', 'time=%d sec' % (datetime.utcnow() - start_time).seconds)
def test():
log_debug('install', 'start testing ')
login_cnt = 0
install_cnt = 0
get_top_cnt = 0
has_pkg_cnt = 0
download_cnt = 0
get_top_details_cnt = 0
for i in range(TEST_ROUNDS):
if SHOW_TIME:
start_time = datetime.utcnow()
username = USER + str(USER_START + i)
message = json.dumps({
'op': 'login',
'user': username,
'password': PASSWORD
})
ws = create_connection("ws://%s:%d/ws" % (get_manager(), get_port()))
ws.send(message)
ret = ws.recv()
ws.close()
result = json.loads(ret)
if not result or 'login' != result['op'] or username != result[
'user'] or not result['data']:
log_err('login', 'failed to login %s.' % str(username))
return False
uid = result['data'][0]
key = result['data'][1]
if not uid or not key:
log_err('install->login',
'failed to login %s, invalid uid or key.' % str(username))
return False
if SHOW_TIME:
log_debug('install->login',
'time=%d sec' % (datetime.utcnow() - start_time).seconds)
login_cnt += 1
log_debug('install->login', 'login_cnt=%d' % login_cnt)
if SHOW_TIME:
start_time = datetime.utcnow()
if RANDOM_PKG:
cat_num = randint(0, len(CATEGORIES.keys()) - 1)
cat_name = CATEGORIES.keys()[cat_num]
pkg_num = randint(50, PKG_NUM)
package = cat_name + '_' + PKG + str(pkg_num)
else:
package = PACKAGE + str(PKG_START + i)
message = json.dumps({
'op': 'download',
'uid': uid,
'package': package,
'version': VERSION
})
ws = create_connection("ws://%s:%d/ws" % (get_manager(), get_port()))
ws.send(message)
ret = ws.recv()
ws.close()
result = json.loads(ret)
log_debug('download', 'result=%s' % result)
if not result or 'download' != result['op'] or uid != result[
'uid'] or not result['data']:
log_err('download', 'failed to download app %s' % str(package))
return False
ret = result['data']
if not ret:
log_err(
'download',
'failed to download app %s, invalid return message' %
str(package))
return False
download_cnt += 1
log_debug('install', 'download_cnt=%d' % download_cnt)
if SHOW_TIME:
log_debug('download',
'time=%d sec' % (datetime.utcnow() - start_time).seconds)
if SHOW_TIME:
start_time = datetime.utcnow()
message = json.dumps({
'op': 'install',
'uid': uid,
'package': package,
'version': VERSION,
'type': APP,
'content': ret
})
ws = create_connection("ws://%s:%d/ws" % (get_manager(), get_port()))
ws.send(message)
ret = ws.recv()
ws.close()
result = json.loads(ret)
if not result or 'install' != result['op'] or uid != result[
'uid'] or not result['data']:
log_err('install', 'failed to install app %s' % str(package))
return False
ret = result['data']
if not ret:
log_err(
'install', 'failed to install app %s, invalid return message' %
str(package))
return False
install_cnt += 1
log_debug('install', 'install_cnt=%d' % install_cnt)
if SHOW_TIME:
log_debug('install',
'time=%d sec' % (datetime.utcnow() - start_time).seconds)
if SHOW_TIME:
start_time = datetime.utcnow()
message = json.dumps({
'op': 'has_package',
'uid': uid,
'package': package
})
ws = create_connection("ws://%s:%d/ws" % (get_manager(), get_port()))
ws.send(message)
ret = ws.recv()
ws.close()
result = json.loads(ret)
if not result or 'has_package' != result['op'] or package != result[
'package'] or not result['data']:
log_err('install-->has_package',
'failed to has app %s' % str(package))
return False
has_pkg_cnt += 1
log_debug('has_package', 'has_pkg_cnt=%d' % has_pkg_cnt)
if SHOW_TIME:
log_debug('has_package',
'time=%d sec' % (datetime.utcnow() - start_time).seconds)
if not CATEGORIES.has_key(cat_name):
log_err(
'get_top', 'failed to get the top, invalid category is %s' %
str(cat_name))
return False
cate = CATEGORIES[cat_name]
message = json.dumps({'op': 'get_top', 'category': cate})
ws = create_connection("ws://%s:%d/ws" % (get_manager(), get_port()))
ws.send(message)
ret = ws.recv()
ws.close()
result = json.loads(ret)
if not result or 'get_top' != result['op'] or cate != result[
'category']:
log_err('get_top',
'failed to get the top packages of %s' % str(cate))
return False
ret = result['data']
get_top_cnt += 1
log_debug('get_top', 'get_top_cnt=%d' % get_top_cnt)
log_debug('get_top', 'top packages=%s' % str(ret))
message = json.dumps({'op': 'get_top_details', 'category': cate})
ws = create_connection("ws://%s:%d/ws" % (get_manager(), get_port()))
ws.send(message)
ret = ws.recv()
ws.close()
result = json.loads(ret)
if not result or 'get_top_details' != result['op'] or cate != result[
'category']:
log_err('get_top_details',
'failed to get the top packages details of %s' % str(cate))
return False
ret = result['data']
get_top_details_cnt += 1
log_debug('get_top_details',
'get_top_details_cnt=%d' % get_top_details_cnt)
log_debug('get_top_details', 'top details packages=%s' % str(ret))
def add(self, name, addr):
if self._nodes.has_key(name):
log_err(self, 'cannot add again')
raise Exception(log_get(self, 'cannot add again'))
self._nodes.update({name: {'addr': addr, 'time': time.time()}})
def _check_dep(path):
if os.path.isfile(path):
with open(path) as file_dependency:
lines = file_dependency.readlines()
for line in lines:
try:
package_version = ''
installer_name = ''
res = []
for str_equal in line.split('='):
if str_equal.strip(): # not blank
for str_blank in str_equal.split():
res.append(str_blank)
if len(res) % 2 == 0:
if len(res):
log_err(
'util',
'failed to check dependency, invalid format')
return False
continue # if it is blank, then continue
else:
package_name = res[0]
for index_to_match in range(1, len(res), 2):
if res[index_to_match] == 'installer':
installer_name = res[index_to_match + 1]
continue
if res[index_to_match] == 'version':
package_version = res[index_to_match + 1]
continue
if installer_name == '':
installers = ['pip', 'apt-get']
for installer in installers:
installer_name = installer
if package_version == '':
cmd = '%s install %s' % (
str(installer_name), str(package_name))
else:
cmd = '%s install %s==%s' % (
str(installer_name), str(package_name),
str(package_version))
status, output = commands.getstatusoutput(cmd)
if status == 0:
log_debug(
'util',
'check dependency, finished installing %s'
% package_name)
break
if status != 0:
log_err(
'util',
'check dependency, invalid installer, failed to install %s'
% str(package_name))
return False
else:
if package_version == '':
cmd = '%s install %s' % (str(installer_name),
str(package_name))
else:
cmd = '%s install %s==%s' % (
str(installer_name), str(package_name),
str(package_version))
status, output = commands.getstatusoutput(cmd)
if status == 0:
log_debug(
'util',
'check dependency, finished installing %s'
% str(package_name))
else:
log_err(
'util',
'check dependency, failed to install %s' %
str(package_name))
return False
except:
continue # if it is blank, continue. else return False
return True
def test():
log_debug('upload', 'start testing ')
login_cnt = 0
upload_cnt = 0
get_pkgs_details_cnt = 0
for i in range(TEST_ROUNDS):
if SHOW_TIME:
start_time = datetime.utcnow()
username = USER + str(USER_START + i)
uid, key = login(username, PASSWORD)
if not uid or not key:
log_err('upload->login', 'failed to login %s.' % str(username))
return False
login_cnt += 1
log_debug('upload->login', 'login_cnt=%d' % login_cnt)
package = PACKAGE + str(50 + i)
if not upload(PATH, uid, package, VERSION, APP, key):
log_err('upload', 'failed to upload app %s' % str(package))
return False
if SHOW_TIME:
log_debug('upload',
'time=%d sec' % (datetime.utcnow() - start_time).seconds)
upload_cnt += 1
log_debug('upload', 'upload_cnt=%d' % upload_cnt)
if SHOW_TIME:
start_time = datetime.utcnow()
message = json.dumps({'op': 'get_counter', 'category': CATEGORY})
ws = create_connection("ws://%s:%d/ws" % (get_manager(), get_port()))
ws.send(message)
ret = ws.recv()
ws.close()
result = json.loads(ret)
if not result or 'get_counter' != result['op'] or CATEGORY != result[
'category'] or not result['data']:
log_err('get_counter', 'failed to get counter')
return False
counter = result['data']
if not counter:
log_err('get_counter',
'failed to get the total number of %s ' % str(CATEGORY))
return False
if SHOW_TIME:
log_debug('get_counter',
'time=%d sec' % (datetime.utcnow() - start_time).seconds)
log_debug('get_counter', 'counter=%s' % str(counter))
if int(counter) < 1:
rank = 0
else:
rank = randint(0, (int(counter) + PAGE_SIZE - 1) / PAGE_SIZE - 1)
log_debug('get_counter', 'rank=%d' % rank)
if SHOW_TIME:
start_time = datetime.utcnow()
message = json.dumps({
'op': 'get_packages_details',
'category': CATEGORY,
'rank': rank
})
ws = create_connection("ws://%s:%d/ws" % (get_manager(), get_port()))
ws.send(message)
ret = ws.recv()
ws.close()
result = json.loads(ret)
if not result or 'get_packages_details' != result[
'op'] or CATEGORY != result['category'] or rank != result[
'rank'] or not result['data']:
log_err('get_packages_details', 'failed to get packages details')
return False
ret = result['data']
for item in ret:
if not item['pkg'] or not item['title'] or not item['auth']:
log_err('get_packages_details', 'failed to get valid details')
return False
if SHOW_TIME:
log_debug('get_packages_details',
'time=%d sec' % (datetime.utcnow() - start_time).seconds)
get_pkgs_details_cnt += 1
log_debug('get_packages_details',
'get_pkgs_details_cnt=%d' % get_pkgs_details_cnt)
log_debug('get_packages_details', 'packages_details=%s' % str(ret))
def show_error(cls, text):
log_err(cls.__class__.__name__, text)
def pack(op, args, kwargs):
if type(op) != str or type(args) != tuple or type(kwargs) != dict:
log_err('package', 'failed to pack, invalid type')
return
buf = {'op': op, 'args': args, 'kwargs': kwargs}
return dumps(buf)
