def test():
log_debug('install', 'start testing ')
login_cnt = 0
install_cnt = 0
for i in range(TEST_ROUNDS):
if SHOW_TIME:
start_time = datetime.utcnow()
username = USER + str(USER_START + i)
uid, key = login(username, PASSWORD)
if not uid or not key:
log_err('install->login', 'failed to login %s.' % str(username))
return False
login_cnt += 1
log_debug('install->login', 'login_cnt=%d' % login_cnt)
if SHOW_TIME:
log_debug('install->login', 'time=%d sec' % (datetime.utcnow() - start_time).seconds)
start_time = datetime.utcnow()
if RANDOM_DRIVER:
driver_num = randint(0, DRIVER_NUM - 1)
package = DRIVER_PACKAGE + str(driver_num)
else:
package = DRIVER_PACKAGE + str(DRIVER_START + i)
if not install_driver(uid, package, VERSION):
log_err('instll', 'failed to install driver %s' % str(package))
return False
if SHOW_TIME:
log_debug('install', 'time=%d sec' % (datetime.utcnow() - start_time).seconds)
install_cnt += 1
log_debug('install', 'install_cnt=%d' % install_cnt)
def request(self, op, *args, **kwargs):
log_debug('RPCClient', 'start to request, op=%s' % str(op))
buf = pack(op, args, kwargs)
res = self.dpmclient.request(self.addr, self.port, buf)
if res:
ret = loads(res)
return ret['res']
def test():
log_debug('install', 'start testing ')
login_cnt = 0
install_cnt = 0
for i in range(TEST_ROUNDS):
if SHOW_TIME:
start_time = datetime.utcnow()
username = USER + str(USER_START + i)
uid, key = login(username, PASSWORD)
if not uid or not key:
log_err('install->login', 'failed to login %s.' % str(username))
return False
login_cnt += 1
log_debug('install->login', 'login_cnt=%d' % login_cnt)
if SHOW_TIME:
log_debug('install->login',
'time=%d sec' % (datetime.utcnow() - start_time).seconds)
start_time = datetime.utcnow()
if RANDOM_DRIVER:
driver_num = randint(40, DRIVER_NUM)
package = DRIVER_PACKAGE + str(driver_num)
else:
package = DRIVER_PACKAGE + str(DRIVER_START + i)
if not install_driver(uid, package, VERSION):
log_err('instll', 'failed to install driver %s' % str(package))
return False
if SHOW_TIME:
log_debug('install',
'time=%d sec' % (datetime.utcnow() - start_time).seconds)
install_cnt += 1
log_debug('install', 'install_cnt=%d' % install_cnt)
def _login(self, user, password):
length = len(user)
if length > 0 and length < USERNAME_SIZE and re.match('^[0-9a-zA-Z]+$', user):
name = user + (USERNAME_SIZE - length) * '*'
else:
log_err(self, 'failed to login, invalid user name')
return
if len(password) != PASSWORD_SIZE:
log_err(self, 'failed to login, invalid password')
return
if EXPOSE:
mode = MODE_VISI
else:
mode = 0
cnt = LOGIN_RETRY_MAX
while cnt >= 0:
try:
req = Request(name, password)
res = req.user.login(node=get_node(), mode=mode)
if res:
log_debug(self, 'login, uid=%s, addr=%s' % (res['uid'], res['addr']))
if not channel.has_network(res['addr']):
return (res['uid'], res['addr'], res['token'], res['key'])
except:
pass
cnt -= 1
if cnt >= 0:
time.sleep(LOGIN_RETRY_INTERVAL)
log_err(self, 'failed to login')
def _request(self, op, args, event=None, timeout=None):
if not args:
log_debug(self, 'failed to request, no arguments')
return
ev = None
if event:
ev = self._channel_event.get(event['ev_type'], event['ev_name'])
if not ev:
log_err(self, 'failed to get event')
return
ret = None
try:
cli = zerorpc.Client()
cli.connect(zmqaddr(ADAPTER_ADDR, ADAPTER_PORT))
try:
cli.request(op, args);
finally:
cli.close()
if event:
if not timeout:
timeout = EV_TIMEOUT[event['ev_type']]
ret = self._channel_event.wait(ev, timeout)
finally:
if event:
err = self._channel_event.put(event['ev_type'], event['ev_name'])
if err != None:
ret = err
return ret
def request(self, op, *args, **kwargs):
log_debug('RPCClient', 'start to request, op=%s' % str(op))
buf = pack(op, args, kwargs)
res = self.dpmclient.request(self.addr, self.port, buf)
if res:
ret = loads(res)
return ret['res']
def _check_dep(path):
if os.path.isfile(path):
with open(path) as file_dependency:
lines = file_dependency.readlines()
for line in lines:
try:
package_version = ''
installer_name = ''
res = []
for str_equal in line.split('='):
if str_equal.strip(): # not blank
for str_blank in str_equal.split():
res.append(str_blank)
if len(res) % 2 == 0:
if len(res):
log_err('util', 'failed to check dependency, invalid format' )
return False
continue # if it is blank, then continue
else:
package_name = res[0]
for index_to_match in range(1, len(res), 2):
if res[index_to_match] == 'installer':
installer_name = res[index_to_match + 1]
continue
if res[index_to_match] == 'version':
package_version = res[index_to_match + 1]
continue
if installer_name == '':
installers = ['pip', 'apt-get']
for installer in installers:
installer_name = installer
if package_version == '':
cmd = '%s install %s' % (str(installer_name), str(package_name))
else :
cmd = '%s install %s==%s' % (str(installer_name), str(package_name), str(package_version))
status, output = commands.getstatusoutput(cmd)
if status == 0:
log_debug('util', 'check dependency, finished installing %s' % package_name)
break
if status != 0:
log_err('util', 'check dependency, invalid installer, failed to install %s' % str(package_name))
return False
else:
if package_version == '':
cmd = '%s install %s' % (str(installer_name), str(package_name))
else:
cmd = '%s install %s==%s' % (str(installer_name), str(package_name), str(package_version))
status, output = commands.getstatusoutput(cmd)
if status == 0:
log_debug('util', 'check dependency, finished installing %s' % str(package_name))
else:
log_err('util', 'check dependency, failed to install %s' % str(package_name))
return False
except:
continue # if it is blank, continue. else return False
return True
def get_attr(self, name, attr, typ):
buf = self._read(name, attr)
if buf:
try:
return typ(buf)
except:
log_debug(self, 'failed to convert %s to type %s' % (str(buf), str(typ)))
def upload(self, username, name, version, src_type, buf):
print "upload in backend starts!"
ret = self.rpcclient.request(
"upload", [username], {"name": name, "version": version, "src_type": src_type, "buf": buf}
)
log_debug("Backend.upload()", "the retrun of Repository.upload() is : %s" % str(ret))
return ret
def add_installer(allocator, installer):
log_debug('manage', 'add_installer, allocator=%s' % str(allocator))
rpcclient = RPCClient(allocator, ALLOCATOR_PORT)
res = rpcclient.request('add_installer', addr=installer)
if res:
log_debug('manage', 'add_installer, installer=%s' % str(installer))
return True
return False
def is_app_installed(username, name):
log_debug('log_app.is_app_installed()', 'is_app_installed() starts!')
register_info = shelve.open(PATH_INSTALL_LOG_APP)
if register_info.has_key(username):
if register_info[username].has_key(name):
return True
return False
def _release(self, addr, detach=True):
try:
self._disconnect(addr)
if self._channels.has_key(addr):
del self._channels[addr]
if detach == True:
self._detach(addr)
except:
log_debug(self, 'failed to release')
def _do_check(self, uid, node, addr):
token = self._query.token.get(uid)
if not token:
log_err(self, 'no token, uid=%s, addr=%s' % (str(uid), str(addr)))
return
if signaling.exist(uid, addr, token):
return token
else:
log_debug(self, 'no connection, uid=%s, addr=%s' % (str(uid), str(addr)))
def is_app_installed(username, name):
log_debug('log_app.is_app_installed()', 'is_app_installed() starts!')
register_info = shelve.open(PATH_INSTALL_LOG_APP)
if register_info.has_key(username):
if register_info[username].has_key(name):
return True
return False
def _allocate_channel_safe(self, uid, addr, key):
try:
channel.allocate(uid, addr, key)
return True
except:
log_debug(self, "failed to allocate channel, uid=%s, addr=%s" % (str(uid), str(addr)))
conductor.remove_key(uid, node)
conductor.remove_device(name)
return False
def test():
log_debug('has_package', 'start testing ')
login_cnt = 0
has_pkg_cnt = 0
for i in range(TEST_ROUNDS):
if SHOW_TIME:
start_time = datetime.utcnow()
username = USER + str(USER_START + i)
message = json.dumps({
'op': 'login',
'user': username,
'password': PASSWORD
})
ws = create_connection("ws://%s:%d/ws" % (get_manager(), get_port()))
ws.send(message)
ret = ws.recv()
ws.close()
result = json.loads(ret)
if not result or 'login' != result['op'] or username != result[
'user'] or not result['data']:
log_err('login', 'failed to login %s.' % str(username))
return False
uid = result['data'][0]
if not uid:
log_err('has_package->login',
'failed to login %s, invalid uid or key.' % str(username))
return False
if SHOW_TIME:
log_debug('has_package->login',
'time=%d sec' % (datetime.utcnow() - start_time).seconds)
login_cnt += 1
log_debug('has_package->login', 'login_cnt=%d' % login_cnt)
if SHOW_TIME:
start_time = datetime.utcnow()
package = PACKAGE + str(PKG_START + i)
message = json.dumps({
'op': 'has_package',
'uid': uid,
'package': package
})
ws = create_connection("ws://%s:%d/ws" % (get_manager(), get_port()))
ws.send(message)
ret = ws.recv()
ws.close()
result = json.loads(ret)
if not result or 'has_package' != result['op'] or package != result[
'package'] or not result['data']:
log_err('has_package', 'failed to install app %s' % str(package))
return False
if SHOW_TIME:
log_debug('has_package',
'time=%d sec' % (datetime.utcnow() - start_time).seconds)
has_pkg_cnt += 1
log_debug('has_package', 'has_pkg_cnt=%d' % has_pkg_cnt)
def _proc_safe(self, target, args, timeout):
ret = None
pool = ThreadPool(processes=1)
result = pool.apply_async(target, args=args)
try:
ret = result.get(timeout=timeout)
except TimeoutError:
log_debug(self, "timeout")
finally:
pool.terminate()
return ret
def set(self, ev_type, ev_args):
name = ev_args.get('name')
if name:
ev = self._event[ev_type].get(name)
if ev:
if ev_args.has_key('error'):
self._error[ev_type][name] = ev_args['error']
ev.set()
else:
log_debug(self, 'cannot set, no event, ev_type=%s, ev_args=%s' % (str(ev_type), str(ev_args)))
else:
log_debug(self, 'cannot set, no name, ev_type=%s, ev_args=%s' % (str(ev_type), str(ev_args)))
def _log(self, text):
if LOG_MASTER:
if self._port == USR_FINDER_PORT:
role = 'USR_FINDER@Master'
elif self._port == DEV_FINDER_PORT:
role = 'DEV_FINDER@Master'
elif self._port == USR_MAPPER_PORT:
role = 'USR_MAPPER@Master'
elif self._port == DEV_MAPPER_PORT:
role = 'DEV_MAPPER@Master'
else:
role = 'DATA_SERVER@Master'
log_debug(role, text)
def upload(self, addr, port, name, version, buf):
tmpdir = create_tmpdir()
zipfilename = generate_zipfile(name, version, buf, tmpdir)
zipfilepath = os.path.join(tmpdir, zipfilename)
self.ftp.connect(addr, port)
self.ftp.login(ADMIN, ADMINPASSWORD) # must be normal user, for anonymous users are forbidden to change the filesystem
ret = self.ftp.storbinary('STOR '+ zipfilename, open(zipfilepath, 'rb'), 1024)
log_debug('FTPClient.upload()', 'the return of FTP.storbinary is : %s' % str(ret))
self.ftp.quit()
delete_tmpdir(tmpdir)
return ret
def test():
log_debug('get_author', 'start testing ')
auth_cnt = 0
for i in range(TEST_ROUNDS):
if SHOW_TIME:
start_time = datetime.utcnow()
if RANDOM_PKG:
cat_num = randint(0, len(CAT) - 1)
cat_name = CAT[cat_num]
pkg_num = randint(0, PKG_NUM - 1)
package = cat_name + '_' + PKG + str(pkg_num)
else:
package = PACKAGE + str(PKG_START + i)
message = json.dumps({'operator':'get_author', 'package':package})
ws = create_connection("ws://%s:%d/" % (get_manager(), get_port()))
ws.send(message)
ret = ws.recv()
ws.close()
result = json.loads(ret)
if not result or 'get_author' != result['operator'] or package != result['package'] or not result['data']:
log_err('get_author', 'failed to get the author of %s ' % str(package))
return False
auth_cnt += 1
log_debug('get_author', 'auth_cnt=%d' % auth_cnt)
log_debug('get_author', 'the author of %s is %s' % (str(package), str(result['data'])))
if SHOW_TIME:
log_debug('get_author', 'time=%d sec' % (datetime.utcnow() - start_time).seconds)
def test():
log_debug('get_author', 'start testing ')
auth_cnt = 0
for i in range(TEST_ROUNDS):
if SHOW_TIME:
start_time = datetime.utcnow()
if RANDOM_PKG:
cat_num = randint(0, len(CATEGORIES.keys()) - 1)
cat_name = CATEGORIES.keys()[cat_num]
pkg_num = randint(50, PKG_NUM)
package = cat_name + '_' + PKG + str(pkg_num)
else:
package = PACKAGE + str(PKG_START + i)
message = json.dumps({'op':'get_author', 'package':package})
ws = create_connection("ws://%s:%d/ws" % (get_manager(), get_port()))
ws.send(message)
ret = ws.recv()
ws.close()
result = json.loads(ret)
if not result or 'get_author' != result['op'] or package != result['package'] or not result['data']:
log_err('get_author', 'failed to get the author of %s ' % str(package))
return False
auth_cnt += 1
log_debug('get_author', 'auth_cnt=%d' % auth_cnt)
log_debug('get_author', 'the author of %s is %s' % (str(package), str(result['data'])))
if SHOW_TIME:
log_debug('get_author', 'time=%d sec' % (datetime.utcnow() - start_time).seconds)
def test():
log_debug('register', 'start testing ')
get_top_details_cnt = 0
for i in range(TEST_ROUNDS):
if SHOW_TIME:
start_time = datetime.utcnow()
if RANDOM_CAT:
cat_num = randint(0, len(CAT) - 1)
cat_name = CAT[cat_num]
if not CATEGORIES.has_key(cat_name):
log_err('get_top_details', 'failed to get the top details, invalid category is %s' % str(cat_name))
return False
cate = CATEGORIES[cat_name]
else:
cate = CATEGORY
message = json.dumps({'operator':'get_top_details', 'category':cate})
ws = create_connection("ws://%s:%d/" % (get_manager(), get_port()))
ws.send(message)
ret = ws.recv()
ws.close()
result = json.loads(ret)
if not result or 'get_top_details' != result['operator'] or cate != result['category']:
log_err('get_top_details', 'failed to get the top packages details of %s' % str(cate))
return False
if SHOW_TIME:
log_debug('register', 'time=%d sec' % (datetime.utcnow() - start_time).seconds)
ret = result['data']
get_top_details_cnt += 1
log_debug('get_top_details', 'get_top_details_cnt=%d' % get_top_details_cnt)
log_debug('get_top_details', 'top details packages=%s' % str(ret))
def is_version_number_correct(name, version):
log_debug('log_app.is_version_number_correct()',
'is_version_number_correct() starts!')
release_info = shelve.open(PATH_UPLOAD_LOG_APP)
for username in release_info:
if release_info[username].has_key(name):
try:
release_info[username][name].index(version)
return True
except ValueError:
continue # no action, to perfect the format
return False
def _log(self, cmd):
if LOG_STUB:
if cmd == CMD_OPEN:
text = 'open'
elif cmd == CMD_CLOSE:
text = 'close'
elif cmd == CMD_GET:
text = 'get'
elif cmd == CMD_PUT:
text = 'put'
else:
text = 'unknown'
text += ', driver=%s, name=%s [%d]' % (str(self._driver), self._driver.get_name(), self._log_cnt)
log_debug(self, text)
self._log_cnt += 1
def test():
log_debug('get_counter', 'start testing ')
get_count = 0
for i in range(TEST_ROUNDS):
if SHOW_TIME:
start_time = datetime.utcnow()
if RANDOM_CAT:
cat_num = randint(0, len(CATEGORIES.keys()) - 1)
cat_name = CATEGORIES.keys()[cat_num]
if not CATEGORIES.has_key(cat_name):
log_err(
'get_counter',
'failed to get the category, invalid category is %s' %
str(cat_name))
return False
cate = CATEGORIES[cat_name]
else:
cate = CATEGORY
message = json.dumps({'op': 'get_counter', 'category': cate})
ws = create_connection("ws://%s:%d/ws" % (get_manager(), get_port()))
ws.send(message)
ret = ws.recv()
ws.close()
result = json.loads(ret)
if not result or 'get_counter' != result['op'] or cate != result[
'category'] or not result['data']:
log_err('get_counter', 'failed to get counter')
return False
counter = result['data']
if not counter:
log_err('get_counter',
'failed to get the total number of %s ' % str(cate))
return False
get_count += 1
log_debug('get_inst', 'get_count=%d' % get_count)
if SHOW_TIME:
log_debug('get_counter',
'time=%d sec' % (datetime.utcnow() - start_time).seconds)
log_debug('get_counter', 'counter=%s' % str(counter))
if int(counter) < 1:
rank = 0
else:
rank = randint(0, (int(counter) + PAGE_SIZE - 1) / PAGE_SIZE - 1)
log_debug('get_counter', 'rank=%d' % rank)
def upload(self, addr, port, name, version, buf):
tmpdir = create_tmpdir()
zipfilename = generate_zipfile(name, version, buf, tmpdir)
zipfilepath = os.path.join(tmpdir, zipfilename)
self.ftp.connect(addr, port)
self.ftp.login(
ADMIN, ADMINPASSWORD
) # must be normal user, for anonymous users are forbidden to change the filesystem
ret = self.ftp.storbinary('STOR ' + zipfilename,
open(zipfilepath, 'rb'), 1024)
log_debug('FTPClient.upload()',
'the return of FTP.storbinary is : %s' % str(ret))
self.ftp.quit()
delete_tmpdir(tmpdir)
return ret
def is_version_number_correct(name, version):
log_debug('log_app.is_version_number_correct()', 'is_version_number_correct() starts!')
release_info = shelve.open(PATH_UPLOAD_LOG_APP)
for username in release_info:
if release_info[username].has_key(name):
try :
release_info[username][name].index(version)
return True
except ValueError:
continue # no action, to perfect the format
return False
def _read(self, name, attr):
ret = None
path = self._get_path(name, attr)
self._log('loading, name=%s, attr=%s, path=%s' % (str(name), str(attr), str(path)))
try:
fd = os.open(path, os.O_RDONLY)
try:
ret = os.read(fd, LOAD_MAX)
finally:
os.close(fd)
except:
pass
if ret:
if len(ret) == LOAD_MAX:
log_debug(self, 'failed to load %s, invalid size' % str(name))
return
self._log('finished, name=%s, %s=%s' % (str(name), str(attr), str(ret)))
return ret
def test():
log_debug('register', 'start testing ')
login_cnt = 0
has_pkg_cnt = 0
for i in range(TEST_ROUNDS):
if SHOW_TIME:
start_time = datetime.utcnow()
username = USER + str(USER_START + i)
message = json.dumps({'operator':'login', 'user':username, 'password':PASSWORD})
ws = create_connection("ws://%s:%d/" % (get_manager(), get_port()))
ws.send(message)
ret = ws.recv()
ws.close()
result = json.loads(ret)
if not result or 'login' != result['operator'] or username != result['user'] or not result['data']:
log_err('login', 'failed to login %s.' % str(username))
return False
uid = result['data'][0]
if not uid:
log_err('has_package->login', 'failed to login %s, invalid uid or key.' % str(username))
return False
if SHOW_TIME:
log_debug('has_package->login', 'time=%d sec' % (datetime.utcnow() - start_time).seconds)
login_cnt += 1
log_debug('has_package->login', 'login_cnt=%d' % login_cnt)
if SHOW_TIME:
start_time = datetime.utcnow()
package = PACKAGE + str(PKG_START + i)
message = json.dumps({'operator':'has_package', 'uid':uid,'package':package})
ws = create_connection("ws://%s:%d/" % (get_manager(), get_port()))
ws.send(message)
ret = ws.recv()
ws.close()
result = json.loads(ret)
if not result or 'has_package' != result['operator'] or package != result['package'] or not result['data']:
log_err('has_package', 'failed to install app %s' % str(package))
return False
if SHOW_TIME:
log_debug('has_package', 'time=%d sec' % (datetime.utcnow() - start_time).seconds)
has_pkg_cnt += 1
log_debug('has_package', 'has_pkg_cnt=%d' % has_pkg_cnt)
def write_upload_log(username, name, version, src_type):
log_debug('upload_register.write_upload_log()',
'write_upload_log() starts!')
if src_type == APP:
path_upload_log = PATH_UPLOAD_LOG_APP
if src_type == DRIVER:
path_upload_log = PATH_UPLOAD_LOG_DRIVER
parent_dir = os.path.dirname(path_upload_log)
if not os.path.isdir(parent_dir):
os.makedirs(parent_dir)
release_info = shelve.open(path_upload_log, writeback=True)
if release_info.has_key(username):
if release_info[username].has_key(name):
release_info[username][name].append(version)
else:
release_info[username][name] = [version]
else:
release_info[username] = {name: [version]}
log_debug('upload_register.write_upload_log()',
"%s'%s upload logs are as follows:" % (username, src_type))
log_debug('upload_register.write_upload_log()',
'the release_info is : %s' % release_info[username])
def write_upload_log(username, name, version, src_type):
log_debug('upload_register.write_upload_log()', 'write_upload_log() starts!')
if src_type == APP:
path_upload_log = PATH_UPLOAD_LOG_APP
if src_type == DRIVER:
path_upload_log = PATH_UPLOAD_LOG_DRIVER
parent_dir = os.path.dirname(path_upload_log)
if not os.path.isdir(parent_dir):
os.makedirs(parent_dir)
release_info = shelve.open(path_upload_log, writeback = True)
if release_info.has_key(username):
if release_info[username].has_key(name):
release_info[username][name].append(version)
else :
release_info[username][name] = [version]
else :
release_info[username] = {name:[version]}
log_debug('upload_register.write_upload_log()', "%s'%s upload logs are as follows:" % (username, src_type))
log_debug('upload_register.write_upload_log()', 'the release_info is : %s' % release_info[username])
def test():
log_debug('get_categories', 'start testing ')
cat_cnt = 0
for _ in range(TEST_ROUNDS):
if SHOW_TIME:
start_time = datetime.utcnow()
message = json.dumps({'operator':'get_categories'})
ws = create_connection("ws://%s:%d/" % (get_manager(), get_port()))
ws.send(message)
ret = ws.recv()
ws.close()
result = json.loads(ret)
if not result or 'get_categories' != result['operator'] or not result['data']:
log_err('get_categories', 'failed to get categories')
return False
cat_cnt += 1
log_debug('get_categories', 'cat_cnt=%d' % cat_cnt)
log_debug('get_categories', 'categories=%s' % str(result['data']))
if SHOW_TIME:
log_debug('get_categories', 'time=%d sec' % (datetime.utcnow() - start_time).seconds)
def test():
log_debug('get_categories', 'start testing ')
cat_cnt = 0
for _ in range(TEST_ROUNDS):
if SHOW_TIME:
start_time = datetime.utcnow()
message = json.dumps({'op': 'get_categories'})
ws = create_connection("ws://%s:%d/ws" % (get_manager(), get_port()))
ws.send(message)
ret = ws.recv()
ws.close()
result = json.loads(ret)
if not result or 'get_categories' != result['op'] or not result['data']:
log_err('get_categories', 'failed to get categories')
return False
cat_cnt += 1
log_debug('get_categories', 'cat_cnt=%d' % cat_cnt)
log_debug('get_categories', 'categories=%s' % str(result['data']))
if SHOW_TIME:
log_debug('get_categories',
'time=%d sec' % (datetime.utcnow() - start_time).seconds)
def test():
log_debug('get_top_details', 'start testing ')
get_top_details_cnt = 0
for i in range(TEST_ROUNDS):
if SHOW_TIME:
start_time = datetime.utcnow()
if RANDOM_CAT:
cat_num = randint(0, len(CATEGORIES.keys()) - 1)
cat_name = CATEGORIES.keys()[cat_num]
if not CATEGORIES.has_key(cat_name):
log_err(
'get_top_details',
'failed to get the top details, invalid category is %s' %
str(cat_name))
return False
cate = CATEGORIES[cat_name]
else:
cate = CATEGORY
message = json.dumps({'op': 'get_top_details', 'category': cate})
ws = create_connection("ws://%s:%d/ws" % (get_manager(), get_port()))
ws.send(message)
ret = ws.recv()
ws.close()
result = json.loads(ret)
if not result or 'get_top_details' != result['op'] or cate != result[
'category']:
log_err('get_top_details',
'failed to get the top packages details of %s' % str(cate))
return False
if SHOW_TIME:
log_debug('get_top_details',
'time=%d sec' % (datetime.utcnow() - start_time).seconds)
ret = result['data']
get_top_details_cnt += 1
log_debug('get_top_details',
'get_top_details_cnt=%d' % get_top_details_cnt)
log_debug('get_top_details', 'top details packages=%s' % str(ret))
def write_upload_log(username, name, version):
log_debug('log_app.write_upload_log()', 'write_upload_log() starts!')
release_info = shelve.open(PATH_UPLOAD_LOG_APP, writeback = True)
if release_info.has_key(username):
if release_info[username].has_key(name):
release_info[username][name].append(version)
else :
release_info[username][name] = [version]
else :
release_info[username] = {name:[version]}
log_debug('log_app.write_upload_log()', "%s'app upload logs are as follows:" % username)
log_debug('log_app.write_upload_log()', 'the release_info is : %s' % release_info[username])
def test():
log_debug('upload', 'start testing ')
login_cnt = 0
upload_cnt = 0
for i in range(TEST_ROUNDS):
if SHOW_TIME:
start_time = datetime.utcnow()
username = USER + str(USER_START + i)
uid, key = login(username, PASSWORD)
if not uid or not key:
log_err('upload->login', 'failed to login %s.' % str(username))
return False
login_cnt += 1
log_debug('upload->login', 'login_cnt=%d' % login_cnt)
package = DRIVER_PACKAGE + str(DRIVER_START + i)
if not upload(DRIVER_PATH, uid, package, VERSION, DRIVER, key):
log_err('upload', 'failed to upload driver %s' % str(package))
return False
if SHOW_TIME:
log_debug('upload', 'time=%d sec' % (datetime.utcnow() - start_time).seconds)
upload_cnt += 1
log_debug('upload', 'upload_cnt=%d' % upload_cnt)
def write_upload_log(username, name, version):
log_debug('log_app.write_upload_log()', 'write_upload_log() starts!')
release_info = shelve.open(PATH_UPLOAD_LOG_APP, writeback=True)
if release_info.has_key(username):
if release_info[username].has_key(name):
release_info[username][name].append(version)
else:
release_info[username][name] = [version]
else:
release_info[username] = {name: [version]}
log_debug('log_app.write_upload_log()',
"%s'app upload logs are as follows:" % username)
log_debug('log_app.write_upload_log()',
'the release_info is : %s' % release_info[username])
def test():
log_debug('upload', 'start testing ')
login_cnt = 0
upload_cnt = 0
for i in range(TEST_ROUNDS):
if SHOW_TIME:
start_time = datetime.utcnow()
username = USER + str(USER_START + i)
uid, key = login(username, PASSWORD)
if not uid or not key:
log_err('upload->login', 'failed to login %s.' % str(username))
return False
login_cnt += 1
log_debug('upload->login', 'login_cnt=%d' % login_cnt)
package = DRIVER_PACKAGE + str(DRIVER_START + i)
if not upload(DRIVER_PATH, uid, package, VERSION, DRIVER, key):
log_err('upload', 'failed to upload driver %s' % str(package))
return False
if SHOW_TIME:
log_debug('upload',
'time=%d sec' % (datetime.utcnow() - start_time).seconds)
upload_cnt += 1
log_debug('upload', 'upload_cnt=%d' % upload_cnt)
def write_install_log(username, name, version, output):
log_debug('log_app.write_install_log()', 'write_install_log() starts!')
register_info = shelve.open(PATH_INSTALL_LOG_APP, writeback = True)
if(register_info.has_key(username)):
if(register_info[username].has_key(name)):
register_info[username][name][version] = output
else:
register_info[username][name] = {version:output}
else:
register_info[username] = {name:{version:output}}
log_debug('log_app.write_install_log()', "%s'app install logs are as follows:" % username)
log_debug('log_app.write_install_log()', 'the register_info is : %s' % register_info[username])
register_info.close()
def write_install_log(username, name, version, output):
log_debug('log_app.write_install_log()', 'write_install_log() starts!')
register_info = shelve.open(PATH_INSTALL_LOG_APP, writeback=True)
if (register_info.has_key(username)):
if (register_info[username].has_key(name)):
register_info[username][name][version] = output
else:
register_info[username][name] = {version: output}
else:
register_info[username] = {name: {version: output}}
log_debug('log_app.write_install_log()',
"%s'app install logs are as follows:" % username)
log_debug('log_app.write_install_log()',
'the register_info is : %s' % register_info[username])
register_info.close()
def test():
log_debug('register', 'start testing ')
register_cnt = 0
for i in range(TEST_ROUNDS):
if SHOW_TIME:
start_time = datetime.utcnow()
username = USER + str(USER_START + i)
message = json.dumps({'operator':'register', 'user':username, 'password':PASSWORD, 'email':EMAIL})
ws = create_connection("ws://%s:%d/" % (get_manager(), get_port()))
ws.send(message)
ret = ws.recv()
ws.close()
result = json.loads(ret)
if not result or 'register' != result['operator'] or username != result['user'] or not result['data']:
log_err('register', 'failed to register %s.' % str(username))
return False
register_cnt += 1
log_debug('register', 'register_cnt=%d' % register_cnt)
if SHOW_TIME:
log_debug('register', 'test, time=%d sec' % (datetime.utcnow() - start_time).seconds)
def test():
log_debug('register', 'start testing ')
register_cnt = 0
for i in range(TEST_ROUNDS):
if SHOW_TIME:
start_time = datetime.utcnow()
username = USER + str(USER_START + i)
message = json.dumps({'op':'register', 'user':username, 'password':PASSWORD, 'email':EMAIL})
ws = create_connection("ws://%s:%d/ws" % (get_manager(), get_port()))
ws.send(message)
ret = ws.recv()
ws.close()
result = json.loads(ret)
if not result or 'register' != result['op'] or username != result['user'] or not result['data']:
log_err('register', 'failed to register %s.' % str(username))
return False
register_cnt += 1
log_debug('register', 'register_cnt=%d' % register_cnt)
if SHOW_TIME:
log_debug('register', 'test, time=%d sec' % (datetime.utcnow() - start_time).seconds)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-d', '--driver', action='store_true')
parser.add_argument('-a', '--app', action='store_true')
parser.add_argument('-i', dest='uid', default=None)
parser.add_argument('-u', dest='username', default=None)
parser.add_argument('-p', dest='password', default=None)
parser.add_argument('-v', dest='version', default=None)
parser.add_argument('-s', dest='source', default=None)
if len(sys.argv) == 1:
parser.print_help()
sys.exit()
cmd = sys.argv[1]
if cmd == 'login':
args = parser.parse_args(sys.argv[2:])
uid, key = login(args.username, args.password)
log_debug('dpm',
'finished login, uid=%s, key=%s' % (str(uid), str(key)))
sys.exit()
else:
package = sys.argv[2]
args = parser.parse_args(sys.argv[3:])
user = args.username
password = args.password
version = args.version
path = args.source
uid = args.uid
if len(package.split('==')) > 1:
package, version = get_version(package)
if args.app:
if cmd == 'upload':
uid, key = login(user, password)
if upload_app(path, uid, package, version, key):
log_debug('dpm', 'finished uploading app %s' % package)
elif cmd == 'install':
uid, key = login(user, password)
if install_app(uid, package, version):
log_debug('dpm', 'finished installing app %s' % package)
elif cmd == 'uninstall':
uid, key = login(user, password)
if uninstall_app(uid, package):
log_debug('dpm', 'finished removing app %s' % package)
elif args.driver:
if cmd == 'upload':
uid, key = login(user, password)
if upload_driver(path, uid, package, version):
log_debug('dpm', 'finished uploading driver %s' % package)
elif cmd == 'install':
uid, key = login(user, password)
if install_driver(uid, package, version):
log_debug('dpm', 'finished installing driver %s' % package)
def test():
log_debug('install', 'start testing ')
login_cnt = 0
install_cnt = 0
get_top_cnt = 0
has_pkg_cnt = 0
download_cnt = 0
get_top_details_cnt = 0
for i in range(TEST_ROUNDS):
if SHOW_TIME:
start_time = datetime.utcnow()
username = USER + str(USER_START + i)
message = json.dumps({
'op': 'login',
'user': username,
'password': PASSWORD
})
ws = create_connection("ws://%s:%d/ws" % (get_manager(), get_port()))
ws.send(message)
ret = ws.recv()
ws.close()
result = json.loads(ret)
if not result or 'login' != result['op'] or username != result[
'user'] or not result['data']:
log_err('login', 'failed to login %s.' % str(username))
return False
uid = result['data'][0]
key = result['data'][1]
if not uid or not key:
log_err('install->login',
'failed to login %s, invalid uid or key.' % str(username))
return False
if SHOW_TIME:
log_debug('install->login',
'time=%d sec' % (datetime.utcnow() - start_time).seconds)
login_cnt += 1
log_debug('install->login', 'login_cnt=%d' % login_cnt)
if SHOW_TIME:
start_time = datetime.utcnow()
if RANDOM_PKG:
cat_num = randint(0, len(CATEGORIES.keys()) - 1)
cat_name = CATEGORIES.keys()[cat_num]
pkg_num = randint(50, PKG_NUM)
package = cat_name + '_' + PKG + str(pkg_num)
else:
package = PACKAGE + str(PKG_START + i)
message = json.dumps({
'op': 'download',
'uid': uid,
'package': package,
'version': VERSION
})
ws = create_connection("ws://%s:%d/ws" % (get_manager(), get_port()))
ws.send(message)
ret = ws.recv()
ws.close()
result = json.loads(ret)
log_debug('download', 'result=%s' % result)
if not result or 'download' != result['op'] or uid != result[
'uid'] or not result['data']:
log_err('download', 'failed to download app %s' % str(package))
return False
ret = result['data']
if not ret:
log_err(
'download',
'failed to download app %s, invalid return message' %
str(package))
return False
download_cnt += 1
log_debug('install', 'download_cnt=%d' % download_cnt)
if SHOW_TIME:
log_debug('download',
'time=%d sec' % (datetime.utcnow() - start_time).seconds)
if SHOW_TIME:
start_time = datetime.utcnow()
message = json.dumps({
'op': 'install',
'uid': uid,
'package': package,
'version': VERSION,
'type': APP,
'content': ret
})
ws = create_connection("ws://%s:%d/ws" % (get_manager(), get_port()))
ws.send(message)
ret = ws.recv()
ws.close()
result = json.loads(ret)
if not result or 'install' != result['op'] or uid != result[
'uid'] or not result['data']:
log_err('install', 'failed to install app %s' % str(package))
return False
ret = result['data']
if not ret:
log_err(
'install', 'failed to install app %s, invalid return message' %
str(package))
return False
install_cnt += 1
log_debug('install', 'install_cnt=%d' % install_cnt)
if SHOW_TIME:
log_debug('install',
'time=%d sec' % (datetime.utcnow() - start_time).seconds)
if SHOW_TIME:
start_time = datetime.utcnow()
message = json.dumps({
'op': 'has_package',
'uid': uid,
'package': package
})
ws = create_connection("ws://%s:%d/ws" % (get_manager(), get_port()))
ws.send(message)
ret = ws.recv()
ws.close()
result = json.loads(ret)
if not result or 'has_package' != result['op'] or package != result[
'package'] or not result['data']:
log_err('install-->has_package',
'failed to has app %s' % str(package))
return False
has_pkg_cnt += 1
log_debug('has_package', 'has_pkg_cnt=%d' % has_pkg_cnt)
if SHOW_TIME:
log_debug('has_package',
'time=%d sec' % (datetime.utcnow() - start_time).seconds)
if not CATEGORIES.has_key(cat_name):
log_err(
'get_top', 'failed to get the top, invalid category is %s' %
str(cat_name))
return False
cate = CATEGORIES[cat_name]
message = json.dumps({'op': 'get_top', 'category': cate})
ws = create_connection("ws://%s:%d/ws" % (get_manager(), get_port()))
ws.send(message)
ret = ws.recv()
ws.close()
result = json.loads(ret)
if not result or 'get_top' != result['op'] or cate != result[
'category']:
log_err('get_top',
'failed to get the top packages of %s' % str(cate))
return False
ret = result['data']
get_top_cnt += 1
log_debug('get_top', 'get_top_cnt=%d' % get_top_cnt)
log_debug('get_top', 'top packages=%s' % str(ret))
message = json.dumps({'op': 'get_top_details', 'category': cate})
ws = create_connection("ws://%s:%d/ws" % (get_manager(), get_port()))
ws.send(message)
ret = ws.recv()
ws.close()
result = json.loads(ret)
if not result or 'get_top_details' != result['op'] or cate != result[
'category']:
log_err('get_top_details',
'failed to get the top packages details of %s' % str(cate))
return False
ret = result['data']
get_top_details_cnt += 1
log_debug('get_top_details',
'get_top_details_cnt=%d' % get_top_details_cnt)
log_debug('get_top_details', 'top details packages=%s' % str(ret))
def _log(self, text):
if LOG_ATTR:
log_debug(self, text)
def upload(self, username, name, version, src_type, buf):
print 'upload in backend starts!'
ret = self.rpcclient.request('upload', [username], {'name':name, 'version':version, 'src_type':src_type, 'buf':buf})
log_debug('Backend.upload()', 'the retrun of Repository.upload() is : %s' % str(ret))
return ret
def upload_src(self, name, version, buf):
addr, port = calculate_the_ftpserver_addr()
ret = self.ftpclient.upload(addr, port, name, version, buf)
log_debug('Repository.upload()', 'the return of FTPClient.upload() is : %s' % str(ret))
return ret
def _check_dep(path):
if os.path.isfile(path):
with open(path) as file_dependency:
lines = file_dependency.readlines()
for line in lines:
try:
package_version = ''
installer_name = ''
res = []
for str_equal in line.split('='):
if str_equal.strip(): # not blank
for str_blank in str_equal.split():
res.append(str_blank)
if len(res) % 2 == 0:
if len(res):
log_err(
'util',
'failed to check dependency, invalid format')
return False
continue # if it is blank, then continue
else:
package_name = res[0]
for index_to_match in range(1, len(res), 2):
if res[index_to_match] == 'installer':
installer_name = res[index_to_match + 1]
continue
if res[index_to_match] == 'version':
package_version = res[index_to_match + 1]
continue
if installer_name == '':
installers = ['pip', 'apt-get']
for installer in installers:
installer_name = installer
if package_version == '':
cmd = '%s install %s' % (
str(installer_name), str(package_name))
else:
cmd = '%s install %s==%s' % (
str(installer_name), str(package_name),
str(package_version))
status, output = commands.getstatusoutput(cmd)
if status == 0:
log_debug(
'util',
'check dependency, finished installing %s'
% package_name)
break
if status != 0:
log_err(
'util',
'check dependency, invalid installer, failed to install %s'
% str(package_name))
return False
else:
if package_version == '':
cmd = '%s install %s' % (str(installer_name),
str(package_name))
else:
cmd = '%s install %s==%s' % (
str(installer_name), str(package_name),
str(package_version))
status, output = commands.getstatusoutput(cmd)
if status == 0:
log_debug(
'util',
'check dependency, finished installing %s'
% str(package_name))
else:
log_err(
'util',
'check dependency, failed to install %s' %
str(package_name))
return False
except:
continue # if it is blank, continue. else return False
return True
def show_info(cls, text):
log_debug(cls.__class__.__name__, text)
def test():
log_debug('upload', 'start testing ')
login_cnt = 0
upload_cnt = 0
get_pkgs_details_cnt = 0
for i in range(TEST_ROUNDS):
if SHOW_TIME:
start_time = datetime.utcnow()
username = USER + str(USER_START + i)
uid, key = login(username, PASSWORD)
if not uid or not key:
log_err('upload->login', 'failed to login %s.' % str(username))
return False
login_cnt += 1
log_debug('upload->login', 'login_cnt=%d' % login_cnt)
package = PACKAGE + str(50 + i)
if not upload(PATH, uid, package, VERSION, APP, key):
log_err('upload', 'failed to upload app %s' % str(package))
return False
if SHOW_TIME:
log_debug('upload',
'time=%d sec' % (datetime.utcnow() - start_time).seconds)
upload_cnt += 1
log_debug('upload', 'upload_cnt=%d' % upload_cnt)
if SHOW_TIME:
start_time = datetime.utcnow()
message = json.dumps({'op': 'get_counter', 'category': CATEGORY})
ws = create_connection("ws://%s:%d/ws" % (get_manager(), get_port()))
ws.send(message)
ret = ws.recv()
ws.close()
result = json.loads(ret)
if not result or 'get_counter' != result['op'] or CATEGORY != result[
'category'] or not result['data']:
log_err('get_counter', 'failed to get counter')
return False
counter = result['data']
if not counter:
log_err('get_counter',
'failed to get the total number of %s ' % str(CATEGORY))
return False
if SHOW_TIME:
log_debug('get_counter',
'time=%d sec' % (datetime.utcnow() - start_time).seconds)
log_debug('get_counter', 'counter=%s' % str(counter))
if int(counter) < 1:
rank = 0
else:
rank = randint(0, (int(counter) + PAGE_SIZE - 1) / PAGE_SIZE - 1)
log_debug('get_counter', 'rank=%d' % rank)
if SHOW_TIME:
start_time = datetime.utcnow()
message = json.dumps({
'op': 'get_packages_details',
'category': CATEGORY,
'rank': rank
})
ws = create_connection("ws://%s:%d/ws" % (get_manager(), get_port()))
ws.send(message)
ret = ws.recv()
ws.close()
result = json.loads(ret)
if not result or 'get_packages_details' != result[
'op'] or CATEGORY != result['category'] or rank != result[
'rank'] or not result['data']:
log_err('get_packages_details', 'failed to get packages details')
return False
ret = result['data']
for item in ret:
if not item['pkg'] or not item['title'] or not item['auth']:
log_err('get_packages_details', 'failed to get valid details')
return False
if SHOW_TIME:
log_debug('get_packages_details',
'time=%d sec' % (datetime.utcnow() - start_time).seconds)
get_pkgs_details_cnt += 1
log_debug('get_packages_details',
'get_pkgs_details_cnt=%d' % get_pkgs_details_cnt)
log_debug('get_packages_details', 'packages_details=%s' % str(ret))
def test():
log_debug('get_installed_packages', 'start testing ')
login_cnt = 0
get_inst_pkgs_cnt = 0
for i in range(TEST_ROUNDS):
if SHOW_TIME:
start_time = datetime.utcnow()
username = USER + str(USER_START + i)
message = json.dumps({'op':'login', 'user':username, 'password':PASSWORD})
ws = create_connection("ws://%s:%d/ws" % (get_manager(), get_port()))
ws.send(message)
ret = ws.recv()
ws.close()
result = json.loads(ret)
if not result or 'login' != result['op'] or username != result['user'] or not result['data']:
log_err('get_installed_packages->login', 'failed to login %s.' % str(username))
return False
uid = result['data'][0]
if not uid:
log_err('get_installed_packages->login', 'failed to login %s, invalid uid.' % str(username))
return False
if SHOW_TIME:
log_debug('get_installed_packages->login', 'time=%d sec' % (datetime.utcnow() - start_time).seconds)
login_cnt += 1
log_debug('get_installed_packages->login', 'login_cnt=%d' % login_cnt)
if SHOW_TIME:
start_time = datetime.utcnow()
message = json.dumps({'op':'get_installed_packages', 'uid':uid})
ws = create_connection("ws://%s:%d/ws" % (get_manager(), get_port()))
ws.send(message)
ret = ws.recv()
ws.close()
result = json.loads(ret)
if not result or 'get_installed_packages' != result['op'] or uid != result['uid']:
log_err('get_installed_packages', 'failed to get installed packages')
return False
get_inst_pkgs_cnt += 1
log_debug('get_installed_packages', 'get_inst_pkgs_cnt=%d' % get_inst_pkgs_cnt)
log_debug('get_installer_packages', 'the packages of %s has installed are %s' % (str(username), str(result['data'])))
if SHOW_TIME:
log_debug('get_installed_packages', 'time=%d sec' % (datetime.utcnow() - start_time).seconds)
def _log(self, text):
if LOG_REMOTEFS:
log_debug(self, text)
def upload(self, username, name, version, src_type, buf):
#rpcclient = RPCClient('127.0.0.1', 9002)
ret = self.rpcclient.request('upload', [username], {'name':name, 'version':version, 'src_type':src_type, 'buf':buf})
log_debug('Frontend.upload()', 'the return of Backend.upload() is : %s' % str(ret))
#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! lack a judgement of returning True or False
return True