class Repository(RPCServer):
def _print(self, text):
if PRINT:
show_info(self, text)
def __init__(self, addr, port):
RPCServer.__init__(self, addr, port)
len_up = len(UPLOAD_SERVERS)
len_repo = len(REPOSITORY_SERVERS)
if len_up < len_repo or len_up % len_repo != 0:
show_error(self, 'failed to initialize')
raise Exception('failed to initialize')
addr = localhost()
if addr not in REPOSITORY_SERVERS:
show_error(self, 'failed to initialize')
raise Exception('failed to initialize REPOSITORY_SERVERS')
for i in range(len(REPOSITORY_SERVERS)):
if addr == REPOSITORY_SERVERS[i]:
break
total = len_up / len_repo
self._upload_servers = UPLOAD_SERVERS[i * total:(i + 1) * total]
self._print('upload_servers=%s' % str(self._upload_servers))
if HDFS:
self._port = HDFS_PORT
self._client = HDFSClient()
else:
self._port = FTP_PORT
self._client = FTPClient()
self._server = FTPServer()
if REPO_DB:
self._db = Database(addr=REPO_DB)
else:
self._db = Database(addr=addr)
locks = []
for _ in range(LOCK_MAX):
locks.append(Lock())
self._locks = HashRing(locks)
if DEBUG:
self._upload_cnt = 0
self._download_cnt = 0
def _get_addr(self, package):
ring = HashRing(self._upload_servers)
addr = ring.get_node(package)
return addr
def _get_lock(self, package):
return self._locks.get_node(package)
def _upload(self, package, version, buf):
addr = self._get_addr(package)
return self._client.upload(addr, self._port, package, version, buf)
def upload(self, uid, package, version, buf):
self._print('start to upload, uid=%s, package=%s, version=%s' %
(str(uid), str(package), str(version)))
lock = self._get_lock(package)
lock.acquire()
try:
if SHOW_TIME:
start_time = datetime.utcnow()
owner, ver = self._db.get_version(package)
if owner and owner != uid:
show_error(
self,
'failed to upload, invalid owner, package=%s, version=%s' %
(str(package), str(version)))
return False
if ver == version: #The version of package has been uploaded.
show_error(
self,
'failed to upload, invalid version, package=%s, version=%s'
% (str(package), str(version)))
return False
else:
ret = self._upload(package, version, buf)
if not ret:
show_error(
self,
'failed to upload, %s cannot upload %s-%s to hdfs' %
(str(uid), str(package), str(version)))
return False
self._db.set_package(uid, package, version, '')
if not ver or ver < version:
self._db.set_version(uid, package, version)
self._print('finished uploading, package=%s, version=%s' %
(str(package), str(version)))
if DEBUG:
self._upload_cnt += 1
self._print('upload, count=%d' % self._upload_cnt)
if SHOW_TIME:
self._print('upload, time=%d sec' %
(datetime.utcnow() - start_time).seconds)
return True
finally:
lock.release()
def download(self, package, version):
self._print('start to download, package=%s, version=%s' %
(str(package), str(version)))
try:
if SHOW_TIME:
start_time = datetime.utcnow()
addr = self._get_addr(package)
uid, ver = self._db.get_version(package)
if not version:
version = ver
if not self._db.has_package(uid, package, version):
show_error(
self,
'failed to download, invalid version, package=%s, version=%s'
% (str(package), str(version)))
return
if DEBUG:
self._download_cnt += 1
self._print('download, count=%d' % self._download_cnt)
if SHOW_TIME:
self._print('download, time=%d sec' %
(datetime.utcnow() - start_time).seconds)
return self._client.download(addr, self._port, package, version)
except:
show_error(self, 'failed to download')
def version(self, package):
self._print('start to get version, uid=%s, package=%s' %
(str(uid), str(package)))
_, ver = self._db.get_version(package)
return ver
def start(self):
t = Thread(target=self.run)
t.start()
if not HDFS:
self._server.run()
t.join()
class App():
def __init__(self):
self._lock = Lock()
if APP_DB:
self._db = Database(addr=APP_DB, domain=APP)
else:
self._db = Database(addr=localhost(), domain=APP)
def _print(self, text):
if PRINT:
show_info(self, text)
def _get_repo(self, package):
ring = HashRing(REPOSITORY_SERVERS)
server = ring.get_node(package)
return server
def _install(self, buf):
dirname = tempfile.mkdtemp()
try:
src = os.path.join(dirname, 'app.zip')
with open(src, 'wb') as f:
f.write(buf)
dest = os.path.join(dirname, 'app')
unzip_file(src, dest)
cmd = 'python %s %s' % (PATH_INSTALLER, dest)
status, output = commands.getstatusoutput(cmd)
if status == 0:
return output
finally:
shutil.rmtree(dirname)
def install(self, uid, package, version):
self._lock.acquire()
try:
if self._db.has_package(uid, package, None):
show_error(
self,
'failed to install, cannot install %s again' % package)
return
else:
addr = self._get_repo(package)
rpcclient = RPCClient(addr, REPOSITORY_PORT)
if not version:
version = rpcclient.request('version', package=package)
if not version:
show_error(
self,
'failed to install, invalid version, uid=%s, package=%s'
% (uid, package))
return
ret = rpcclient.request('download',
package=package,
version=version)
if ret:
result = self._install(ret)
if result:
self._db.set_package(uid, package, version, result)
self._print('finished installing %s, version=%s' %
(package, version))
return True
show_error(self, 'failed to install %s' % package)
return
finally:
self._lock.release()
def _uninstall(self, uid, package, info):
pass
def uninstall(self, uid, package):
self._lock.acquire()
try:
if not self._db.has_package(uid, package, None):
show_error(self, 'failed to uninstall %s ' % package)
return
version, info = self._db.get_package(uid, package, None)
if info:
self._uninstall(uid, package, info)
self._db.rm_package(uid, package, version)
return True
finally:
self._lock.release()
def get_packages(self, uid):
return self._db.get_packages(uid)
def has_package(self, uid, package):
return self._db.has_package(uid, package)
class App():
def __init__(self):
self._lock = Lock()
if SERVER_APPDB:
self._db = Database(addr=SERVER_APPDB[0], domain=APP)
else:
self._db = Database(addr=localhost(), domain=APP)
def _print(self, text):
if LOG_APP:
show_info(self, text)
def _get_repo(self, package):
ring = HashRing(SERVER_REPO)
server = ring.get_node(package)
return server
def _install(self, uid, content):
if content:
return True
# output = installer.install(uid, content)
# if output:
# return output
else:
return str(None)
def install(self, uid, package, version, content):
self._lock.acquire()
try:
if self._db.has_package(uid, package, None):
show_error(
self,
'failed to install, cannot install %s again' % package)
return
else:
result = self._install(uid, content)
if result:
self._db.set_package(uid, package, version, result)
self._print('finished installing %s, version=%s' %
(package, version))
return True
finally:
self._lock.release()
def _uninstall(self, uid, package, info):
pass
def uninstall(self, uid, package):
self._lock.acquire()
try:
if not self._db.has_package(uid, package, None):
show_error(self, 'failed to uninstall %s' % package)
return
version, info = self._db.get_package(uid, package, None)
if info:
self._uninstall(uid, package, info)
self._db.rm_package(uid, package, version)
return True
finally:
self._lock.release()
def get_packages(self, uid):
return self._db.get_packages(uid)
def has_package(self, uid, package):
return self._db.has_package(uid, package)
class App():
def __init__(self):
self._lock = Lock()
if APP_DB:
self._db = Database(addr=APP_DB, domain=APP)
else:
self._db = Database(addr=localhost(), domain=APP)
def _print(self, text):
if PRINT:
show_info(self, text)
def _get_repo(self, package):
ring = HashRing(REPOSITORY_SERVERS)
server = ring.get_node(package)
return server
def _install(self, buf):
dirname = tempfile.mkdtemp()
try:
src = os.path.join(dirname, 'app.zip')
with open(src, 'wb') as f:
f.write(buf)
dest = os.path.join(dirname, 'app')
unzip_file(src, dest)
cmd = 'python %s %s' % (PATH_INSTALLER, dest)
status, output = commands.getstatusoutput(cmd)
if status == 0:
return output
finally:
shutil.rmtree(dirname)
def install(self, uid, package, version):
self._lock.acquire()
try:
if self._db.has_package(uid, package, None):
show_error(self, 'failed to install, cannot install %s again' % package)
return
else:
addr = self._get_repo(package)
rpcclient = RPCClient(addr, REPOSITORY_PORT)
if not version:
version = rpcclient.request('version', package=package)
if not version:
show_error(self, 'failed to install, invalid version, uid=%s, package=%s' % (uid, package))
return
ret = rpcclient.request('download', package=package, version=version)
if ret:
result = self._install(ret)
if result:
self._db.set_package(uid, package, version, result)
self._print('finished installing %s, version=%s' % (package, version))
return True
show_error(self, 'failed to install %s' % package)
return
finally:
self._lock.release()
def _uninstall(self, uid, package, info):
pass
def uninstall(self, uid, package):
self._lock.acquire()
try:
if not self._db.has_package(uid, package, None):
show_error(self, 'failed to uninstall %s ' % package)
return
version, info = self._db.get_package(uid, package, None)
if info:
self._uninstall(uid, package, info)
self._db.rm_package(uid, package, version)
return True
finally:
self._lock.release()
def get_packages(self, uid):
return self._db.get_packages(uid)
def has_package(self, uid, package):
return self._db.has_package(uid, package)
class Repository(RPCServer):
def _print(self, text):
if PRINT:
show_info(self, text)
def __init__(self, addr, port):
RPCServer.__init__(self, addr, port)
len_up = len(UPLOAD_SERVERS)
len_repo = len(REPOSITORY_SERVERS)
if len_up < len_repo or len_up % len_repo != 0:
show_error(self, 'failed to initialize')
raise Exception('failed to initialize')
addr = localhost()
if addr not in REPOSITORY_SERVERS:
show_error(self, 'failed to initialize')
raise Exception('failed to initialize REPOSITORY_SERVERS')
for i in range(len(REPOSITORY_SERVERS)):
if addr == REPOSITORY_SERVERS[i]:
break
total = len_up / len_repo
self._upload_servers = UPLOAD_SERVERS[i * total:(i + 1) * total]
self._print('upload_servers=%s' % str(self._upload_servers))
if HDFS:
self._port = HDFS_PORT
self._client = HDFSClient()
else:
self._port = FTP_PORT
self._client = FTPClient()
self._server = FTPServer()
if REPO_DB:
self._db = Database(addr=REPO_DB)
else:
self._db = Database(addr=addr)
locks = []
for _ in range(LOCK_MAX):
locks.append(Lock())
self._locks = HashRing(locks)
if DEBUG:
self._upload_cnt = 0
self._download_cnt = 0
def _get_addr(self, package):
ring = HashRing(self._upload_servers)
addr = ring.get_node(package)
return addr
def _get_lock(self, package):
return self._locks.get_node(package)
def _upload(self, package, version, buf):
addr = self._get_addr(package)
return self._client.upload(addr, self._port, package, version, buf)
def upload(self, uid, package, version, buf):
self._print('start to upload, uid=%s, package=%s, version=%s' % (str(uid), str(package), str(version)))
lock = self._get_lock(package)
lock.acquire()
try:
if SHOW_TIME:
start_time = datetime.utcnow()
owner, ver = self._db.get_version(package)
if owner and owner != uid:
show_error(self, 'failed to upload, invalid owner, package=%s, version=%s' % (str(package), str(version)))
return False
if ver == version:#The version of package has been uploaded.
show_error(self, 'failed to upload, invalid version, package=%s, version=%s' % (str(package), str(version)))
return False
else:
ret = self._upload(package, version, buf)
if not ret:
show_error(self, 'failed to upload, %s cannot upload %s-%s to hdfs' % (str(uid), str(package), str(version)))
return False
self._db.set_package(uid, package, version, '')
if not ver or ver < version:
self._db.set_version(uid, package, version)
self._print('finished uploading, package=%s, version=%s' % (str(package), str(version)))
if DEBUG:
self._upload_cnt += 1
self._print('upload, count=%d' % self._upload_cnt)
if SHOW_TIME:
self._print('upload, time=%d sec' % (datetime.utcnow() - start_time).seconds)
return True
finally:
lock.release()
def download(self, package, version):
self._print('start to download, package=%s, version=%s' % (str(package), str(version)))
try:
if SHOW_TIME:
start_time = datetime.utcnow()
addr = self._get_addr(package)
uid, ver = self._db.get_version(package)
if not version:
version = ver
if not self._db.has_package(uid, package, version):
show_error(self, 'failed to download, invalid version, package=%s, version=%s' % (str(package), str(version)))
return
if DEBUG:
self._download_cnt += 1
self._print('download, count=%d' % self._download_cnt)
if SHOW_TIME:
self._print( 'download, time=%d sec' % (datetime.utcnow() - start_time).seconds)
return self._client.download(addr, self._port, package, version)
except:
show_error(self, 'failed to download')
def version(self, package):
self._print('start to get version, uid=%s, package=%s' % (str(uid), str(package)))
_, ver = self._db.get_version(package)
return ver
def start(self):
t = Thread(target=self.run)
t.start()
if not HDFS:
self._server.run()
t.join()