Example #1
0
 def __init__(self, addr, port):
     RPCServer.__init__(self, addr, port)
     len_up = len(UPLOAD_SERVERS)
     len_repo = len(REPOSITORY_SERVERS)
     if len_up < len_repo or len_up % len_repo != 0:
         show_error(self, "failed to initialize")
         raise Exception("failed to initialize")
     addr = localhost()
     if addr not in REPOSITORY_SERVERS:
         show_error(self, "failed to initialize")
         raise Exception("failed to initialize REPOSITORY_SERVERS")
     for i in range(len(REPOSITORY_SERVERS)):
         if addr == REPOSITORY_SERVERS[i]:
             break
     total = len_up / len_repo
     self._upload_servers = UPLOAD_SERVERS[i * total : (i + 1) * total]
     self._print("upload_servers=%s" % str(self._upload_servers))
     if HDFS:
         self._port = HDFS_PORT
         self._client = HDFSClient()
     else:
         self._port = FTP_PORT
         self._client = FTPClient()
         self._server = FTPServer()
     if REPO_DB:
         self._db = Database(addr=REPO_DB)
     else:
         self._db = Database(addr=addr)
     locks = []
     for _ in range(LOCK_MAX):
         locks.append(Lock())
     self._locks = HashRing(locks)
     if DEBUG:
         self._upload_cnt = 0
         self._download_cnt = 0
Example #2
0
 def prepare(self, IP):
     self.db = Database(DBPATH)
     self.ip = IP
     self.zombie = sina_zombie(IP)
     zombie_info = self.db.get_zombie(0, 1, now())[0]
     self.upass = zombie_info[2].strip("\r").strip("\n")
     self.uname = zombie_info[1]
     self.uid = zombie_info[0]
Example #3
0
class Thread_Zombie(threading.Thread):
    """docstring for ClassName"""

    def prepare(self, IP):
        self.db = Database(DBPATH)
        self.ip = IP
        self.zombie = sina_zombie(IP)
        zombie_info = self.db.get_zombie(0, 1, now())[0]
        self.upass = zombie_info[2].strip("\r").strip("\n")
        self.uname = zombie_info[1]
        self.uid = zombie_info[0]

    def run(self):
        zombie_routine(self.zombie, self.uname, self.upass, self.uid, self.db)
        print self.ip, self.uname, self.upass, self.uid
Example #4
0
class Repository(RPCServer):
    def _print(self, text):
        if PRINT:
            show_info(self, text)

    def __init__(self, addr, port):
        RPCServer.__init__(self, addr, port)
        len_up = len(UPLOAD_SERVERS)
        len_repo = len(REPOSITORY_SERVERS)
        if len_up < len_repo or len_up % len_repo != 0:
            show_error(self, "failed to initialize")
            raise Exception("failed to initialize")
        addr = localhost()
        if addr not in REPOSITORY_SERVERS:
            show_error(self, "failed to initialize")
            raise Exception("failed to initialize REPOSITORY_SERVERS")
        for i in range(len(REPOSITORY_SERVERS)):
            if addr == REPOSITORY_SERVERS[i]:
                break
        total = len_up / len_repo
        self._upload_servers = UPLOAD_SERVERS[i * total : (i + 1) * total]
        self._print("upload_servers=%s" % str(self._upload_servers))
        if HDFS:
            self._port = HDFS_PORT
            self._client = HDFSClient()
        else:
            self._port = FTP_PORT
            self._client = FTPClient()
            self._server = FTPServer()
        if REPO_DB:
            self._db = Database(addr=REPO_DB)
        else:
            self._db = Database(addr=addr)
        locks = []
        for _ in range(LOCK_MAX):
            locks.append(Lock())
        self._locks = HashRing(locks)
        if DEBUG:
            self._upload_cnt = 0
            self._download_cnt = 0

    def _get_addr(self, package):
        ring = HashRing(self._upload_servers)
        addr = ring.get_node(package)
        return addr

    def _get_lock(self, package):
        return self._locks.get_node(package)

    def _upload(self, package, version, buf):
        addr = self._get_addr(package)
        return self._client.upload(addr, self._port, package, version, buf)

    def upload(self, uid, package, version, buf):
        self._print("start to upload, uid=%s, package=%s, version=%s" % (str(uid), str(package), str(version)))
        lock = self._get_lock(package)
        lock.acquire()
        try:
            if SHOW_TIME:
                start_time = datetime.utcnow()
            owner, ver = self._db.get_version(package)
            if owner and owner != uid:
                show_error(
                    self, "failed to upload, invalid owner, package=%s, version=%s" % (str(package), str(version))
                )
                return False
            if ver == version:  # The version of package has been uploaded.
                show_error(
                    self, "failed to upload, invalid version, package=%s, version=%s" % (str(package), str(version))
                )
                return False
            else:
                ret = self._upload(package, version, buf)
                if not ret:
                    show_error(
                        self,
                        "failed to upload, %s cannot upload %s-%s to hdfs" % (str(uid), str(package), str(version)),
                    )
                    return False
                self._db.set_package(uid, package, version, "")
                if not ver or ver < version:
                    self._db.set_version(uid, package, version)
                self._print("finished uploading, package=%s, version=%s" % (str(package), str(version)))
                if DEBUG:
                    self._upload_cnt += 1
                    self._print("upload, count=%d" % self._upload_cnt)
                if SHOW_TIME:
                    self._print("upload, time=%d sec" % (datetime.utcnow() - start_time).seconds)
                return True
        finally:
            lock.release()

    def download(self, package, version):
        self._print("start to download, package=%s, version=%s" % (str(package), str(version)))
        try:
            if SHOW_TIME:
                start_time = datetime.utcnow()
            addr = self._get_addr(package)
            uid, ver = self._db.get_version(package)
            if not version:
                version = ver
            if not self._db.has_package(uid, package, version):
                show_error(
                    self, "failed to download, invalid version, package=%s, version=%s" % (str(package), str(version))
                )
                return
            if DEBUG:
                self._download_cnt += 1
                self._print("download, count=%d" % self._download_cnt)
            if SHOW_TIME:
                self._print("download, time=%d sec" % (datetime.utcnow() - start_time).seconds)
            return self._client.download(addr, self._port, package, version)
        except:
            show_error(self, "failed to download")

    def version(self, package):
        self._print("start to get version, uid=%s, package=%s" % (str(uid), str(package)))
        _, ver = self._db.get_version(package)
        return ver

    def start(self):
        t = Thread(target=self.run)
        t.start()
        if not HDFS:
            self._server.run()
        t.join()
Example #5
0
File: app.py Project: tx626/dpm
 def __init__(self):
     self._lock = Lock()
     if APP_DB:
         self._db = Database(addr=APP_DB, domain=APP)
     else:
         self._db = Database(addr=localhost(), domain=APP)
Example #6
0
File: app.py Project: tx626/dpm
class App:
    def __init__(self):
        self._lock = Lock()
        if APP_DB:
            self._db = Database(addr=APP_DB, domain=APP)
        else:
            self._db = Database(addr=localhost(), domain=APP)

    def _print(self, text):
        if PRINT:
            show_info(self, text)

    def _get_repo(self, package):
        ring = HashRing(REPOSITORY_SERVERS)
        server = ring.get_node(package)
        return server

    def _install(self, buf):
        dirname = tempfile.mkdtemp()
        try:
            src = os.path.join(dirname, "app.zip")
            with open(src, "wb") as f:
                f.write(buf)
            dest = os.path.join(dirname, "app")
            unzip_file(src, dest)
            cmd = "python %s %s" % (PATH_INSTALLER, dest)
            status, output = commands.getstatusoutput(cmd)
            if status == 0:
                return output
        finally:
            shutil.rmtree(dirname)

    def install(self, uid, package, version):
        self._lock.acquire()
        try:
            if self._db.has_package(uid, package, None):
                show_error(self, "failed to install, cannot install %s again" % package)
                return
            else:
                addr = self._get_repo(package)
                rpcclient = RPCClient(addr, REPOSITORY_PORT)
                if not version:
                    version = rpcclient.request("version", package=package)
                    if not version:
                        show_error(self, "failed to install, invalid version, uid=%s, package=%s" % (uid, package))
                        return
                ret = rpcclient.request("download", package=package, version=version)
                if ret:
                    result = self._install(ret)
                    if result:
                        self._db.set_package(uid, package, version, result)
                        self._print("finished installing %s, version=%s" % (package, version))
                        return True
            show_error(self, "failed to install %s" % package)
            return
        finally:
            self._lock.release()

    def _uninstall(self, uid, package, info):
        pass

    def uninstall(self, uid, package):
        self._lock.acquire()
        try:
            if not self._db.has_package(uid, package, None):
                show_error(self, "failed to uninstall %s " % package)
                return
            version, info = self._db.get_package(uid, package, None)
            if info:
                self._uninstall(uid, package, info)
            self._db.rm_package(uid, package, version)
            return True
        finally:
            self._lock.release()

    def get_packages(self, uid):
        return self._db.get_packages(uid)

    def has_package(self, uid, package):
        return self._db.has_package(uid, package)