def __init__(self, bokor): self.bokor = bokor self.db = self.get('memory', 'sqlitedir', '/tmp') + '/' + 'post.db' self.memory = PostMemory(self.db) self.path_session = self.get("rtorrent", "path_session").rstrip('/') + '/' self.path_dl = self.get("rtorrent", "path_dl").rstrip('/') + '/' self.path_log = self.get("rtorrent", "path_log").rstrip('/') + '/' self.base_url = self.get("post", "base_url", "") self.protocol = self.get("post", "protocol") self.post_thread = PostThread(self.db) self.post_thread.setName('post') self.post_thread.start()
def __init__(self, db): Thread.__init__(self) logging.info("init post thread") self.setDaemon(True) self.memory = PostMemory(db)
class PostThread(Thread): def __init__(self, db): Thread.__init__(self) logging.info("init post thread") self.setDaemon(True) self.memory = PostMemory(db) def run(self): while True: time.sleep(5) # get all rp to treat (status todo) to_treat = self.memory.get_relative_path("status", "todo") for rp in to_treat: rp["status"] = "treating" self.memory.update_relative_path(rp) # for all movement in rp for movement in rp["movements"]: try: self.treat_movement(rp, movement) except Exception as e: self.movement_error(movement, "unknown : %s" % (str(e))) rp["status"] = "error" if rp["status"] == "treating": rp["status"] = "done" self.memory.update_relative_path(rp) def treat_movement(self, rp, movement): if movement["status"] in ["error", "done"]: return now = int(time.time()) movement["begin"] = now movement["status"] = "treating" self.memory.update_movement(movement) # checking sources movement["src_exist"] = True if not os.path.exists(rp["src"]): now = int(time.time()) movement["src_exist"] = False movement["status"] = "error" movement["end"] = now movement["error"] = "source file : %s does not exist" % (rp["src"]) self.memory.update_movement(movement) return # adapt in function of protocol if movement["protocol"] == "cp": self.do_cp(movement, rp) return if movement["protocol"] == "ftp": self.do_ftp(movement, rp) return now = int(time.time()) movement["status"] = "error" movement["end"] = now movement["error"] = "protocol %s not known" % (movement["protocol"]) self.memory.update_movement(movement) rp["status"] = "done" self.memory.update_relative_path(rp) def do_cp(self, movement, rp): # init of exist movement["dst_exist"] = True if not os.path.isdir(os.path.dirname(movement["dst_url"])): movement["status"] = "creating" movement["dst_exist"] = False self.memory.update_movement(movement) try: os.makedirs(os.path.dirname(movement["dst_url"])) except Exception as e: movement["dst_access"] = False self.movement_error(movement, "error creating dirs %s" % (str(e))) return movement["status"] = "ready" self.memory.update_movement(movement) try: movement["status"] = "copying" self.memory.update_movement(movement) shutil.copy(rp["src"], movement["dst_url"]) except Exception as e: movement["dst_access"] = False self.movement_error(movement, "error during copy %s" % (str(e))) return now = int(time.time()) movement["status"] = "done" movement["end"] = now self.memory.update_movement(movement) def do_ftp(self, movement, rp): args = re.match("ftp://(?P<login>[^:]*):(?P<pwd>[^@]*)@(?P<host>[^/]*)(?P<path>.*)", movement["dst_url"]) if not args: self.movement_error(movement, "dst_url : %s is not a valid ftp url") return login = args.group("login") pwd = args.group("pwd") host = args.group("host") path = args.group("path") movement["status"] = "connecting" self.memory.update_movement(movement) # try to connect try: ftp = FTP(host, login, pwd, timeout=60) except Exception as e: movement["dst_access"] = False self.movement_error(movement, "error while connecting ftp : %s" % (str(e))) return movement["status"] = "connected" self.memory.update_movement(movement) # try to go to directory try: self.ftp_chdir(os.path.dirname(path), ftp, movement) except Exception as e: movement["dst_access"] = False self.movement_error(movement, "error while browsing ftp : %s" % (str(e))) return # try to send try: with open(rp["src"], "rb") as f: movement["status"] = "sending" self.memory.update_movement(movement) ftp.storbinary("STOR " + os.path.basename(rp["src"]), f) logging.debug("push FTP done") except Exception as e: self.movement_error(movement, "error while sending to ftp : %s" % (str(e))) return now = int(time.time()) movement["status"] = "done" movement["end"] = now self.memory.update_movement(movement) def movement_error(self, movement, error): logging.error(error) now = int(time.time()) movement["status"] = "error" movement["error"] = error movement["end"] = now self.memory.update_movement(movement) def ftp_chdir(self, dir, ftp, movement): """Change ftp directory, create if necessary @type dir: str @param dir: path of the directory @type ftp: ftplib.FTP object @param ftp: FTP object already initialized @return: none """ logging.info("ask to push in %s" % dir) dirs = dir.split("/") for f in dirs: if not f: continue logging.info("looking for sub path %s" % f) if not self.ftp_directory_exists(f, ftp): logging.info("creating sub path %s" % f) movement["status"] = "creating" movement["exist"] = False self.memory.update_movement(movement) ftp.mkd(f) logging.info("moving to sub path %s" % f) ftp.cwd(f) movement["status"] = "ready" self.memory.update_movement(movement) def ftp_directory_exists(self, dir, ftp): """ Check if a directory exists in ftp @type dir: str @param dir: path of directory @type ftp: ftplib.FTP object @param ftp: FTP object already initialized @return: True if directory exists, False otherwise """ filelist = [] ftp.retrlines("LIST", filelist.append) for f in filelist: if f.split()[-1].strip("/") == str(dir).strip("/") and f.upper().startswith("D"): return True return False
This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. @file bokor_master_rtorrent.py @author Olivier ANDRE <*****@*****.**> @date 2014 @brief update relative path in order to allow post treatement """ import sys from bokor.memory.postdlmemory import PostMemory if __name__ == "__main__": hash_file = sys.argv[1] src = sys.argv[3] + '/' + sys.argv[2] post = PostMemory(sys.argv[4]) rps = post.get_relative_path('info_hash', hash_file, '=', False) for rp in rps: rp['status'] = 'todo' rp['src'] = src post.update_relative_path(rp)
You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. @file bokor_master_rtorrent.py @author Olivier ANDRE <*****@*****.**> @date 2014 @brief update relative path in order to allow post treatement """ import sys from bokor.memory.postdlmemory import PostMemory if __name__ == "__main__" : hash_file = sys.argv[1] src = sys.argv[3] + '/' + sys.argv[2] post = PostMemory(sys.argv[4]) rps = post.get_relative_path('info_hash', hash_file, '=', False) for rp in rps : rp['status'] = 'todo' rp['src'] = src post.update_relative_path(rp)
class SlavePost(Configurable): needed_configuration = { "rtorrent": { "path_session": { 'mandatory': True, 'type': 'file', 'permission': 'drwx', 'exist': True }, "path_dl": { 'mandatory': True, 'type': 'file', 'permission': 'drwx', 'exist': True }, "path_log": { 'mandatory': True, 'type': 'file', 'permission': 'drwx', 'exist': True }, }, "post": { "protocol": { 'mandatory': True, 'type': 'string' }, "base_url": { 'mandatory': False, 'type': 'string' }, }, "memory": { "sqlitedir": { 'mandatory': False, 'type': 'file', 'permission': 'drwx', 'exist': True }, }, } def __init__(self, bokor): self.bokor = bokor self.db = self.get('memory', 'sqlitedir', '/tmp') + '/' + 'post.db' self.memory = PostMemory(self.db) self.path_session = self.get("rtorrent", "path_session").rstrip('/') + '/' self.path_dl = self.get("rtorrent", "path_dl").rstrip('/') + '/' self.path_log = self.get("rtorrent", "path_log").rstrip('/') + '/' self.base_url = self.get("post", "base_url", "") self.protocol = self.get("post", "protocol") self.post_thread = PostThread(self.db) self.post_thread.setName('post') self.post_thread.start() @aFeature def get_dst_url(self, src, relative_path): """ compute destination url from a src and a relative path """ dst = "" if self.protocol == "ftp": dst = self.base_url else: if self.base_url: dst = self.base_url else: dst = self.path_dl dst += '/' + relative_path.rstrip('/') + '/' + os.path.basename(src) return dst @aFeature def create_relative_path(self, infohash, relative_path, src, status): """ add a new relative path for an asset and add the mouvement """ rp = { 'info_hash': infohash, 'relative_path': relative_path, 'src': os.path.expanduser(src), 'status': status } rp['id'] = self.memory.create_relative_path(rp) dst = self.get_dst_url(src, relative_path) if dst == src: return rp self.add_mv(rp, dst, status) return rp @aFeature def create_relative_path_if_needed(self, info_hash, relative_path, src, status): rp = self.memory.get_a_relative_path(info_hash, relative_path) if not rp: rp = self.create_relative_path(info_hash, relative_path, src, status) #create a rp and a movement return rp for movement in rp['movements']: if movement['status'] in ['downloading', 'todo']: return rp dst_url = self.get_dst_url(src, relative_path) self.add_mv(rp, dst_url, status) return rp @aFeature def add_movement(self, idrp, protocol, dst_url): rp = self.get_rp('id', idrp) if not rp: self._bokor_code = 701 return "no idrp valid" rp = rp[0] rp['status'] = 'todo' self.memory.update_relative_path(rp) return self.add_mv(rp, dst_url, 'todo') def add_mv(self, rp, dst_url, status): movement = { "id": None, "idrp": rp['id'], "protocol": self.protocol, "begin": None, "status": status, "src_exist": None, "dst_url": dst_url, "dst_exist": None, "dst_access": None, "dst_creation": None, "send_start": None, "end": None, "error": None, } return self.memory.create_movement(movement) @aFeature def get_relative_path(self, hash_file): return self.memory.get_relative_path('info_hash', hash_file) @aFeature def treat_rp(self, idrp): rp = self.get_rp('id', idrp) if not rp: self._bokor_code = 701 return "no idrp valid" rp[0]['status'] = 'todo' self.memory.update_relative_path(rp[0]) return idrp @aFeature def get_rp(self, filters=None, value=None, compare="="): return self.memory.get_relative_path(filters, value, compare) @aFeature def get_mv(self, filters=None, value=None, compare="="): return self.memory.get_movements(filters, value, compare) @aFeature def delete_rp(self, filters=None, value=None, compare="="): return self.memory.delete_relative_path(filters, value, compare) @aFeature def delete_relative_path(self, idrp): return self.delete_rp('id', idrp) @aFeature def delete_mouvement(self, idmv): return self.delete_mv('id', idmv) @aFeature def delete_mv(self, filters, value, compare="="): return self.memory.delete_movement(filters, value, compare) def ftp_directory_exists(self, dir, ftp): """ Check if a directory exists in ftp @type dir: str @param dir: path of directory @type ftp: ftplib.FTP object @param ftp: FTP object already initialized @return: True if directory exists, False otherwise """ filelist = [] ftp.retrlines('LIST', filelist.append) for f in filelist: if f.split()[-1].strip('/') == dir.strip( '/') and f.upper().startswith('D'): return True return False @aFeature def ls_ftp(self, host=None, login=None, pwd=None, path=None): if not host: args = re.match( 'ftp://(?P<login>[^:]*):(?P<pwd>[^@]*)@(?P<host>[^/]*)(?P<path>.*)', self.base_url) if not args: return "no default values in conf : %s" % self.base_url login = args.group('login') pwd = args.group('pwd') host = args.group('host') if (path == None): path = args.group('path') ftp = FTP(host, login, pwd, timeout=60) files = [] ftp.cwd(path) ftp.retrlines('LIST', files.append) return files @aFeature def delete_ftp(self, host, login, pwd, path): ftp = FTP(host, login, pwd, timeout=60) return ftp.delete(path) @aFeature def status_post_thread(self): return self.post_thread.isAlive() @aFeature def start_post_thread(self): if not self.post_thread.isAlive(): self.post_thread.start()
class PostThread(Thread): def __init__(self, db): Thread.__init__(self) logging.info("init post thread") self.setDaemon(True) self.memory = PostMemory(db) def run(self): while True: time.sleep(5) # get all rp to treat (status todo) to_treat = self.memory.get_relative_path('status', 'todo') for rp in to_treat: rp['status'] = 'treating' self.memory.update_relative_path(rp) # for all movement in rp for movement in rp['movements']: try: self.treat_movement(rp, movement) except Exception as e: self.movement_error(movement, "unknown : %s" % (str(e))) rp['status'] = 'error' if rp['status'] == 'treating': rp['status'] = 'done' self.memory.update_relative_path(rp) def treat_movement(self, rp, movement): if movement['status'] in ['error', 'done']: return now = int(time.time()) movement['begin'] = now movement['status'] = 'treating' self.memory.update_movement(movement) # checking sources movement['src_exist'] = True if not os.path.exists(rp['src']): now = int(time.time()) movement['src_exist'] = False movement['status'] = 'error' movement['end'] = now movement['error'] = "source file : %s does not exist" % (rp['src']) self.memory.update_movement(movement) return # adapt in function of protocol if movement['protocol'] == 'cp': self.do_cp(movement, rp) return if movement['protocol'] == 'ftp': self.do_ftp(movement, rp) return now = int(time.time()) movement['status'] = 'error' movement['end'] = now movement['error'] = "protocol %s not known" % (movement['protocol']) self.memory.update_movement(movement) rp['status'] = 'done' self.memory.update_relative_path(rp) def do_cp(self, movement, rp): #init of exist movement['dst_exist'] = True if not os.path.isdir(os.path.dirname(movement['dst_url'])): movement['status'] = 'creating' movement['dst_exist'] = False self.memory.update_movement(movement) try: os.makedirs(os.path.dirname(movement['dst_url'])) except Exception as e: movement['dst_access'] = False self.movement_error(movement, "error creating dirs %s" % (str(e))) return movement['status'] = 'ready' self.memory.update_movement(movement) try: movement['status'] = 'copying' self.memory.update_movement(movement) shutil.copy(rp['src'], movement['dst_url']) except Exception as e: movement['dst_access'] = False self.movement_error(movement, "error during copy %s" % (str(e))) return now = int(time.time()) movement['status'] = 'done' movement['end'] = now self.memory.update_movement(movement) def do_ftp(self, movement, rp): args = re.match( 'ftp://(?P<login>[^:]*):(?P<pwd>[^@]*)@(?P<host>[^/]*)(?P<path>.*)', movement['dst_url']) if not args: self.movement_error(movement, "dst_url : %s is not a valid ftp url") return login = args.group('login') pwd = args.group('pwd') host = args.group('host') path = args.group('path') movement['status'] = 'connecting' self.memory.update_movement(movement) # try to connect try: ftp = FTP(host, login, pwd, timeout=60) except Exception as e: movement['dst_access'] = False self.movement_error(movement, "error while connecting ftp : %s" % (str(e))) return movement['status'] = 'connected' self.memory.update_movement(movement) # try to go to directory try: self.ftp_chdir(os.path.dirname(path), ftp, movement) except Exception as e: movement['dst_access'] = False self.movement_error(movement, "error while browsing ftp : %s" % (str(e))) return # try to send try: with open(rp['src'], 'rb') as f: movement['status'] = 'sending' self.memory.update_movement(movement) ftp.storbinary('STOR ' + os.path.basename(rp['src']), f) logging.debug("push FTP done") except Exception as e: self.movement_error(movement, "error while sending to ftp : %s" % (str(e))) return now = int(time.time()) movement['status'] = 'done' movement['end'] = now self.memory.update_movement(movement) def movement_error(self, movement, error): logging.error(error) now = int(time.time()) movement['status'] = 'error' movement['error'] = error movement['end'] = now self.memory.update_movement(movement) def ftp_chdir(self, dir, ftp, movement): """Change ftp directory, create if necessary @type dir: str @param dir: path of the directory @type ftp: ftplib.FTP object @param ftp: FTP object already initialized @return: none """ logging.info("ask to push in %s" % dir) dirs = dir.split("/") for f in dirs: if not f: continue logging.info("looking for sub path %s" % f) if not self.ftp_directory_exists(f, ftp): logging.info("creating sub path %s" % f) movement['status'] = 'creating' movement['exist'] = False self.memory.update_movement(movement) ftp.mkd(f) logging.info("moving to sub path %s" % f) ftp.cwd(f) movement['status'] = 'ready' self.memory.update_movement(movement) def ftp_directory_exists(self, dir, ftp): """ Check if a directory exists in ftp @type dir: str @param dir: path of directory @type ftp: ftplib.FTP object @param ftp: FTP object already initialized @return: True if directory exists, False otherwise """ filelist = [] ftp.retrlines('LIST', filelist.append) for f in filelist: if f.split()[-1].strip('/') == str(dir).strip( '/') and f.upper().startswith('D'): return True return False