def __init__(self, db, mktorrent, token, tmp, path_session, path_dl, notify, ftp_server, ftp_user, ftp_password): Thread.__init__(self) self.setDaemon(True) self.memory = MkTorrentMemory(db) self.mktorrent = mktorrent self.token = token self.tmp = tmp self.path_session = path_session self.path_dl = path_dl self.notify = notify self.ftp_server = ftp_server self.ftp_user = ftp_user self.ftp_password = ftp_password
def __init__(self, bokor): self.bokor = bokor self.cinegoserv = self.get("slave", "master") self.path_dl = self.get("rtorrent", "path_dl").rstrip('/') + '/' self.path_session = self.get("rtorrent", "path_session").rstrip('/') + '/' self.path_log = self.get("rtorrent", "path_log").rstrip('/') + '/' self.tmp = '/tmp' #FIXME, in threading structure self.mktorrent = self.get("upload", "mktorrent") self.ftp_server = self.get("upload", "server") self.ftp_user = self.get("upload", "user") self.ftp_password = self.get("upload", "password") self.notify = self.get("upload", "notify_url") self.db = self.get('memory', 'sqlitedir', '/tmp') + '/' + 'mktorrent.db' self.memory = MkTorrentMemory(self.db) self.prepare_thread = PrepareTorrentThread( self.db, self.mktorrent, self.bokor.token, self.tmp, self.path_session, self.path_dl, self.notify, self.ftp_server, self.ftp_user, self.ftp_password) self.prepare_thread.setName('prepare') self.prepare_thread.start()
def __init__(self, bokor): self.bokor = bokor self.cinegoserv = self.get("slave", "master") self.path_dl = self.get("rtorrent", "path_dl").rstrip('/') + '/' self.path_session = self.get("rtorrent", "path_session").rstrip('/') + '/' self.path_log = self.get("rtorrent", "path_log").rstrip('/') + '/' self.tmp = '/tmp' #FIXME, in threading structure self.mktorrent = self.get("upload", "mktorrent") self.ftp_server = self.get("upload", "server") self.ftp_user = self.get("upload", "user") self.ftp_password = self.get("upload", "password") self.notify = self.get("upload", "notify_url") self.db = self.get('memory', 'sqlitedir', '/tmp') + '/' + 'mktorrent.db' self.memory = MkTorrentMemory(self.db) self.prepare_thread = PrepareTorrentThread(self.db, self.mktorrent, self.bokor.token, self.tmp, self.path_session, self.path_dl, self.notify, self.ftp_server, self.ftp_user, self.ftp_password) self.prepare_thread.setName('prepare') self.prepare_thread.start()
class SlaveMKTorrent(Configurable): dependances = { "configuration": [], "interface": [UnsafeSocketClient], "transmission": [], "feature": [] } needed_configuration = { "upload": { "mktorrent": { 'mandatory': True, 'type': 'file', 'permission': 'rx', 'exist': True }, "user": { 'mandatory': True, 'type': 'string' }, "server": { 'mandatory': True, 'type': 'host' }, "prepare": { 'mandatory': True, 'type': 'file', 'permission': 'r', 'exist': True }, "password": { 'mandatory': True, 'type': 'string' }, "notify_url": { 'mandatory': True, 'type': 'string' }, }, "memory": { "sqlitedir": { 'mandatory': False, 'type': 'file', 'permission': 'drwx', 'exist': True }, }, } def __init__(self, bokor): self.bokor = bokor self.cinegoserv = self.get("slave", "master") self.path_dl = self.get("rtorrent", "path_dl").rstrip('/') + '/' self.path_session = self.get("rtorrent", "path_session").rstrip('/') + '/' self.path_log = self.get("rtorrent", "path_log").rstrip('/') + '/' self.tmp = '/tmp' #FIXME, in threading structure self.mktorrent = self.get("upload", "mktorrent") self.ftp_server = self.get("upload", "server") self.ftp_user = self.get("upload", "user") self.ftp_password = self.get("upload", "password") self.notify = self.get("upload", "notify_url") self.db = self.get('memory', 'sqlitedir', '/tmp') + '/' + 'mktorrent.db' self.memory = MkTorrentMemory(self.db) self.prepare_thread = PrepareTorrentThread( self.db, self.mktorrent, self.bokor.token, self.tmp, self.path_session, self.path_dl, self.notify, self.ftp_server, self.ftp_user, self.ftp_password) self.prepare_thread.setName('prepare') self.prepare_thread.start() @aFeature def get_assets(self, field=None, value=None): return self.memory.get_assets(field, value) @aFeature def add_file(self, iddl, token_file, path_file, tracker_url): res = None if os.path.isdir(path_file): idassets = self.prepareTREE(iddl, tracker_url, token_file, path_file) else: idassets = [ self.prepare_file(iddl, tracker_url, token_file, path_file, None, 1) ] # os.remove('/tmp/' + token_site + '_' + token) if not idassets: self._bokor_code = PREPARE_ERROR return idassets def prepare_file(self, iddl, tracker_url, token_file, path_file, relative_path=None, nb_assets=0): """Prepare the upload of a single file @type tracker_url: str @param tracker_url: url for opentracker @type token_file: str @param token_file: token of the file @type path_file: str @param path_file: path of the TREE @type asset: str @param asset: hash of the TREE file torrent, "none" for a real single file @type relative_path: str @param relative_path: final path to put the downloaded file @type assets: str @param assets: number of assets linked to this file ('0' by default) @return: returns the token of the uploaded file """ apath = os.path.expanduser(path_file) asset = self.memory.get_assets('file', apath) new_asset = { 'id': None, 'iddl': iddl, 'info_hash': None, 'relative_path': relative_path.strip('/') if relative_path else None, 'file': apath, 'link': None, 'tmp_torrent': None, 'torrent_file': None, 'ftp_url': None, 'size_torrent': None, 'tracker': tracker_url, 'begin': int(time.time()), 'status': 'new', 'file_exist': os.path.exists(apath), 'file_access': os.path.isfile(apath), 'mktorrent_start': None, 'send_start': None, 'end': None, 'error': None } if not new_asset['file_exist']: #new_asset['status'] = 'error' new_asset['error'] = "file %s doesn't exist" % apath elif not new_asset['file_access']: #new_asset['status'] = 'error' new_asset['error'] = "can't read file %s" % apath if asset: asset = asset[0] #new_asset['status'] = 'known' new_asset['info_hash'] = asset['info_hash'] new_asset['torrent_file'] = asset['torrent_file'] new_asset['ftp_url'] = asset['ftp_url'] new_asset['size_torrent'] = asset['size_torrent'] new_asset['error'] = "file already treated, see %s" % asset['id'] if new_asset['status'] != 'new': new_asset['end'] = int(time.time()) new_asset['id'] = self.memory.create_asset(new_asset) print new_asset['id'], new_asset['error'] return new_asset['id'] def prepareTREE(self, iddl, tracker_url, token_file, path): """Prepare a TREE file: create one torrent for TREE_xxx.xml itself and one for each asset @type tracker_url: str @param tracker_url: url for opentracker @type token_file: str @param token_file: token of the file @type path: str @param path: path of the TREE @return: returns the token of the uploaded file """ logging.info("prepare TREE %s %s %s %s %s" % (iddl, tracker_url, self.bokor.token, token_file, path)) tree = FileTree(path) afiles = tree.assets ref_path = os.path.dirname(path) assets = [] for relative_path, afile in afiles: path_to_file = ref_path + "/" + "/".join( relative_path.split("/")[1:]) + "/" + afile assets.append( self.prepare_file(iddl, tracker_url, token_file, path_to_file, relative_path, len(afiles))) return assets @aFeature def status_prepare_thread(self): return self.prepare_thread.isAlive() @aFeature def start_prepare_thread(self): if not self.prepare_thread.isAlive(): self.prepare_thread.start() @aFeature def delete_assets(self, filters, value, compare="="): return self.memory.delete_assets(filters, value, compare) @aFeature def delete_asset(self, idasset): return self.delete_assets('id', idasset)
class PrepareTorrentThread(Thread): def __init__(self, db, mktorrent, token, tmp, path_session, path_dl, notify, ftp_server, ftp_user, ftp_password): Thread.__init__(self) self.setDaemon(True) self.memory = MkTorrentMemory(db) self.mktorrent = mktorrent self.token = token self.tmp = tmp self.path_session = path_session self.path_dl = path_dl self.notify = notify self.ftp_server = ftp_server self.ftp_user = ftp_user self.ftp_password = ftp_password def run(self): while True : time.sleep(5) try : to_treat = self.memory.get_assets('status', 'new') for asset in to_treat : if asset['error'] : if asset['info_hash'] : asset['status'] = 'known' else : asset['status'] = 'error' elif not self.run_mktorrent(asset) or \ not self.link_file(asset) or \ not self.mv_torrent(asset) or \ not self.send_torrent(asset) : if asset['status'] != "known" : asset['status'] = 'error' else : asset['status'] = 'done' if not asset['relative_path'] else "unvalidated" opts = dict( seeder = self.token, infohash = asset['info_hash'], iddl = asset['iddl'], relative_path = asset['relative_path'], file_name = os.path.basename(asset['file']), status = asset['status'], size = asset['size_torrent']) self.notify_master(asset, opts) now = int(time.time()) asset['end'] = now self.memory.update_asset(asset) except Exception as err: logging.error("exception %s"%(err)) def run_mktorrent(self, asset): now = int(time.time()) tmp_p_torrent = self.tmp + "/" + os.path.basename(asset['file']) + "_" + str(now) + ".torrent_t" asset['tmp_torrent'] = tmp_p_torrent asset['mktorrent_start'] = now asset['status'] = 'hashing' self.memory.update_asset(asset) command = '%(mktorrent)s -a "http://%(tracker_address)s/announce?token=%(token)s/" -o "%(ptorrent)s" "%(tfile)s"'\ % {'mktorrent': self.mktorrent, 'tracker_address': asset['tracker'], 'ptorrent': asset['tmp_torrent'], 'token': self.token, 'tfile': asset['file'], } p = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) (output, err) = p.communicate() if err : asset['error'] = "mktorrent :" + err asset['status'] = 'error' self.memory.update_asset(asset) return False asset['info_hash'] = getHash(asset['tmp_torrent']) asset['size_torrent'] = os.path.getsize(asset['file']) asset['status'] = 'hashed' self.memory.update_asset(asset) return True def link_file(self, asset): if asset['relative_path']: asset['link'] = self.path_dl + '/' + asset['relative_path'] + '/' + os.path.basename(asset['file']) else: asset['link'] = self.path_dl + '/' + os.path.basename(asset['file']) if not os.path.isdir(os.path.dirname(asset['link'])): os.makedirs(os.path.dirname(asset['link'])) if os.path.exists(asset['link']) : asset['error'] = "Warn : link already exists : " + asset['link'] force_symlink(asset['file'], asset['link']) asset['status'] = 'linked' self.memory.update_asset(asset) return True def mv_torrent(self, asset): if asset['relative_path']: asset['torrent_file'] = self.path_session + '/' + asset['info_hash'] + '.torrent_upload' else: asset['torrent_file'] = self.path_session + '/' + asset['info_hash'] + '.torrent_upload_direct' if os.path.exists( asset['torrent_file']) : asset['status'] = 'known' asset['error'] = "torrent_file already exists : " + asset['torrent_file'] self.memory.update_asset(asset) return False shutil.copy(asset['tmp_torrent'], asset['torrent_file']) asset['status'] = 'local' self.memory.update_asset(asset) return True def send_torrent(self, asset) : now = int(time.time()) asset['send_start'] = now asset['ftp_url'] = 'ftp://' + self.ftp_user + ':' + self.ftp_password + '@' + self.ftp_server + '/' + asset['info_hash'] + ".torrent" asset['status'] = 'sending' self.memory.update_asset(asset) try : ftp = FTP(self.ftp_server, self.ftp_user, self.ftp_password) msg = ftp.storbinary('STOR ' + asset['info_hash'] + ".torrent", open(asset['tmp_torrent'], 'rb')) except Exception as e: msg = str(e) if not msg.startswith('226-') : asset['error'] = "ftp :" + msg return False asset['status'] = 'send' self.memory.update_asset(asset) return True def notify_master(self, asset, opts) : p = subprocess.Popen('curl -L -k -s -o /dev/null "%s?%s"' % (self.notify, urllib.urlencode(opts)), stdout=subprocess.PIPE, shell=True) (output, err) = p.communicate() if err : asset['error'] = "notify :" + err asset['status'] = 'error' self.memory.update_asset(asset)
class PrepareTorrentThread(Thread): def __init__(self, db, mktorrent, token, tmp, path_session, path_dl, notify, ftp_server, ftp_user, ftp_password): Thread.__init__(self) self.setDaemon(True) self.memory = MkTorrentMemory(db) self.mktorrent = mktorrent self.token = token self.tmp = tmp self.path_session = path_session self.path_dl = path_dl self.notify = notify self.ftp_server = ftp_server self.ftp_user = ftp_user self.ftp_password = ftp_password def run(self): while True: time.sleep(5) try: to_treat = self.memory.get_assets('status', 'new') for asset in to_treat: if asset['error']: if asset['info_hash']: asset['status'] = 'known' else: asset['status'] = 'error' elif not self.run_mktorrent(asset) or \ not self.link_file(asset) or \ not self.mv_torrent(asset) or \ not self.send_torrent(asset) : if asset['status'] != "known": asset['status'] = 'error' else: asset['status'] = 'done' if not asset[ 'relative_path'] else "unvalidated" opts = dict(seeder=self.token, infohash=asset['info_hash'], iddl=asset['iddl'], relative_path=asset['relative_path'], file_name=os.path.basename(asset['file']), status=asset['status'], size=asset['size_torrent']) self.notify_master(asset, opts) now = int(time.time()) asset['end'] = now self.memory.update_asset(asset) except Exception as err: logging.error("exception %s" % (err)) def run_mktorrent(self, asset): now = int(time.time()) tmp_p_torrent = self.tmp + "/" + os.path.basename( asset['file']) + "_" + str(now) + ".torrent_t" asset['tmp_torrent'] = tmp_p_torrent asset['mktorrent_start'] = now asset['status'] = 'hashing' self.memory.update_asset(asset) command = '%(mktorrent)s -a "http://%(tracker_address)s/announce?token=%(token)s/" -o "%(ptorrent)s" "%(tfile)s"'\ % {'mktorrent': self.mktorrent, 'tracker_address': asset['tracker'], 'ptorrent': asset['tmp_torrent'], 'token': self.token, 'tfile': asset['file'], } p = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) (output, err) = p.communicate() if err: asset['error'] = "mktorrent :" + err asset['status'] = 'error' self.memory.update_asset(asset) return False asset['info_hash'] = getHash(asset['tmp_torrent']) asset['size_torrent'] = os.path.getsize(asset['file']) asset['status'] = 'hashed' self.memory.update_asset(asset) return True def link_file(self, asset): if asset['relative_path']: asset['link'] = self.path_dl + '/' + asset[ 'relative_path'] + '/' + os.path.basename(asset['file']) else: asset['link'] = self.path_dl + '/' + os.path.basename( asset['file']) if not os.path.isdir(os.path.dirname(asset['link'])): os.makedirs(os.path.dirname(asset['link'])) if os.path.exists(asset['link']): asset['error'] = "Warn : link already exists : " + asset['link'] force_symlink(asset['file'], asset['link']) asset['status'] = 'linked' self.memory.update_asset(asset) return True def mv_torrent(self, asset): if asset['relative_path']: asset['torrent_file'] = self.path_session + '/' + asset[ 'info_hash'] + '.torrent_upload' else: asset['torrent_file'] = self.path_session + '/' + asset[ 'info_hash'] + '.torrent_upload_direct' if os.path.exists(asset['torrent_file']): asset['status'] = 'known' asset['error'] = "torrent_file already exists : " + asset[ 'torrent_file'] self.memory.update_asset(asset) return False shutil.copy(asset['tmp_torrent'], asset['torrent_file']) asset['status'] = 'local' self.memory.update_asset(asset) return True def send_torrent(self, asset): now = int(time.time()) asset['send_start'] = now asset[ 'ftp_url'] = 'ftp://' + self.ftp_user + ':' + self.ftp_password + '@' + self.ftp_server + '/' + asset[ 'info_hash'] + ".torrent" asset['status'] = 'sending' self.memory.update_asset(asset) try: ftp = FTP(self.ftp_server, self.ftp_user, self.ftp_password) msg = ftp.storbinary('STOR ' + asset['info_hash'] + ".torrent", open(asset['tmp_torrent'], 'rb')) except Exception as e: msg = str(e) if not msg.startswith('226-'): asset['error'] = "ftp :" + msg return False asset['status'] = 'send' self.memory.update_asset(asset) return True def notify_master(self, asset, opts): p = subprocess.Popen('curl -L -k -s -o /dev/null "%s?%s"' % (self.notify, urllib.urlencode(opts)), stdout=subprocess.PIPE, shell=True) (output, err) = p.communicate() if err: asset['error'] = "notify :" + err asset['status'] = 'error' self.memory.update_asset(asset)
class SlaveMKTorrent(Configurable): dependances = { "configuration" : [] , "interface" : [UnsafeSocketClient], "transmission" : [], "feature" : []} needed_configuration = {"upload": { "mktorrent": {'mandatory' : True, 'type' : 'file' , 'permission' : 'rx', 'exist' : True}, "user": {'mandatory' : True, 'type' : 'string'}, "server": {'mandatory' : True, 'type' : 'host'}, "prepare": {'mandatory' : True, 'type' : 'file' , 'permission' : 'r', 'exist' : True}, "password": {'mandatory' : True, 'type' : 'string'}, "notify_url": {'mandatory' : True, 'type' : 'string'}, }, "memory" : { "sqlitedir": {'mandatory' : False, 'type' : 'file' , 'permission' : 'drwx', 'exist' : True}, }, } def __init__(self, bokor): self.bokor = bokor self.cinegoserv = self.get("slave", "master") self.path_dl = self.get("rtorrent", "path_dl").rstrip('/') + '/' self.path_session = self.get("rtorrent", "path_session").rstrip('/') + '/' self.path_log = self.get("rtorrent", "path_log").rstrip('/') + '/' self.tmp = '/tmp' #FIXME, in threading structure self.mktorrent = self.get("upload", "mktorrent") self.ftp_server = self.get("upload", "server") self.ftp_user = self.get("upload", "user") self.ftp_password = self.get("upload", "password") self.notify = self.get("upload", "notify_url") self.db = self.get('memory', 'sqlitedir', '/tmp') + '/' + 'mktorrent.db' self.memory = MkTorrentMemory(self.db) self.prepare_thread = PrepareTorrentThread(self.db, self.mktorrent, self.bokor.token, self.tmp, self.path_session, self.path_dl, self.notify, self.ftp_server, self.ftp_user, self.ftp_password) self.prepare_thread.setName('prepare') self.prepare_thread.start() @aFeature def get_assets(self, field = None, value = None): return self.memory.get_assets(field, value) @aFeature def add_file(self, iddl, token_file, path_file, tracker_url): res = None if os.path.isdir(path_file) : idassets = self.prepareTREE(iddl, tracker_url, token_file, path_file) else: idassets = [self.prepare_file(iddl, tracker_url, token_file, path_file, None, 1)] # os.remove('/tmp/' + token_site + '_' + token) if not idassets : self._bokor_code = PREPARE_ERROR return idassets def prepare_file(self, iddl, tracker_url, token_file, path_file, relative_path=None, nb_assets=0): """Prepare the upload of a single file @type tracker_url: str @param tracker_url: url for opentracker @type token_file: str @param token_file: token of the file @type path_file: str @param path_file: path of the TREE @type asset: str @param asset: hash of the TREE file torrent, "none" for a real single file @type relative_path: str @param relative_path: final path to put the downloaded file @type assets: str @param assets: number of assets linked to this file ('0' by default) @return: returns the token of the uploaded file """ apath = os.path.expanduser(path_file) asset = self.memory.get_assets('file', apath) new_asset = { 'id' : None, 'iddl' : iddl, 'info_hash' : None, 'relative_path' : relative_path.strip('/') if relative_path else None, 'file' : apath, 'link' : None, 'tmp_torrent' : None, 'torrent_file' : None, 'ftp_url' : None, 'size_torrent' : None, 'tracker' : tracker_url, 'begin' : int(time.time()), 'status' : 'new', 'file_exist' : os.path.exists(apath), 'file_access' : os.path.isfile(apath), 'mktorrent_start' : None, 'send_start' : None, 'end' : None, 'error' : None } if not new_asset['file_exist'] : #new_asset['status'] = 'error' new_asset['error'] = "file %s doesn't exist"%apath elif not new_asset['file_access'] : #new_asset['status'] = 'error' new_asset['error'] = "can't read file %s"%apath if asset : asset = asset[0] #new_asset['status'] = 'known' new_asset['info_hash'] = asset['info_hash'] new_asset['torrent_file'] = asset['torrent_file'] new_asset['ftp_url'] = asset['ftp_url'] new_asset['size_torrent'] = asset['size_torrent'] new_asset['error'] = "file already treated, see %s"%asset['id'] if new_asset['status'] != 'new' : new_asset['end'] = int(time.time()) new_asset['id'] = self.memory.create_asset(new_asset) print new_asset['id'], new_asset['error'] return new_asset['id'] def prepareTREE(self, iddl, tracker_url, token_file, path): """Prepare a TREE file: create one torrent for TREE_xxx.xml itself and one for each asset @type tracker_url: str @param tracker_url: url for opentracker @type token_file: str @param token_file: token of the file @type path: str @param path: path of the TREE @return: returns the token of the uploaded file """ logging.info("prepare TREE %s %s %s %s %s" % (iddl, tracker_url, self.bokor.token, token_file, path)) tree = FileTree(path) afiles = tree.assets ref_path = os.path.dirname(path) assets = [] for relative_path, afile in afiles: path_to_file = ref_path + "/" + "/".join(relative_path.split("/")[1:]) + "/" + afile assets.append(self.prepare_file(iddl, tracker_url, token_file, path_to_file, relative_path, len(afiles))) return assets @aFeature def status_prepare_thread(self) : return self.prepare_thread.isAlive() @aFeature def start_prepare_thread(self) : if not self.prepare_thread.isAlive() : self.prepare_thread.start() @aFeature def delete_assets(self, filters, value, compare = "="): return self.memory.delete_assets(filters, value, compare) @aFeature def delete_asset(self, idasset): return self.delete_assets('id', idasset)