def __init__(self, bokor):
self.bokor = bokor
self.db = self.get('memory', 'sqlitedir', '/tmp') + '/' + 'post.db'
self.memory = PostMemory(self.db)
self.path_session = self.get("rtorrent",
"path_session").rstrip('/') + '/'
self.path_dl = self.get("rtorrent", "path_dl").rstrip('/') + '/'
self.path_log = self.get("rtorrent", "path_log").rstrip('/') + '/'
self.base_url = self.get("post", "base_url", "")
self.protocol = self.get("post", "protocol")
self.post_thread = PostThread(self.db)
self.post_thread.setName('post')
self.post_thread.start()
def __init__(self, db):
Thread.__init__(self)
logging.info("init post thread")
self.setDaemon(True)
self.memory = PostMemory(db)
class PostThread(Thread):
def __init__(self, db):
Thread.__init__(self)
logging.info("init post thread")
self.setDaemon(True)
self.memory = PostMemory(db)
def run(self):
while True:
time.sleep(5)
# get all rp to treat (status todo)
to_treat = self.memory.get_relative_path("status", "todo")
for rp in to_treat:
rp["status"] = "treating"
self.memory.update_relative_path(rp)
# for all movement in rp
for movement in rp["movements"]:
try:
self.treat_movement(rp, movement)
except Exception as e:
self.movement_error(movement, "unknown : %s" % (str(e)))
rp["status"] = "error"
if rp["status"] == "treating":
rp["status"] = "done"
self.memory.update_relative_path(rp)
def treat_movement(self, rp, movement):
if movement["status"] in ["error", "done"]:
return
now = int(time.time())
movement["begin"] = now
movement["status"] = "treating"
self.memory.update_movement(movement)
# checking sources
movement["src_exist"] = True
if not os.path.exists(rp["src"]):
now = int(time.time())
movement["src_exist"] = False
movement["status"] = "error"
movement["end"] = now
movement["error"] = "source file : %s does not exist" % (rp["src"])
self.memory.update_movement(movement)
return
# adapt in function of protocol
if movement["protocol"] == "cp":
self.do_cp(movement, rp)
return
if movement["protocol"] == "ftp":
self.do_ftp(movement, rp)
return
now = int(time.time())
movement["status"] = "error"
movement["end"] = now
movement["error"] = "protocol %s not known" % (movement["protocol"])
self.memory.update_movement(movement)
rp["status"] = "done"
self.memory.update_relative_path(rp)
def do_cp(self, movement, rp):
# init of exist
movement["dst_exist"] = True
if not os.path.isdir(os.path.dirname(movement["dst_url"])):
movement["status"] = "creating"
movement["dst_exist"] = False
self.memory.update_movement(movement)
try:
os.makedirs(os.path.dirname(movement["dst_url"]))
except Exception as e:
movement["dst_access"] = False
self.movement_error(movement, "error creating dirs %s" % (str(e)))
return
movement["status"] = "ready"
self.memory.update_movement(movement)
try:
movement["status"] = "copying"
self.memory.update_movement(movement)
shutil.copy(rp["src"], movement["dst_url"])
except Exception as e:
movement["dst_access"] = False
self.movement_error(movement, "error during copy %s" % (str(e)))
return
now = int(time.time())
movement["status"] = "done"
movement["end"] = now
self.memory.update_movement(movement)
def do_ftp(self, movement, rp):
args = re.match("ftp://(?P<login>[^:]*):(?P<pwd>[^@]*)@(?P<host>[^/]*)(?P<path>.*)", movement["dst_url"])
if not args:
self.movement_error(movement, "dst_url : %s is not a valid ftp url")
return
login = args.group("login")
pwd = args.group("pwd")
host = args.group("host")
path = args.group("path")
movement["status"] = "connecting"
self.memory.update_movement(movement)
# try to connect
try:
ftp = FTP(host, login, pwd, timeout=60)
except Exception as e:
movement["dst_access"] = False
self.movement_error(movement, "error while connecting ftp : %s" % (str(e)))
return
movement["status"] = "connected"
self.memory.update_movement(movement)
# try to go to directory
try:
self.ftp_chdir(os.path.dirname(path), ftp, movement)
except Exception as e:
movement["dst_access"] = False
self.movement_error(movement, "error while browsing ftp : %s" % (str(e)))
return
# try to send
try:
with open(rp["src"], "rb") as f:
movement["status"] = "sending"
self.memory.update_movement(movement)
ftp.storbinary("STOR " + os.path.basename(rp["src"]), f)
logging.debug("push FTP done")
except Exception as e:
self.movement_error(movement, "error while sending to ftp : %s" % (str(e)))
return
now = int(time.time())
movement["status"] = "done"
movement["end"] = now
self.memory.update_movement(movement)
def movement_error(self, movement, error):
logging.error(error)
now = int(time.time())
movement["status"] = "error"
movement["error"] = error
movement["end"] = now
self.memory.update_movement(movement)
def ftp_chdir(self, dir, ftp, movement):
"""Change ftp directory, create if necessary
@type dir: str
@param dir: path of the directory
@type ftp: ftplib.FTP object
@param ftp: FTP object already initialized
@return: none
"""
logging.info("ask to push in %s" % dir)
dirs = dir.split("/")
for f in dirs:
if not f:
continue
logging.info("looking for sub path %s" % f)
if not self.ftp_directory_exists(f, ftp):
logging.info("creating sub path %s" % f)
movement["status"] = "creating"
movement["exist"] = False
self.memory.update_movement(movement)
ftp.mkd(f)
logging.info("moving to sub path %s" % f)
ftp.cwd(f)
movement["status"] = "ready"
self.memory.update_movement(movement)
def ftp_directory_exists(self, dir, ftp):
""" Check if a directory exists in ftp
@type dir: str
@param dir: path of directory
@type ftp: ftplib.FTP object
@param ftp: FTP object already initialized
@return: True if directory exists, False otherwise
"""
filelist = []
ftp.retrlines("LIST", filelist.append)
for f in filelist:
if f.split()[-1].strip("/") == str(dir).strip("/") and f.upper().startswith("D"):
return True
return False
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software Foundation,
Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
@file bokor_master_rtorrent.py
@author Olivier ANDRE <*****@*****.**>
@date 2014
@brief update relative path in order to allow post treatement
"""
import sys
from bokor.memory.postdlmemory import PostMemory
if __name__ == "__main__":
hash_file = sys.argv[1]
src = sys.argv[3] + '/' + sys.argv[2]
post = PostMemory(sys.argv[4])
rps = post.get_relative_path('info_hash', hash_file, '=', False)
for rp in rps:
rp['status'] = 'todo'
rp['src'] = src
post.update_relative_path(rp)
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software Foundation,
Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
@file bokor_master_rtorrent.py
@author Olivier ANDRE <*****@*****.**>
@date 2014
@brief update relative path in order to allow post treatement
"""
import sys
from bokor.memory.postdlmemory import PostMemory
if __name__ == "__main__" :
hash_file = sys.argv[1]
src = sys.argv[3] + '/' + sys.argv[2]
post = PostMemory(sys.argv[4])
rps = post.get_relative_path('info_hash', hash_file, '=', False)
for rp in rps :
rp['status'] = 'todo'
rp['src'] = src
post.update_relative_path(rp)
class SlavePost(Configurable):
needed_configuration = {
"rtorrent": {
"path_session": {
'mandatory': True,
'type': 'file',
'permission': 'drwx',
'exist': True
},
"path_dl": {
'mandatory': True,
'type': 'file',
'permission': 'drwx',
'exist': True
},
"path_log": {
'mandatory': True,
'type': 'file',
'permission': 'drwx',
'exist': True
},
},
"post": {
"protocol": {
'mandatory': True,
'type': 'string'
},
"base_url": {
'mandatory': False,
'type': 'string'
},
},
"memory": {
"sqlitedir": {
'mandatory': False,
'type': 'file',
'permission': 'drwx',
'exist': True
},
},
}
def __init__(self, bokor):
self.bokor = bokor
self.db = self.get('memory', 'sqlitedir', '/tmp') + '/' + 'post.db'
self.memory = PostMemory(self.db)
self.path_session = self.get("rtorrent",
"path_session").rstrip('/') + '/'
self.path_dl = self.get("rtorrent", "path_dl").rstrip('/') + '/'
self.path_log = self.get("rtorrent", "path_log").rstrip('/') + '/'
self.base_url = self.get("post", "base_url", "")
self.protocol = self.get("post", "protocol")
self.post_thread = PostThread(self.db)
self.post_thread.setName('post')
self.post_thread.start()
@aFeature
def get_dst_url(self, src, relative_path):
""" compute destination url from a src and a relative path
"""
dst = ""
if self.protocol == "ftp":
dst = self.base_url
else:
if self.base_url:
dst = self.base_url
else:
dst = self.path_dl
dst += '/' + relative_path.rstrip('/') + '/' + os.path.basename(src)
return dst
@aFeature
def create_relative_path(self, infohash, relative_path, src, status):
"""
add a new relative path for an asset and add the mouvement
"""
rp = {
'info_hash': infohash,
'relative_path': relative_path,
'src': os.path.expanduser(src),
'status': status
}
rp['id'] = self.memory.create_relative_path(rp)
dst = self.get_dst_url(src, relative_path)
if dst == src:
return rp
self.add_mv(rp, dst, status)
return rp
@aFeature
def create_relative_path_if_needed(self, info_hash, relative_path, src,
status):
rp = self.memory.get_a_relative_path(info_hash, relative_path)
if not rp:
rp = self.create_relative_path(info_hash, relative_path, src,
status)
#create a rp and a movement
return rp
for movement in rp['movements']:
if movement['status'] in ['downloading', 'todo']:
return rp
dst_url = self.get_dst_url(src, relative_path)
self.add_mv(rp, dst_url, status)
return rp
@aFeature
def add_movement(self, idrp, protocol, dst_url):
rp = self.get_rp('id', idrp)
if not rp:
self._bokor_code = 701
return "no idrp valid"
rp = rp[0]
rp['status'] = 'todo'
self.memory.update_relative_path(rp)
return self.add_mv(rp, dst_url, 'todo')
def add_mv(self, rp, dst_url, status):
movement = {
"id": None,
"idrp": rp['id'],
"protocol": self.protocol,
"begin": None,
"status": status,
"src_exist": None,
"dst_url": dst_url,
"dst_exist": None,
"dst_access": None,
"dst_creation": None,
"send_start": None,
"end": None,
"error": None,
}
return self.memory.create_movement(movement)
@aFeature
def get_relative_path(self, hash_file):
return self.memory.get_relative_path('info_hash', hash_file)
@aFeature
def treat_rp(self, idrp):
rp = self.get_rp('id', idrp)
if not rp:
self._bokor_code = 701
return "no idrp valid"
rp[0]['status'] = 'todo'
self.memory.update_relative_path(rp[0])
return idrp
@aFeature
def get_rp(self, filters=None, value=None, compare="="):
return self.memory.get_relative_path(filters, value, compare)
@aFeature
def get_mv(self, filters=None, value=None, compare="="):
return self.memory.get_movements(filters, value, compare)
@aFeature
def delete_rp(self, filters=None, value=None, compare="="):
return self.memory.delete_relative_path(filters, value, compare)
@aFeature
def delete_relative_path(self, idrp):
return self.delete_rp('id', idrp)
@aFeature
def delete_mouvement(self, idmv):
return self.delete_mv('id', idmv)
@aFeature
def delete_mv(self, filters, value, compare="="):
return self.memory.delete_movement(filters, value, compare)
def ftp_directory_exists(self, dir, ftp):
""" Check if a directory exists in ftp
@type dir: str
@param dir: path of directory
@type ftp: ftplib.FTP object
@param ftp: FTP object already initialized
@return: True if directory exists, False otherwise
"""
filelist = []
ftp.retrlines('LIST', filelist.append)
for f in filelist:
if f.split()[-1].strip('/') == dir.strip(
'/') and f.upper().startswith('D'):
return True
return False
@aFeature
def ls_ftp(self, host=None, login=None, pwd=None, path=None):
if not host:
args = re.match(
'ftp://(?P<login>[^:]*):(?P<pwd>[^@]*)@(?P<host>[^/]*)(?P<path>.*)',
self.base_url)
if not args:
return "no default values in conf : %s" % self.base_url
login = args.group('login')
pwd = args.group('pwd')
host = args.group('host')
if (path == None): path = args.group('path')
ftp = FTP(host, login, pwd, timeout=60)
files = []
ftp.cwd(path)
ftp.retrlines('LIST', files.append)
return files
@aFeature
def delete_ftp(self, host, login, pwd, path):
ftp = FTP(host, login, pwd, timeout=60)
return ftp.delete(path)
@aFeature
def status_post_thread(self):
return self.post_thread.isAlive()
@aFeature
def start_post_thread(self):
if not self.post_thread.isAlive():
self.post_thread.start()
def __init__(self, db):
Thread.__init__(self)
logging.info("init post thread")
self.setDaemon(True)
self.memory = PostMemory(db)
class PostThread(Thread):
def __init__(self, db):
Thread.__init__(self)
logging.info("init post thread")
self.setDaemon(True)
self.memory = PostMemory(db)
def run(self):
while True:
time.sleep(5)
# get all rp to treat (status todo)
to_treat = self.memory.get_relative_path('status', 'todo')
for rp in to_treat:
rp['status'] = 'treating'
self.memory.update_relative_path(rp)
# for all movement in rp
for movement in rp['movements']:
try:
self.treat_movement(rp, movement)
except Exception as e:
self.movement_error(movement,
"unknown : %s" % (str(e)))
rp['status'] = 'error'
if rp['status'] == 'treating':
rp['status'] = 'done'
self.memory.update_relative_path(rp)
def treat_movement(self, rp, movement):
if movement['status'] in ['error', 'done']:
return
now = int(time.time())
movement['begin'] = now
movement['status'] = 'treating'
self.memory.update_movement(movement)
# checking sources
movement['src_exist'] = True
if not os.path.exists(rp['src']):
now = int(time.time())
movement['src_exist'] = False
movement['status'] = 'error'
movement['end'] = now
movement['error'] = "source file : %s does not exist" % (rp['src'])
self.memory.update_movement(movement)
return
# adapt in function of protocol
if movement['protocol'] == 'cp':
self.do_cp(movement, rp)
return
if movement['protocol'] == 'ftp':
self.do_ftp(movement, rp)
return
now = int(time.time())
movement['status'] = 'error'
movement['end'] = now
movement['error'] = "protocol %s not known" % (movement['protocol'])
self.memory.update_movement(movement)
rp['status'] = 'done'
self.memory.update_relative_path(rp)
def do_cp(self, movement, rp):
#init of exist
movement['dst_exist'] = True
if not os.path.isdir(os.path.dirname(movement['dst_url'])):
movement['status'] = 'creating'
movement['dst_exist'] = False
self.memory.update_movement(movement)
try:
os.makedirs(os.path.dirname(movement['dst_url']))
except Exception as e:
movement['dst_access'] = False
self.movement_error(movement,
"error creating dirs %s" % (str(e)))
return
movement['status'] = 'ready'
self.memory.update_movement(movement)
try:
movement['status'] = 'copying'
self.memory.update_movement(movement)
shutil.copy(rp['src'], movement['dst_url'])
except Exception as e:
movement['dst_access'] = False
self.movement_error(movement, "error during copy %s" % (str(e)))
return
now = int(time.time())
movement['status'] = 'done'
movement['end'] = now
self.memory.update_movement(movement)
def do_ftp(self, movement, rp):
args = re.match(
'ftp://(?P<login>[^:]*):(?P<pwd>[^@]*)@(?P<host>[^/]*)(?P<path>.*)',
movement['dst_url'])
if not args:
self.movement_error(movement,
"dst_url : %s is not a valid ftp url")
return
login = args.group('login')
pwd = args.group('pwd')
host = args.group('host')
path = args.group('path')
movement['status'] = 'connecting'
self.memory.update_movement(movement)
# try to connect
try:
ftp = FTP(host, login, pwd, timeout=60)
except Exception as e:
movement['dst_access'] = False
self.movement_error(movement,
"error while connecting ftp : %s" % (str(e)))
return
movement['status'] = 'connected'
self.memory.update_movement(movement)
# try to go to directory
try:
self.ftp_chdir(os.path.dirname(path), ftp, movement)
except Exception as e:
movement['dst_access'] = False
self.movement_error(movement,
"error while browsing ftp : %s" % (str(e)))
return
# try to send
try:
with open(rp['src'], 'rb') as f:
movement['status'] = 'sending'
self.memory.update_movement(movement)
ftp.storbinary('STOR ' + os.path.basename(rp['src']), f)
logging.debug("push FTP done")
except Exception as e:
self.movement_error(movement,
"error while sending to ftp : %s" % (str(e)))
return
now = int(time.time())
movement['status'] = 'done'
movement['end'] = now
self.memory.update_movement(movement)
def movement_error(self, movement, error):
logging.error(error)
now = int(time.time())
movement['status'] = 'error'
movement['error'] = error
movement['end'] = now
self.memory.update_movement(movement)
def ftp_chdir(self, dir, ftp, movement):
"""Change ftp directory, create if necessary
@type dir: str
@param dir: path of the directory
@type ftp: ftplib.FTP object
@param ftp: FTP object already initialized
@return: none
"""
logging.info("ask to push in %s" % dir)
dirs = dir.split("/")
for f in dirs:
if not f: continue
logging.info("looking for sub path %s" % f)
if not self.ftp_directory_exists(f, ftp):
logging.info("creating sub path %s" % f)
movement['status'] = 'creating'
movement['exist'] = False
self.memory.update_movement(movement)
ftp.mkd(f)
logging.info("moving to sub path %s" % f)
ftp.cwd(f)
movement['status'] = 'ready'
self.memory.update_movement(movement)
def ftp_directory_exists(self, dir, ftp):
""" Check if a directory exists in ftp
@type dir: str
@param dir: path of directory
@type ftp: ftplib.FTP object
@param ftp: FTP object already initialized
@return: True if directory exists, False otherwise
"""
filelist = []
ftp.retrlines('LIST', filelist.append)
for f in filelist:
if f.split()[-1].strip('/') == str(dir).strip(
'/') and f.upper().startswith('D'):
return True
return False
