def __init__(self, db, mktorrent, token, tmp, path_session, path_dl,
notify, ftp_server, ftp_user, ftp_password):
Thread.__init__(self)
self.setDaemon(True)
self.memory = MkTorrentMemory(db)
self.mktorrent = mktorrent
self.token = token
self.tmp = tmp
self.path_session = path_session
self.path_dl = path_dl
self.notify = notify
self.ftp_server = ftp_server
self.ftp_user = ftp_user
self.ftp_password = ftp_password
def __init__(self, bokor):
self.bokor = bokor
self.cinegoserv = self.get("slave", "master")
self.path_dl = self.get("rtorrent", "path_dl").rstrip('/') + '/'
self.path_session = self.get("rtorrent",
"path_session").rstrip('/') + '/'
self.path_log = self.get("rtorrent", "path_log").rstrip('/') + '/'
self.tmp = '/tmp' #FIXME, in threading structure
self.mktorrent = self.get("upload", "mktorrent")
self.ftp_server = self.get("upload", "server")
self.ftp_user = self.get("upload", "user")
self.ftp_password = self.get("upload", "password")
self.notify = self.get("upload", "notify_url")
self.db = self.get('memory', 'sqlitedir',
'/tmp') + '/' + 'mktorrent.db'
self.memory = MkTorrentMemory(self.db)
self.prepare_thread = PrepareTorrentThread(
self.db, self.mktorrent, self.bokor.token, self.tmp,
self.path_session, self.path_dl, self.notify, self.ftp_server,
self.ftp_user, self.ftp_password)
self.prepare_thread.setName('prepare')
self.prepare_thread.start()
def __init__(self, db, mktorrent, token, tmp,
path_session, path_dl, notify,
ftp_server, ftp_user, ftp_password):
Thread.__init__(self)
self.setDaemon(True)
self.memory = MkTorrentMemory(db)
self.mktorrent = mktorrent
self.token = token
self.tmp = tmp
self.path_session = path_session
self.path_dl = path_dl
self.notify = notify
self.ftp_server = ftp_server
self.ftp_user = ftp_user
self.ftp_password = ftp_password
def __init__(self, bokor):
self.bokor = bokor
self.cinegoserv = self.get("slave", "master")
self.path_dl = self.get("rtorrent", "path_dl").rstrip('/') + '/'
self.path_session = self.get("rtorrent", "path_session").rstrip('/') + '/'
self.path_log = self.get("rtorrent", "path_log").rstrip('/') + '/'
self.tmp = '/tmp' #FIXME, in threading structure
self.mktorrent = self.get("upload", "mktorrent")
self.ftp_server = self.get("upload", "server")
self.ftp_user = self.get("upload", "user")
self.ftp_password = self.get("upload", "password")
self.notify = self.get("upload", "notify_url")
self.db = self.get('memory', 'sqlitedir', '/tmp') + '/' + 'mktorrent.db'
self.memory = MkTorrentMemory(self.db)
self.prepare_thread = PrepareTorrentThread(self.db, self.mktorrent, self.bokor.token, self.tmp,
self.path_session, self.path_dl, self.notify,
self.ftp_server, self.ftp_user, self.ftp_password)
self.prepare_thread.setName('prepare')
self.prepare_thread.start()
class SlaveMKTorrent(Configurable):
dependances = {
"configuration": [],
"interface": [UnsafeSocketClient],
"transmission": [],
"feature": []
}
needed_configuration = {
"upload": {
"mktorrent": {
'mandatory': True,
'type': 'file',
'permission': 'rx',
'exist': True
},
"user": {
'mandatory': True,
'type': 'string'
},
"server": {
'mandatory': True,
'type': 'host'
},
"prepare": {
'mandatory': True,
'type': 'file',
'permission': 'r',
'exist': True
},
"password": {
'mandatory': True,
'type': 'string'
},
"notify_url": {
'mandatory': True,
'type': 'string'
},
},
"memory": {
"sqlitedir": {
'mandatory': False,
'type': 'file',
'permission': 'drwx',
'exist': True
},
},
}
def __init__(self, bokor):
self.bokor = bokor
self.cinegoserv = self.get("slave", "master")
self.path_dl = self.get("rtorrent", "path_dl").rstrip('/') + '/'
self.path_session = self.get("rtorrent",
"path_session").rstrip('/') + '/'
self.path_log = self.get("rtorrent", "path_log").rstrip('/') + '/'
self.tmp = '/tmp' #FIXME, in threading structure
self.mktorrent = self.get("upload", "mktorrent")
self.ftp_server = self.get("upload", "server")
self.ftp_user = self.get("upload", "user")
self.ftp_password = self.get("upload", "password")
self.notify = self.get("upload", "notify_url")
self.db = self.get('memory', 'sqlitedir',
'/tmp') + '/' + 'mktorrent.db'
self.memory = MkTorrentMemory(self.db)
self.prepare_thread = PrepareTorrentThread(
self.db, self.mktorrent, self.bokor.token, self.tmp,
self.path_session, self.path_dl, self.notify, self.ftp_server,
self.ftp_user, self.ftp_password)
self.prepare_thread.setName('prepare')
self.prepare_thread.start()
@aFeature
def get_assets(self, field=None, value=None):
return self.memory.get_assets(field, value)
@aFeature
def add_file(self, iddl, token_file, path_file, tracker_url):
res = None
if os.path.isdir(path_file):
idassets = self.prepareTREE(iddl, tracker_url, token_file,
path_file)
else:
idassets = [
self.prepare_file(iddl, tracker_url, token_file, path_file,
None, 1)
]
# os.remove('/tmp/' + token_site + '_' + token)
if not idassets:
self._bokor_code = PREPARE_ERROR
return idassets
def prepare_file(self,
iddl,
tracker_url,
token_file,
path_file,
relative_path=None,
nb_assets=0):
"""Prepare the upload of a single file
@type tracker_url: str
@param tracker_url: url for opentracker
@type token_file: str
@param token_file: token of the file
@type path_file: str
@param path_file: path of the TREE
@type asset: str
@param asset: hash of the TREE file torrent, "none" for a real single file
@type relative_path: str
@param relative_path: final path to put the downloaded file
@type assets: str
@param assets: number of assets linked to this file ('0' by default)
@return: returns the token of the uploaded file
"""
apath = os.path.expanduser(path_file)
asset = self.memory.get_assets('file', apath)
new_asset = {
'id': None,
'iddl': iddl,
'info_hash': None,
'relative_path':
relative_path.strip('/') if relative_path else None,
'file': apath,
'link': None,
'tmp_torrent': None,
'torrent_file': None,
'ftp_url': None,
'size_torrent': None,
'tracker': tracker_url,
'begin': int(time.time()),
'status': 'new',
'file_exist': os.path.exists(apath),
'file_access': os.path.isfile(apath),
'mktorrent_start': None,
'send_start': None,
'end': None,
'error': None
}
if not new_asset['file_exist']:
#new_asset['status'] = 'error'
new_asset['error'] = "file %s doesn't exist" % apath
elif not new_asset['file_access']:
#new_asset['status'] = 'error'
new_asset['error'] = "can't read file %s" % apath
if asset:
asset = asset[0]
#new_asset['status'] = 'known'
new_asset['info_hash'] = asset['info_hash']
new_asset['torrent_file'] = asset['torrent_file']
new_asset['ftp_url'] = asset['ftp_url']
new_asset['size_torrent'] = asset['size_torrent']
new_asset['error'] = "file already treated, see %s" % asset['id']
if new_asset['status'] != 'new':
new_asset['end'] = int(time.time())
new_asset['id'] = self.memory.create_asset(new_asset)
print new_asset['id'], new_asset['error']
return new_asset['id']
def prepareTREE(self, iddl, tracker_url, token_file, path):
"""Prepare a TREE file: create one torrent for TREE_xxx.xml itself and one for each asset
@type tracker_url: str
@param tracker_url: url for opentracker
@type token_file: str
@param token_file: token of the file
@type path: str
@param path: path of the TREE
@return: returns the token of the uploaded file
"""
logging.info("prepare TREE %s %s %s %s %s" %
(iddl, tracker_url, self.bokor.token, token_file, path))
tree = FileTree(path)
afiles = tree.assets
ref_path = os.path.dirname(path)
assets = []
for relative_path, afile in afiles:
path_to_file = ref_path + "/" + "/".join(
relative_path.split("/")[1:]) + "/" + afile
assets.append(
self.prepare_file(iddl, tracker_url, token_file, path_to_file,
relative_path, len(afiles)))
return assets
@aFeature
def status_prepare_thread(self):
return self.prepare_thread.isAlive()
@aFeature
def start_prepare_thread(self):
if not self.prepare_thread.isAlive():
self.prepare_thread.start()
@aFeature
def delete_assets(self, filters, value, compare="="):
return self.memory.delete_assets(filters, value, compare)
@aFeature
def delete_asset(self, idasset):
return self.delete_assets('id', idasset)
class PrepareTorrentThread(Thread):
def __init__(self, db, mktorrent, token, tmp,
path_session, path_dl, notify,
ftp_server, ftp_user, ftp_password):
Thread.__init__(self)
self.setDaemon(True)
self.memory = MkTorrentMemory(db)
self.mktorrent = mktorrent
self.token = token
self.tmp = tmp
self.path_session = path_session
self.path_dl = path_dl
self.notify = notify
self.ftp_server = ftp_server
self.ftp_user = ftp_user
self.ftp_password = ftp_password
def run(self):
while True :
time.sleep(5)
try :
to_treat = self.memory.get_assets('status', 'new')
for asset in to_treat :
if asset['error'] :
if asset['info_hash'] :
asset['status'] = 'known'
else :
asset['status'] = 'error'
elif not self.run_mktorrent(asset) or \
not self.link_file(asset) or \
not self.mv_torrent(asset) or \
not self.send_torrent(asset) :
if asset['status'] != "known" :
asset['status'] = 'error'
else :
asset['status'] = 'done' if not asset['relative_path'] else "unvalidated"
opts = dict(
seeder = self.token,
infohash = asset['info_hash'],
iddl = asset['iddl'],
relative_path = asset['relative_path'],
file_name = os.path.basename(asset['file']),
status = asset['status'],
size = asset['size_torrent'])
self.notify_master(asset, opts)
now = int(time.time())
asset['end'] = now
self.memory.update_asset(asset)
except Exception as err:
logging.error("exception %s"%(err))
def run_mktorrent(self, asset):
now = int(time.time())
tmp_p_torrent = self.tmp + "/" + os.path.basename(asset['file']) + "_" + str(now) + ".torrent_t"
asset['tmp_torrent'] = tmp_p_torrent
asset['mktorrent_start'] = now
asset['status'] = 'hashing'
self.memory.update_asset(asset)
command = '%(mktorrent)s -a "http://%(tracker_address)s/announce?token=%(token)s/" -o "%(ptorrent)s" "%(tfile)s"'\
% {'mktorrent': self.mktorrent,
'tracker_address': asset['tracker'],
'ptorrent': asset['tmp_torrent'],
'token': self.token,
'tfile': asset['file'],
}
p = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True)
(output, err) = p.communicate()
if err :
asset['error'] = "mktorrent :" + err
asset['status'] = 'error'
self.memory.update_asset(asset)
return False
asset['info_hash'] = getHash(asset['tmp_torrent'])
asset['size_torrent'] = os.path.getsize(asset['file'])
asset['status'] = 'hashed'
self.memory.update_asset(asset)
return True
def link_file(self, asset):
if asset['relative_path']:
asset['link'] = self.path_dl + '/' + asset['relative_path'] + '/' + os.path.basename(asset['file'])
else:
asset['link'] = self.path_dl + '/' + os.path.basename(asset['file'])
if not os.path.isdir(os.path.dirname(asset['link'])):
os.makedirs(os.path.dirname(asset['link']))
if os.path.exists(asset['link']) :
asset['error'] = "Warn : link already exists : " + asset['link']
force_symlink(asset['file'], asset['link'])
asset['status'] = 'linked'
self.memory.update_asset(asset)
return True
def mv_torrent(self, asset):
if asset['relative_path']:
asset['torrent_file'] = self.path_session + '/' + asset['info_hash'] + '.torrent_upload'
else:
asset['torrent_file'] = self.path_session + '/' + asset['info_hash'] + '.torrent_upload_direct'
if os.path.exists( asset['torrent_file']) :
asset['status'] = 'known'
asset['error'] = "torrent_file already exists : " + asset['torrent_file']
self.memory.update_asset(asset)
return False
shutil.copy(asset['tmp_torrent'], asset['torrent_file'])
asset['status'] = 'local'
self.memory.update_asset(asset)
return True
def send_torrent(self, asset) :
now = int(time.time())
asset['send_start'] = now
asset['ftp_url'] = 'ftp://' + self.ftp_user + ':' + self.ftp_password + '@' + self.ftp_server + '/' + asset['info_hash'] + ".torrent"
asset['status'] = 'sending'
self.memory.update_asset(asset)
try :
ftp = FTP(self.ftp_server, self.ftp_user, self.ftp_password)
msg = ftp.storbinary('STOR ' + asset['info_hash'] + ".torrent", open(asset['tmp_torrent'], 'rb'))
except Exception as e:
msg = str(e)
if not msg.startswith('226-') :
asset['error'] = "ftp :" + msg
return False
asset['status'] = 'send'
self.memory.update_asset(asset)
return True
def notify_master(self, asset, opts) :
p = subprocess.Popen('curl -L -k -s -o /dev/null "%s?%s"' % (self.notify, urllib.urlencode(opts)),
stdout=subprocess.PIPE, shell=True)
(output, err) = p.communicate()
if err :
asset['error'] = "notify :" + err
asset['status'] = 'error'
self.memory.update_asset(asset)
class PrepareTorrentThread(Thread):
def __init__(self, db, mktorrent, token, tmp, path_session, path_dl,
notify, ftp_server, ftp_user, ftp_password):
Thread.__init__(self)
self.setDaemon(True)
self.memory = MkTorrentMemory(db)
self.mktorrent = mktorrent
self.token = token
self.tmp = tmp
self.path_session = path_session
self.path_dl = path_dl
self.notify = notify
self.ftp_server = ftp_server
self.ftp_user = ftp_user
self.ftp_password = ftp_password
def run(self):
while True:
time.sleep(5)
try:
to_treat = self.memory.get_assets('status', 'new')
for asset in to_treat:
if asset['error']:
if asset['info_hash']:
asset['status'] = 'known'
else:
asset['status'] = 'error'
elif not self.run_mktorrent(asset) or \
not self.link_file(asset) or \
not self.mv_torrent(asset) or \
not self.send_torrent(asset) :
if asset['status'] != "known":
asset['status'] = 'error'
else:
asset['status'] = 'done' if not asset[
'relative_path'] else "unvalidated"
opts = dict(seeder=self.token,
infohash=asset['info_hash'],
iddl=asset['iddl'],
relative_path=asset['relative_path'],
file_name=os.path.basename(asset['file']),
status=asset['status'],
size=asset['size_torrent'])
self.notify_master(asset, opts)
now = int(time.time())
asset['end'] = now
self.memory.update_asset(asset)
except Exception as err:
logging.error("exception %s" % (err))
def run_mktorrent(self, asset):
now = int(time.time())
tmp_p_torrent = self.tmp + "/" + os.path.basename(
asset['file']) + "_" + str(now) + ".torrent_t"
asset['tmp_torrent'] = tmp_p_torrent
asset['mktorrent_start'] = now
asset['status'] = 'hashing'
self.memory.update_asset(asset)
command = '%(mktorrent)s -a "http://%(tracker_address)s/announce?token=%(token)s/" -o "%(ptorrent)s" "%(tfile)s"'\
% {'mktorrent': self.mktorrent,
'tracker_address': asset['tracker'],
'ptorrent': asset['tmp_torrent'],
'token': self.token,
'tfile': asset['file'],
}
p = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True)
(output, err) = p.communicate()
if err:
asset['error'] = "mktorrent :" + err
asset['status'] = 'error'
self.memory.update_asset(asset)
return False
asset['info_hash'] = getHash(asset['tmp_torrent'])
asset['size_torrent'] = os.path.getsize(asset['file'])
asset['status'] = 'hashed'
self.memory.update_asset(asset)
return True
def link_file(self, asset):
if asset['relative_path']:
asset['link'] = self.path_dl + '/' + asset[
'relative_path'] + '/' + os.path.basename(asset['file'])
else:
asset['link'] = self.path_dl + '/' + os.path.basename(
asset['file'])
if not os.path.isdir(os.path.dirname(asset['link'])):
os.makedirs(os.path.dirname(asset['link']))
if os.path.exists(asset['link']):
asset['error'] = "Warn : link already exists : " + asset['link']
force_symlink(asset['file'], asset['link'])
asset['status'] = 'linked'
self.memory.update_asset(asset)
return True
def mv_torrent(self, asset):
if asset['relative_path']:
asset['torrent_file'] = self.path_session + '/' + asset[
'info_hash'] + '.torrent_upload'
else:
asset['torrent_file'] = self.path_session + '/' + asset[
'info_hash'] + '.torrent_upload_direct'
if os.path.exists(asset['torrent_file']):
asset['status'] = 'known'
asset['error'] = "torrent_file already exists : " + asset[
'torrent_file']
self.memory.update_asset(asset)
return False
shutil.copy(asset['tmp_torrent'], asset['torrent_file'])
asset['status'] = 'local'
self.memory.update_asset(asset)
return True
def send_torrent(self, asset):
now = int(time.time())
asset['send_start'] = now
asset[
'ftp_url'] = 'ftp://' + self.ftp_user + ':' + self.ftp_password + '@' + self.ftp_server + '/' + asset[
'info_hash'] + ".torrent"
asset['status'] = 'sending'
self.memory.update_asset(asset)
try:
ftp = FTP(self.ftp_server, self.ftp_user, self.ftp_password)
msg = ftp.storbinary('STOR ' + asset['info_hash'] + ".torrent",
open(asset['tmp_torrent'], 'rb'))
except Exception as e:
msg = str(e)
if not msg.startswith('226-'):
asset['error'] = "ftp :" + msg
return False
asset['status'] = 'send'
self.memory.update_asset(asset)
return True
def notify_master(self, asset, opts):
p = subprocess.Popen('curl -L -k -s -o /dev/null "%s?%s"' %
(self.notify, urllib.urlencode(opts)),
stdout=subprocess.PIPE,
shell=True)
(output, err) = p.communicate()
if err:
asset['error'] = "notify :" + err
asset['status'] = 'error'
self.memory.update_asset(asset)
class SlaveMKTorrent(Configurable):
dependances = { "configuration" : [] ,
"interface" : [UnsafeSocketClient],
"transmission" : [],
"feature" : []}
needed_configuration = {"upload":
{
"mktorrent": {'mandatory' : True, 'type' : 'file' , 'permission' : 'rx', 'exist' : True},
"user": {'mandatory' : True, 'type' : 'string'},
"server": {'mandatory' : True, 'type' : 'host'},
"prepare": {'mandatory' : True, 'type' : 'file' , 'permission' : 'r', 'exist' : True},
"password": {'mandatory' : True, 'type' : 'string'},
"notify_url": {'mandatory' : True, 'type' : 'string'},
},
"memory" :
{
"sqlitedir": {'mandatory' : False, 'type' : 'file' , 'permission' : 'drwx', 'exist' : True},
},
}
def __init__(self, bokor):
self.bokor = bokor
self.cinegoserv = self.get("slave", "master")
self.path_dl = self.get("rtorrent", "path_dl").rstrip('/') + '/'
self.path_session = self.get("rtorrent", "path_session").rstrip('/') + '/'
self.path_log = self.get("rtorrent", "path_log").rstrip('/') + '/'
self.tmp = '/tmp' #FIXME, in threading structure
self.mktorrent = self.get("upload", "mktorrent")
self.ftp_server = self.get("upload", "server")
self.ftp_user = self.get("upload", "user")
self.ftp_password = self.get("upload", "password")
self.notify = self.get("upload", "notify_url")
self.db = self.get('memory', 'sqlitedir', '/tmp') + '/' + 'mktorrent.db'
self.memory = MkTorrentMemory(self.db)
self.prepare_thread = PrepareTorrentThread(self.db, self.mktorrent, self.bokor.token, self.tmp,
self.path_session, self.path_dl, self.notify,
self.ftp_server, self.ftp_user, self.ftp_password)
self.prepare_thread.setName('prepare')
self.prepare_thread.start()
@aFeature
def get_assets(self, field = None, value = None):
return self.memory.get_assets(field, value)
@aFeature
def add_file(self, iddl, token_file, path_file, tracker_url):
res = None
if os.path.isdir(path_file) :
idassets = self.prepareTREE(iddl, tracker_url, token_file, path_file)
else:
idassets = [self.prepare_file(iddl, tracker_url, token_file, path_file, None, 1)]
# os.remove('/tmp/' + token_site + '_' + token)
if not idassets :
self._bokor_code = PREPARE_ERROR
return idassets
def prepare_file(self, iddl, tracker_url, token_file, path_file,
relative_path=None, nb_assets=0):
"""Prepare the upload of a single file
@type tracker_url: str
@param tracker_url: url for opentracker
@type token_file: str
@param token_file: token of the file
@type path_file: str
@param path_file: path of the TREE
@type asset: str
@param asset: hash of the TREE file torrent, "none" for a real single file
@type relative_path: str
@param relative_path: final path to put the downloaded file
@type assets: str
@param assets: number of assets linked to this file ('0' by default)
@return: returns the token of the uploaded file
"""
apath = os.path.expanduser(path_file)
asset = self.memory.get_assets('file', apath)
new_asset = { 'id' : None,
'iddl' : iddl,
'info_hash' : None,
'relative_path' : relative_path.strip('/') if relative_path else None,
'file' : apath,
'link' : None,
'tmp_torrent' : None,
'torrent_file' : None,
'ftp_url' : None,
'size_torrent' : None,
'tracker' : tracker_url,
'begin' : int(time.time()),
'status' : 'new',
'file_exist' : os.path.exists(apath),
'file_access' : os.path.isfile(apath),
'mktorrent_start' : None,
'send_start' : None,
'end' : None,
'error' : None }
if not new_asset['file_exist'] :
#new_asset['status'] = 'error'
new_asset['error'] = "file %s doesn't exist"%apath
elif not new_asset['file_access'] :
#new_asset['status'] = 'error'
new_asset['error'] = "can't read file %s"%apath
if asset :
asset = asset[0]
#new_asset['status'] = 'known'
new_asset['info_hash'] = asset['info_hash']
new_asset['torrent_file'] = asset['torrent_file']
new_asset['ftp_url'] = asset['ftp_url']
new_asset['size_torrent'] = asset['size_torrent']
new_asset['error'] = "file already treated, see %s"%asset['id']
if new_asset['status'] != 'new' :
new_asset['end'] = int(time.time())
new_asset['id'] = self.memory.create_asset(new_asset)
print new_asset['id'], new_asset['error']
return new_asset['id']
def prepareTREE(self, iddl, tracker_url, token_file, path):
"""Prepare a TREE file: create one torrent for TREE_xxx.xml itself and one for each asset
@type tracker_url: str
@param tracker_url: url for opentracker
@type token_file: str
@param token_file: token of the file
@type path: str
@param path: path of the TREE
@return: returns the token of the uploaded file
"""
logging.info("prepare TREE %s %s %s %s %s" % (iddl, tracker_url,
self.bokor.token, token_file, path))
tree = FileTree(path)
afiles = tree.assets
ref_path = os.path.dirname(path)
assets = []
for relative_path, afile in afiles:
path_to_file = ref_path + "/" + "/".join(relative_path.split("/")[1:]) + "/" + afile
assets.append(self.prepare_file(iddl, tracker_url, token_file,
path_to_file, relative_path,
len(afiles)))
return assets
@aFeature
def status_prepare_thread(self) :
return self.prepare_thread.isAlive()
@aFeature
def start_prepare_thread(self) :
if not self.prepare_thread.isAlive() :
self.prepare_thread.start()
@aFeature
def delete_assets(self, filters, value, compare = "="):
return self.memory.delete_assets(filters, value, compare)
@aFeature
def delete_asset(self, idasset):
return self.delete_assets('id', idasset)