def retrieve_accounts(bank_name):
w = Weboob()
w.load_backends(caps=(CapBank, ))
bank_backend = w.get_backend(bank_name)
if bank_backend:
bank = Bank.get_or_create(name=bank_name)
accounts = bank_backend.iter_accounts()
for account in accounts:
logger.info(
f'[Accounts] Retrieving account {account.label} - {account.balance} from {bank_name}'
)
db_account = Account.get_or_none(bank=bank[0],
account_id=account.id)
if db_account is None:
db_account = Account.create(bank=bank[0],
account_id=account.id,
user=1,
label=account.label,
balance=account.balance)
else:
db_account.label = account.label
db_account.balance = account.balance
db_account.save()
transactions = bank_backend.iter_history(account)
for transaction in transactions:
db_transanction = Transaction.get_or_create(
account=db_account.id,
label=transaction.label,
category=transaction.category,
amount=transaction.amount,
date=transaction.date)
def get_accounts(bname):
w = Weboob()
w.load_backends(names=[bname])
backend = w.get_backend(bname)
results = {}
for account in backend.iter_accounts():
# a unicode bigger than 8 characters used as key of the table make some bugs in the C++ code
# convert to string before to use it
results[str(account.id)] = {'name': account.label,
'balance': int(account.balance * 100),
'type': int(account.type),
}
return results
def get_accounts(bname):
w = Weboob()
w.load_backends(names=[bname])
backend = w.get_backend(bname)
results = {}
for account in backend.iter_accounts():
# a unicode bigger than 8 characters used as key of the table make some bugs in the C++ code
# convert to string before to use it
results[str(account.id)] = {
'name': account.label,
'balance': int(account.balance * 100),
'type': int(account.type),
}
return results
def get_transactions(bname, accid, maximum):
w = Weboob()
w.load_backends(names=[bname])
backend = w.get_backend(bname)
acc = backend.get_account(accid)
results = {}
results['id'] = acc.id
results['name'] = acc.label
results['balance'] = int(acc.balance * 100)
results['type'] = int(acc.type)
results['transactions'] = []
try:
count = int(maximum)
if count < 1:
count = 0
except:
count = 0
i = 0
first = True
rewriteid = False
seen = set()
for tr in backend.iter_history(acc):
if first:
if tr.id == u'0' or tr.id == u'':
rewriteid = True
first = False
if rewriteid:
tr.id = tr.unique_id(seen)
t = {
'id': tr.id,
'date': tr.date.strftime('%Y-%m-%d'),
'rdate': tr.rdate.strftime('%Y-%m-%d'),
'type': int(tr.type),
'raw': tr.raw,
'category': tr.category,
'label': tr.label,
'amount': int(tr.amount * 100),
}
results['transactions'].append(t)
i += 1
if count != 0 and i >= count:
break
return results
def get_transactions(bname, accid, maximum):
w = Weboob()
w.load_backends(names=[bname])
backend = w.get_backend(bname)
acc = backend.get_account(accid)
results = {}
results['id'] = acc.id
results['name'] = acc.label
results['balance'] = int(acc.balance * 100)
results['type'] = int(acc.type)
results['transactions'] = []
try:
count = int(maximum)
if count < 1:
count = 0
except:
count = 0
i = 0
first = True
rewriteid = False
seen = set()
for tr in backend.iter_history(acc):
if first:
if tr.id == u'0' or tr.id == u'':
rewriteid = True
first = False
if rewriteid:
tr.id = tr.unique_id(seen)
t = {'id': tr.id,
'date': tr.date.strftime('%Y-%m-%d'),
'rdate': tr.rdate.strftime('%Y-%m-%d'),
'type': int(tr.type),
'raw': tr.raw,
'category': tr.category,
'label': tr.label,
'amount': int(tr.amount * 100),
}
results['transactions'].append(t)
i += 1
if count != 0 and i >= count:
break
return results
class Downloadboob(object):
def __init__(self, backend_name, download_directory, links_directory):
self.download_directory = download_directory
self.links_directory = links_directory
self.backend_name = backend_name
self.backend = None
self.weboob = Weboob()
self.weboob.load_backends(modules=[self.backend_name])
self.backend=self.weboob.get_backend(self.backend_name)
def purge(self):
if not os.path.isdir(self.links_directory):
return
dirList=os.listdir(self.links_directory)
for local_link_name in dirList:
link_name = self.links_directory + "/" + local_link_name
if not self.check_link(link_name):
print u"Remove %s" % link_name
os.remove(link_name)
else:
print u"Keep %s" % link_name
def check_link(self, link_name):
if os.path.islink(link_name):
file_name = os.readlink(link_name)
absolute_file_name = os.path.join(self.links_directory, file_name)
if os.path.isfile(absolute_file_name):
return True
return False
else:
return True
def download(self, pattern=None, sortby=CapVideo.SEARCH_RELEVANCE, nsfw=False, max_results=None, title_exclude=[], id_regexp=None):
print "For backend %s, search for '%s'" % (backend_name, pattern)
# create directory for links
print " create link to %s" % self.links_directory
if not os.path.isdir(self.links_directory):
os.makedirs(self.links_directory)
# search for videos
videos = []
for i, video in enumerate(self.backend.search_videos(pattern, sortby, nsfw)):
if i == max_results:
break
if not self.is_downloaded(video):
self.backend.fill_video(video, ('url','title', 'url', 'duration'))
if not(self.is_excluded(video.title, title_exclude)) and self.id_regexp_matched(video.id, id_regexp):
print " %s\n Id:%s\n Duration:%s" % (video.title, video.id, video.duration)
videos.append(video)
else:
print "Already downloaded, check %s" % video.id
self.backend.fill_video(video, ('url','title', 'url', 'duration'))
linkname = self.get_linkname(video)
if not os.path.exists(linkname):
self.remove_download(video)
# download videos
print "Downloading..."
for video in videos:
self.do_download(video)
def is_excluded(self, title, title_exclude):
for exclude in title_exclude:
if title.find(exclude) > -1:
return True
return False
def id_regexp_matched(self, video_id, id_regexp):
if id_regexp:
return re.search(id_regexp, video_id) is not None
return True
def get_filename(self, video, relative=False):
if relative:
directory = os.path.join("..", DOWNLOAD_DIRECTORY, self.backend_name)
else:
directory = os.path.join(self.download_directory, self.backend_name)
if not os.path.exists(directory):
os.makedirs(directory)
ext = video.ext
if not ext:
ext = 'avi'
return u"%s/%s.%s" % (directory, removeNonAscii(video.id), ext)
def get_linkname(self, video):
if not os.path.exists(self.links_directory):
os.makedirs(self.links_directory)
ext = video.ext
if not ext:
ext = 'avi'
misc = video.date
if not misc:
misc = video.id
return u"%s/%s (%s).%s" % (self.links_directory, removeSpecial(video.title), removeSpecial(misc), ext)
def is_downloaded(self, video):
# check if the file is 0 byte
return os.path.isfile(self.get_filename(video))
def remove_download(self, video):
path = self.get_filename(video)
if os.stat(path).st_size == 0:
# already empty
return
print 'Remove video %s' % video.title
# Empty it to keep information we have already downloaded it.
with open(path, 'w'):
pass
def set_linkname(self, video):
linkname = self.get_linkname(video)
idname = self.get_filename(video, relative=True)
absolute_idname = self.get_filename(video, relative=False)
if not os.path.islink(linkname) and os.path.isfile(absolute_idname):
print "%s -> %s" % (linkname, idname)
os.symlink(idname, linkname)
def do_download(self, video):
if not video:
print >>sys.stderr, 'Video not found: %s' % video
return 3
if not video.url:
print >>sys.stderr, 'Error: the direct URL is not available.'
return 4
def check_exec(executable):
with open('/dev/null', 'w') as devnull:
process = subprocess.Popen(['which', executable], stdout=devnull)
if process.wait() != 0:
print >>sys.stderr, 'Please install "%s"' % executable
return False
return True
dest = self.get_filename(video)
if video.url.startswith('rtmp'):
if not check_exec('rtmpdump'):
return 1
args = ('rtmpdump', '-e', '-r', video.url, '-o', dest)
elif video.url.startswith('mms'):
if not check_exec('mimms'):
return 1
args = ('mimms', video.url, dest)
else:
if not check_exec('wget'):
return 1
args = ('wget', video.url, '-O', dest)
os.spawnlp(os.P_WAIT, args[0], *args)
self.set_linkname(video)
class Downloadboob(object):
def __init__(self, backend_name, download_directory, links_directory):
self.download_directory = download_directory
self.links_directory = links_directory
self.backend_name = backend_name
self.backend = None
self.weboob = Weboob()
self.weboob.load_backends(modules=[self.backend_name])
self.backend=self.weboob.get_backend(self.backend_name)
def purge(self):
if not os.path.isdir(self.links_directory):
return
dirList=os.listdir(self.links_directory)
for local_link_name in dirList:
link_name = self.links_directory + "/" + local_link_name
if not self.check_link(link_name):
print u"Remove %s" % link_name
os.remove(link_name)
else:
print u"Keep %s" % link_name
def check_link(self, link_name):
if os.path.islink(link_name):
file_name = os.readlink(link_name)
absolute_file_name = os.path.join(self.links_directory, file_name)
if os.path.isfile(absolute_file_name):
return True
return False
else:
return True
def download(self, pattern=None, sortby=ICapVideo.SEARCH_RELEVANCE, nsfw=False, max_results=None, title_exclude=[], id_regexp=None):
print "For backend %s, search for '%s'" % (backend_name, pattern)
# create directory for links
print " create link to %s" % self.links_directory
if not os.path.isdir(self.links_directory):
os.makedirs(self.links_directory)
# search for videos
videos = []
for i, video in enumerate(self.backend.search_videos(pattern, sortby, nsfw)):
if i == max_results:
break
if not self.is_downloaded(video):
self.backend.fill_video(video, ('url','title', 'url', 'duration'))
if not(self.is_excluded(video.title, title_exclude)) and self.id_regexp_matched(video.id, id_regexp):
print " %s\n Id:%s\n Duration:%s" % (video.title, video.id, video.duration)
videos.append(video)
else:
print "Already downloaded, check %s" % video.id
self.backend.fill_video(video, ('url','title', 'url', 'duration'))
linkname = self.get_linkname(video)
if not os.path.exists(linkname):
self.remove_download(video)
# download videos
print "Downloading..."
for video in videos:
self.do_download(video)
def is_excluded(self, title, title_exclude):
for exclude in title_exclude:
if title.find(exclude) > -1:
return True
return False
def id_regexp_matched(self, video_id, id_regexp):
if id_regexp:
return re.search(id_regexp, video_id) is not None
return True
def get_filename(self, video, relative=False):
if relative:
directory = os.path.join("..", DOWNLOAD_DIRECTORY, self.backend_name)
else:
directory = os.path.join(self.download_directory, self.backend_name)
if not os.path.exists(directory):
os.makedirs(directory)
ext = video.ext
if not ext:
ext = 'avi'
return u"%s/%s.%s" % (directory, removeNonAscii(video.id), ext)
def get_linkname(self, video):
if not os.path.exists(self.links_directory):
os.makedirs(self.links_directory)
ext = video.ext
if not ext:
ext = 'avi'
misc = video.date
if not misc:
misc = video.id
return u"%s/%s (%s).%s" % (self.links_directory, removeSpecial(video.title), removeSpecial(misc), ext)
def is_downloaded(self, video):
# check if the file is 0 byte
return os.path.isfile(self.get_filename(video))
def remove_download(self, video):
path = self.get_filename(video)
if os.stat(path).st_size == 0:
# already empty
return
print 'Remove video %s' % video.title
# Empty it to keep information we have already downloaded it.
with open(path, 'w'):
pass
def set_linkname(self, video):
linkname = self.get_linkname(video)
idname = self.get_filename(video, relative=True)
absolute_idname = self.get_filename(video, relative=False)
if not os.path.islink(linkname) and os.path.isfile(absolute_idname):
print "%s -> %s" % (linkname, idname)
os.symlink(idname, linkname)
def do_download(self, video):
if not video:
print >>sys.stderr, 'Video not found: %s' % video
return 3
if not video.url:
print >>sys.stderr, 'Error: the direct URL is not available.'
return 4
def check_exec(executable):
with open('/dev/null', 'w') as devnull:
process = subprocess.Popen(['which', executable], stdout=devnull)
if process.wait() != 0:
print >>sys.stderr, 'Please install "%s"' % executable
return False
return True
dest = self.get_filename(video)
if video.url.startswith('rtmp'):
if not check_exec('rtmpdump'):
return 1
args = ('rtmpdump', '-e', '-r', video.url, '-o', dest)
elif video.url.startswith('mms'):
if not check_exec('mimms'):
return 1
args = ('mimms', video.url, dest)
else:
if not check_exec('wget'):
return 1
args = ('wget', video.url, '-O', dest)
os.spawnlp(os.P_WAIT, args[0], *args)
self.set_linkname(video)
class DownloadBoob(object):
"""
Search and download
:param name:
:param backend_name:
:param my_download_directory:
:param my_links_directory:
"""
def __init__(self, name, backend_name, my_download_directory,
my_links_directory):
self.download_directory = my_download_directory
self.links_directory = my_links_directory
self.backend_name = backend_name
self.backend = None
self.weboob = Weboob()
self.weboob.load_backends(modules=self.backend_name, )
self.backend = self.weboob.get_backend(self.backend_name)
self.name = name
def get_filename(self, video, relative=False, m3u=False):
"""
Generate filename for the video
:param relative:
:param m3u:
:param video:
:rtype : string
"""
if relative:
directory = os.path.join("..", DOWNLOAD_DIRECTORY,
self.backend_name)
else:
directory = os.path.join(self.download_directory, self.backend_name)
if not os.path.exists(directory.encode('utf8')):
os.makedirs(directory.encode('utf8'))
if not m3u:
ext = video.ext
if not ext:
ext = 'avi'
else:
ext = 'm3u'
return "%s/%s.%s" % (directory, removenonascii(video.id), ext)
def get_linkname(self, video, m3u=False):
"""
Generate filename for the link
:param m3u:
:param video:
:rtype : string
"""
if not os.path.exists(self.links_directory.encode('utf8')):
os.makedirs(self.links_directory.encode('utf8'))
if not m3u:
ext = video.ext
if not ext:
ext = 'avi'
else:
ext = 'm3u'
if not kodi:
misc = video.date.strftime("%y-%m-%d")
if not misc:
misc = video.id
return "%s/%s (%s).%s" % (self.links_directory,
removespecial(video.title),
removespecial(misc), ext)
else:
return "%s/%s.%s" % (self.links_directory,
removespecial(video.title), ext)
def is_downloaded(self, video):
"""
Check if the video has already been downloaded
:param video:
:rtype : bool
"""
if (os.path.isfile(self.get_filename(video).encode('utf8')) or
os.path.isfile(self.get_filename(video,
m3u=True).encode('utf8'))):
logging.info("%s Already downloaded : %s" %
(video.id, video.title))
return True
logging.debug("%s To be downloaded : %s" %
(video.id, video.title))
return False
def init_dir(self):
"""
create directory
"""
if not os.path.isdir(self.links_directory.encode('utf8')):
logging.debug(" create link directory : %s" % self.links_directory)
os.makedirs(self.links_directory.encode('utf8'))
else:
logging.debug(" existing link directory : %s" % self.links_directory)
if kodi:
file_name = os.path.join(self.links_directory, 'tvshow.nfo')
show_name = self.links_directory.split("/")[-1]
if not os.path.isfile(file_name.encode('utf8')):
logging.debug(" create %s" % file_name)
f = codecs.open(file_name, "w", "utf-8")
f.write(u"<tvshow>\n")
f.write(u" <title>" + show_name + u"</title>\n")
f.write(u"</tvshow>\n")
f.close()
else:
logging.debug(" existing %s" % file_name)
def do_search(self,
pattern=None,
pattern_type="search",
sortby=CapVideo.SEARCH_RELEVANCE,
nsfw=False):
"""
Search for videos
:param pattern:
:param pattern_type:
:param sortby:
:param nsfw:
:return:
"""
logging.debug(" Searching for videos for %s" % self.name)
list_videos = []
if pattern_type == "search":
list_videos = self.backend.search_videos(pattern, sortby, nsfw)
elif pattern_type == "ls":
sys.path.insert(
0, backend_directory + "/" + self.backend.name + "/"
) # HACK
if 'video' in sys.modules:
del sys.modules['video']
if self.backend.name == "arte":
from video import ArteVideo as Video_Init
elif self.backend.name == "canalplus":
from video import CanalplusVideo as Video_Init
elif self.backend.name == "arretsurimages":
from video import ArretSurImagesVideo as Video_Init
elif self.backend.name == "dailymotion":
from video import DailymotionVideo as Video_Init
elif self.backend.name == "nolifetv":
from video import NolifeTVVideo as Video_Init
elif self.backend.name == "youtube":
from video import YoutubeVideo as Video_Init
else:
from weboob.capabilities.video import BaseVideo as Video_Init # END OF HACK
for videoid in videoob_list_rep(pattern, self.backend):
list_videos.append(Video_Init(videoid))
if list_videos:
logging.debug(" found videos for %s" % self.name)
else:
logging.error(" did not found videos for %s" % self.name)
return list_videos
def filter_list(self, list_videos, title_regexp, id_regexp, author_regexp,
title_exclude, max_results):
"""
Filter the list after the search
:param list_videos:
:param title_regexp:
:param id_regexp:
:param author_regexp:
:param title_exclude:
:param max_results:
:return:
"""
logging.debug(" filtering list of found video for %s" % self.name)
videos = []
num_videos = 0
for video in list_videos:
if is_ok(video, title_regexp, id_regexp, author_regexp,
title_exclude):
if not self.is_downloaded(video):
videoob_get_info(self.backend, video)
if not video:
logging.error('Error in Video: %s' % video)
elif not video.url:
logging.error(
'Error: the URL is not available : %s (%s)' %
(video.url, video.id))
else:
if is_ok(video, title_regexp, id_regexp, author_regexp,
title_exclude):
num_videos += 1
if not self.is_downloaded(video):
videos.append(video)
print("New Video : %s" % video.title)
print(" Description:%s" % video.description)
print(" Author:%s" % video.author)
print(" Id:%s" % video.id)
print(" Duration:%s" % video.duration)
print(" Date:%s" % video.date)
if num_videos == max_results:
break
return videos
def write_m3u(self, video):
"""
Write m3u file for streaming files
:param video:
:return:
"""
logging.debug(" Write m3u for %s" % video.title)
if video.ext == "m3u" or video.ext == "m3u8":
return do_download(video, self.get_filename(video))
else:
if matched(video.url, "\.m3u") or matched(video.url, "\.m3u8"):
return do_download(video, self.get_filename(video))
else:
dest = self.get_filename(video, m3u=True)
show_name = self.links_directory.split("/")[-1]
logging.debug(" create %s" % dest)
f = codecs.open(dest, "w", "utf-8")
f.write("#EXTINF: ")
if video.duration:
f.write(str(video.duration))
else:
f.write(str(-1))
f.write(", " + show_name + " - " + video.title + "\n")
f.write(video.url)
f.close()
return 0
def set_link(self, video, m3u=False):
"""
Create link file
:param video:
:param m3u:
"""
linkname = self.get_linkname(video, m3u=m3u)
idname = self.get_filename(video, relative=True, m3u=m3u)
absolute_idname = self.get_filename(video, m3u=m3u)
if not os.path.islink(linkname.encode('utf8')) and os.path.isfile(absolute_idname.encode('utf8')):
logging.info(" %s -> %s" % (linkname, idname))
os.symlink(idname.encode('utf8'), linkname.encode('utf8'))
else:
logging.debug(" Not generating link for %s" % video.title)
def do_mv(self, video, m3u=False):
"""
move video file after download
:param video:
:param m3u:
"""
linkname = self.get_linkname(video, m3u=m3u)
absolute_idname = self.get_filename(video, m3u=m3u)
if not os.path.isfile(linkname.encode('utf8')) and os.path.isfile(absolute_idname.encode('utf8')):
logging.info(" %s => %s" % (absolute_idname, linkname))
os.rename(absolute_idname.encode('utf8'), linkname.encode('utf8'))
open(absolute_idname, 'w').close()
else:
logging.debug(" Not moving file %s" % linkname)
def download(self,
pattern=None,
sortby=CapVideo.SEARCH_RELEVANCE,
nsfw=False,
max_results=20,
title_regexp=None,
id_regexp=None,
pattern_type="search",
author_regexp=None,
title_exclude=None):
# create directory for links
"""
Main process
:param pattern:
:param sortby:
:param nsfw:
:param max_results:
:param title_regexp:
:param id_regexp:
:param pattern_type:
:param author_regexp:
:param title_exclude:
"""
self.init_dir()
# search for videos
list_videos = self.do_search(pattern, pattern_type, sortby, nsfw)
# Filter the list of videos
videos = self.filter_list(list_videos, title_regexp, id_regexp,
author_regexp, title_exclude, max_results)
# download videos
if videos:
for video in videos:
print("Downloading... " + video.title)
if kodi: # THE "TITLE" BECOME "S00E00 - TITLE (ID)"
rewrite_title(video)
if down_live: # CREATE LIVE STREAM
ret = self.write_m3u(video)
else: # DOWNLOAD VIDEO
ret = do_download(video, self.get_filename(video)
) # FOR DIRECT LINKS
if not ret:
ret = do_conv(video, self.get_filename(video)
) # FOR INDIRECT LINKS
if not ret:
if not kodi:
self.set_link(
video
) # CREATE LINKS FOR A BEAUTIFULL LIBRARY
else:
self.do_mv(video) # MOVE FILES FOR A BEAUTIFULL LIBRARY
# CREATE NFO FILES FOR KODI
write_nfo(self.get_linkname(video),
self.links_directory, self.backend_name,
video)
print("Downloaded : " + video.title)
else:
assert isinstance(video.title, basestring)
print("Failed download :" + video.title)
def retrieve_accounts():
w = Weboob()
w.load_backends(caps=(CapBank, ))
for i in w.load_backends(CapBank).keys():
Bank.get_or_create(name=i)
user1 = User.get_or_none(username='******', password='******')
user2 = User.get_or_none(username='******', password='******')
if user1 is None:
user1 = User.create(username='******', password='******')
if user2 is None:
user2 = User.create(username='******', password='******')
for bank in Bank.select():
bank_backend = w.get_backend(bank.name)
accounts = bank_backend.iter_accounts()
for account in accounts:
logger.info(
f'[Accounts] Retrieving account {account.id} - {account.label} - {account.balance} from {bank.name}'
)
db_account = Account.get_or_none(bank=bank, account_id=account.id)
if db_account is None:
if Account.select().where(Account.bank == 3):
db_account = Account.create(bank=bank,
account_id=account.id,
user=user2.id,
label=account.label,
balance=account.balance)
else:
db_account = Account.create(bank=bank,
account_id=account.id,
user=user1.id,
label=account.label,
balance=account.balance)
else:
db_account.label = account.label
db_account.balance = account.balance
db_account.save()
transactions = bank_backend.iter_history(bank)
for transaction in transactions:
logger.info(
f'[Transactions] Retrieving account {transaction.label} - {transaction.amount} from {bank.name}'
)
db_transaction = Transaction.get_or_none(
account=db_account, label=transaction.label)
if db_transaction is None:
db_transaction = Transaction.create(
account=db_account,
label=transaction.label,
amount=transaction.amount,
vu=False)
else:
db_transaction.label = transaction.label
db_transaction.amount = transaction.amount
db_transaction.save()
