def receive_file_info(self, cyftinfo):
if cyftinfo.transfer_id in self.accepted_fts:
ym.accept_file(
cyftinfo.sender, cyftinfo.receiver, cyftinfo.filename,
cyftinfo.transfer_id, cyftinfo.transfer_type, cyftinfo.host,
cyftinfo.relay_id
)
transfer_task = self.transfer_tasks[cyftinfo.transfer_id]
finfo = QFileInfo(transfer_task.destination)
if finfo.isDir():
path = '{0}/{1}'.format(transfer_task.destination, cyftinfo.filename)
else:
path = '{0}'.format(transfer_task.destination)
# remove the current file from left files list and download it
transfer_task.remove_file(cyftinfo.filename)
transfer_task.paths.append(path)
self._download_progress(cyftinfo.transfer_id, cyftinfo.filename, 0.0)
self.downloader = FileDownloader(transfer_task, path)
self.downloader.progress.connect(self._download_progress)
self.downloader.finished.connect(self._download_finished)
self.downloader.download(cyftinfo.get_download_url())
def run(dst_path,
datasetconfig,
downloadData,
clean_dst=False,
force_download=True,
overwrite_current_date=None):
try:
print('\n> processing',
datasetconfig["where_to_download"]["identifier"])
dst_dataset_path = os.path.join(
dst_path, datasetconfig["where_to_download"]["identifier"])
if os.path.exists(dst_dataset_path) and clean_dst:
shutil.rmtree(dst_dataset_path)
# download
print('File Downloads:')
fs = FileDownloader(dst_dataset_path)
data = fs.process(datasetconfig,
downloadData,
force=force_download,
current_datetime=overwrite_current_date)
print(data)
except Exception as e:
print('exception in run', e)
exc_type, exc_value, exc_traceback = sys.exc_info()
lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
print(''.join(lines))
def test_multi_thread_download(self):
URL = 'http://olps.cgtransport.org/Esewa/img/ESEWA2.jpg'
resource_size = 24963
with tempfile.TemporaryDirectory() as temp_dir:
logger = self.create_logger('test_multi_thread_download')
download_file = os.path.join(temp_dir, 'download_file')
downloader = FileDownloader(URL, download_file, logger, 4)
downloader.download()
self.assertEqual(os.stat(download_file).st_size, resource_size)
def downloadCaptions(self):
# Disable Buttons
self.disableIOButtons()
self.disableDownloadButtons(False)
root = self.getOutputFolder()
if self.file_downloader == None:
self.file_downloader = FileDownloader(root)
self.file_downloader.attachGUI(self)
self.file_downloader.loadQueueFromList(self.download_queue_captions)
self.file_downloader.startDownloadGui()
def test_setUp(self):
FILE_NAME = 'temp_file'
FILE_SIZE = 4 * 1000 * 1000
SPEED_LIMIT = 256 * 1000 * 1
HOST = "0.0.0.0"
PORT = 10000
with tempfile.TemporaryDirectory() as temp_dir:
logger = self.create_logger('test_simple_server_send')
FILE_SIZE = self.create_dummy_file(
FILE_SIZE, os.path.join(temp_dir, FILE_NAME))
self.assertTrue(os.path.exists(os.path.join(temp_dir, FILE_NAME)))
server = FileServer(HOST, PORT, logger, temp_dir, SPEED_LIMIT)
server.start()
PORT = server.sock.getsockname()[1]
URL = f"http://{HOST}:{PORT}/{FILE_NAME}"
downloader_single_thread = FileDownloader(URL, "simple_download",
logger)
downloader_single_thread.download()
downloader_multi_thread = FileDownloader(URL,
"multi_thread_download",
logger,
threads=4)
downloader_multi_thread.download()
server.stop()
self.assertEqual(os.stat("simple_download").st_size, FILE_SIZE)
self.assertEqual(
os.stat("multi_thread_download").st_size, FILE_SIZE)
def download_from_url(url, output_directory, filename=None, use_cache=True):
"""Download file from a url and put it under output_directory.
:param url: Url that gives response.
:type url: str
:param output_directory: Directory to put the diagram.
:type output_directory: str
:param filename: Optional filename for downloaded file.
:type filename: str
:param use_cache: If there is a cached copy of the file already in the
output directory, do not refetch it (True) or force refecth it (False).
:type use_cache: bool
:returns: File path if success to download, else None
:rtype: str
"""
if filename is None:
filename = get_filename(url)
LOGGER.info('Download file %s from %s' % (filename, url))
file_path = os.path.join(output_directory, filename)
if os.path.exists(file_path) and use_cache:
LOGGER.info('File %s exists, not downloading' % file_path)
return file_path
# Set Proxy in webpage
proxy = get_proxy()
network_manager = QNetworkAccessManager()
if not proxy is None:
network_manager.setProxy(proxy)
# Download Process
# noinspection PyTypeChecker
downloader = FileDownloader(network_manager, url, file_path)
try:
result = downloader.download()
except IOError as ex:
raise DownloadException(ex)
if result[0] is not True:
_, error_message = result
raise DownloadException(error_message)
if os.path.exists(file_path):
return file_path
else:
return None
def download_from_url(url, output_directory, filename=None, use_cache=True):
"""Download file from a url and put it under output_directory.
:param url: Url that gives response.
:type url: str
:param output_directory: Directory to put the diagram.
:type output_directory: str
:param filename: Optional filename for downloaded file.
:type filename: str
:param use_cache: If there is a cached copy of the file already in the
output directory, do not refetch it (True) or force refecth it (False).
:type use_cache: bool
:returns: File path if success to download, else None
:rtype: str
"""
if filename is None:
filename = get_filename(url)
LOGGER.info('Download file %s from %s' % (filename, url))
file_path = os.path.join(output_directory, filename)
if os.path.exists(file_path) and use_cache:
LOGGER.info('File %s exists, not downloading' % file_path)
return file_path
# Set Proxy in webpage
proxy = get_proxy()
network_manager = QNetworkAccessManager()
if not proxy is None:
network_manager.setProxy(proxy)
# Download Process
# noinspection PyTypeChecker
downloader = FileDownloader(network_manager, url, file_path)
try:
result = downloader.download()
except IOError as ex:
raise DownloadException(ex)
if result[0] is not True:
_, error_message = result
raise DownloadException(error_message)
if os.path.exists(file_path):
return file_path
else:
return None
def process_wget(input_data):
data = validate_wget(input_data)
if data:
url = data[0]
if data[1]:
extension = data[1]
# todo: а куда скачивать?
file_downloader = FileDownloader(url, extension)
file_downloader.download_files()
else:
# todo : download with url only
pass
# print(data)
# print('op : ' + str(data[0]))
# print('url : ' + data[1])
# print('extension : ' + str(data[2]))
pass
def start_a_job(self):
current_job = None
for job in self.job_list:
if current_job == None and not job["isQueued"] and not job[
"isFinished"]:
self.current_active_jobz += 1
job["isQueued"] = True
current_job = job
print(f"Current Job is:{current_job}")
if current_job != None:
message = current_job["message"]
downloader = FileDownloader(message, self)
# Start the image process by downloading the image
downloader.get_file_path()
from file_downloader import FileDownloader
with open('io-data/download-logger.txt', 'a') as log_file:
fileDownLoader = FileDownloader(file_with_urls='t-shirt-links.txt', logger=log_file)
# fileDownLoader = FileDownloader(file_with_urls='1.txt')
fileDownLoader.download_files()
import logging
import argparse
from file_downloader import FileDownloader
#URL1 = 'http://olps.cgtransport.org/Esewa/img/ESEWA2.jpg'
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--url')
parser.add_argument('--save_as', type=str, default='download_file')
parser.add_argument('--threads', type=int, default=1)
parser.add_argument('--verbose', '-v', action='store_true')
args = parser.parse_args()
if args.verbose:
log_level = logging.DEBUG
else:
log_level = logging.WARNING
logger = logging.getLogger('file_downloader_logger')
logger.setLevel(log_level)
console_handler = logging.StreamHandler()
console_handler.setLevel(log_level)
formatter = logging.Formatter('%(asctime)s - %(message)s')
console_handler.setFormatter(formatter)
logger.addHandler(console_handler)
downloader = FileDownloader(args.url, args.save_as, logger, args.threads)
downloader.download()
def main(self):
'''
:return:
'''
# Always do these steps regardless of option
# Parse variables files to extract relevant sample information
# Identify samples to upload to BaseSpace
samples_to_upload = identify_samples()
# Load and parse out variables from variables files associated with each sample
all_variables = load_all_variables(samples_to_upload, os.getcwd())
# Identify the worksheet number which will be used as the project name in BaseSpace
worksheet = identify_worksheet(all_variables)
# Pair samples- DNA sample is key, RNA sample to look up- if No RNA sample, it is None
sample_pairs = create_sample_pairs(all_variables)
# Write out sample pairs to log file for checking if needed
log.warning(f"sample pairs are {sample_pairs}")
# Locate the fastqs associated with all samples
all_fastqs = locate_all_fastqs(samples_to_upload, os.getcwd())
# Create a project in BaseSpace- will not create if it already exists, but will still return project id
upload = FileUpload(self.authentication_token, worksheet,
samples_to_upload, all_fastqs)
project = upload.create_basespace_project()
log.info(f"Project {worksheet} created")
log.warning(f"Project id for project name {worksheet} is {project}")
# If whole pipeline required then upload fastq files
if not args.tst170 and not args.smp2 and not args.dl_files:
# Upload fastq files
print(f"uploading fastq files for all samples")
upload.upload_files()
# Create launch app object for TST170 app
launch_tst = LaunchApp(self.authentication_token, worksheet, project,
app_name, app_version, sample_pairs)
# If resuming from TST170 required or full pipeline- launch the TST170 app
if not args.smp2 and not args.dl_files:
# Launch TST170 application for each pair in turn
# IMPORTANT NOTE: Only processes paired data
tst_170 = launch_tst.launch_tst170_pairs()
# Dump data to file
with open(os.path.join(os.getcwd(), "tst_170.json"), 'w') as t:
json.dump(tst_170, t)
# If resuming from SMP2v3 launch load in required TST170 data from file
elif args.smp2:
try:
with open(os.path.join(os.getcwd(), "tst_170.json")) as ts:
tst_170 = json.load(ts)
except FileNotFoundError:
raise FileNotFoundError(
f"Could not find file tst_170.json. Cannot resume pipeline from SMP2 step."
f"Please delete TST170 analysis in BaseSpace and resume pipeline from"
f"TST170 stage.")
# If resuming from SMP2v3 required, resuming from TST170 required or full pipeline- launch the SMP2 app
if not args.dl_files:
# Create launch app object for SMP2 v3 if not just downloading files- poll TST170 and when complete
# launch SMP2
launch_smp = LaunchApp(self.authentication_token, worksheet,
project, smp2_app_name, smp2_app_version,
sample_pairs, tst_170)
# Poll the tst 170 appsessions until completion, then launch smp2 app
smp_appsession = launch_smp.poll_tst170_launch_smp2()
# Dump data to file
with open(os.path.join(os.getcwd(), "smp.json"), 'w') as s:
json.dump(smp_appsession, s)
# If downloading files from a completed SMP2 app required
# Create a LaunchApp object for smp2 app if flag to only download files is set- allows for polling of SMP2
if args.dl_files:
# Load data in required smp2 data from file
try:
with open(os.path.join(os.getcwd(), "smp.json")) as sm:
smp = json.load(sm)
except FileNotFoundError:
raise FileNotFoundError(
f"Could not find file smp.json. Cannot resume pipeline from download step."
f"Please delete SMP2 analysis in BaseSpace and resume pipeline from"
f"SMP2 stage.")
launch_smp = LaunchApp(self.authentication_token, worksheet,
project, smp2_app_name, smp2_app_version,
sample_pairs, None,
smp) # None as tst170 app data not required
# Poll the smp appsessions until completion
smp_appresults = launch_smp.poll_smp2()
# Download all required files- every step requires this
file_download = FileDownloader(self.authentication_token,
smp_appresults, worksheet)
file_download.download_files()
from flask.ext.sqlalchemy import SQLAlchemy
import gevent
from gevent import monkey
# patches stdlib (including socket and ssl modules) to cooperate with other greenlets
monkey.patch_all()
app = Flask(__name__)
app.config.from_object('config')
db = SQLAlchemy(app)
import logging
from logging.handlers import RotatingFileHandler
file_handler = RotatingFileHandler('tmp/downloads.log', 'a', 1 * 1024 * 1024,
10)
#
file_handler.setFormatter(
logging.Formatter(
'%(asctime)s %(levelname)s: %(message)s [in %(pathname)s:%(lineno)d]'))
app.logger.setLevel(logging.DEBUG)
file_handler.setLevel(logging.DEBUG)
app.logger.addHandler(file_handler)
app.logger.info('===startup downloader===')
log = app.logger.info
log2 = app.logger.debug
from downloads import file_manager
from file_downloader import FileDownloader
Downloads = FileDownloader()
from PySide2.QtCore import QObject, Signal, Property, QUrl
from PySide2.QtGui import QGuiApplication, QIcon
from PySide2.QtQml import QQmlApplicationEngine
from file_downloader import FileDownloader
from program_settings import SettingsManager
import resources
import sys
app = QGuiApplication(sys.argv)
app.setWindowIcon(QIcon(':/mi.png'))
engine = QQmlApplicationEngine()
settings_man = SettingsManager()
fd = FileDownloader()
engine.rootContext().setContextProperty("filedownloader", fd)
engine.rootContext().setContextProperty("sm", settings_man)
engine.load(QUrl.fromLocalFile(':/Main.qml'))
if not engine.rootObjects():
sys.exit(-1)
sys.exit(app.exec_())
class ChatWidget(QWidget):
def __init__(self, parent, cybuddy):
super(ChatWidget, self).__init__()
self.widget = uic.loadUi('ui/chatwidget.ui')
self.parent_window = parent
self.app = parent.app
self.cybuddy = cybuddy
self.typingTimer = None
self.is_ready = False
self.queue = []
self.accepted_fts = [] # accepted file transfers
self.transfer_tasks = {}
self.webcam_viewer = None
defaultSettings = QWebSettings.globalSettings()
defaultSettings.setAttribute(QWebSettings.JavascriptEnabled, True)
defaultSettings.setAttribute(QWebSettings.PluginsEnabled, True)
defaultSettings.setAttribute(QWebSettings.DeveloperExtrasEnabled, True)
ym.register_callback(cb.EMUSSA_CALLBACK_TYPING_NOTIFY, self._typing)
self.widget.textEdit.keyPressEvent = self._writing_message
self.widget.sendButton.clicked.connect(self._send_message)
self.widget.filetransfer_btn.clicked.connect(self._send_file)
self.widget.myAvatar.setPixmap(self.app.me.avatar.image)
self.widget.messagesView.setUrl(QUrl('ui/resources/html/chat/index.html'))
self.widget.messagesView.loadFinished.connect(self._document_ready)
self.widget.messagesView.linkClicked.connect(self._link_clicked)
self.widget.messagesView.page().setLinkDelegationPolicy(QWebPage.DelegateAllLinks)
self.cybuddy.update.connect(self._update_buddy)
self.cybuddy.update_status.connect(self._update_status)
self.cybuddy.update_avatar.connect(self._update_avatar)
self.widget.addUserBtn.clicked.connect(self.add_buddy)
self._update_buddy()
self._update_status()
self._update_avatar()
if cybuddy.yahoo_id in self.app.buddylist.buddy_items:
self.widget.addUserBtn.setHidden(True)
self.widget.ignoreUserBtn.setHidden(True)
def _javascript(self, function, *args):
# if the document is not ready, wait a while until we start calling JS functions on it
if not self.is_ready:
self.queue.append([function, args])
arguments_list = []
for arg in args:
jsarg = str(arg).replace("'", "\\'")
arguments_list.append("'" + jsarg + "'")
jscode = function + "(" + ", ".join(arguments_list) + ")"
self.widget.messagesView.page().mainFrame().evaluateJavaScript(jscode)
def _document_ready(self):
self.is_ready = True
if self.app.me.status.code == YAHOO_STATUS_INVISIBLE:
pixmap = QPixmap(":status/resources/user-invisible.png")
self._add_info('You appear offline to <span class="buddyname">{0}</span>'.format(
self.cybuddy.display_name)
)
elif not self.cybuddy.status.online:
pixmap = QPixmap(":status/resources/user-offline.png")
self._add_info('<span class="buddyname">{0}</span> seems to be offline and will'
'receive your messages next time when he/she logs in.'
.format(self.cybuddy.display_name),
pixmap)
for task in self.queue:
self._javascript(task[0], *task[1])
self.queue = []
self._javascript('show_timestamps', 'true')
def _link_clicked(self, url):
if url.scheme() == 'cyuf':
transfer_id = QFileInfo(url.path()).fileName()
if url.host() == 'save':
transfer_task = self.transfer_tasks[transfer_id]
path = None
if len(transfer_task.files) == 1:
path = QFileDialog.getSaveFileName(
self.widget,
"Save file",
"{0}/{1}".format(QDir.homePath(), transfer_task.files[0].filename),
"Any file (*.*)"
)
else:
path = QFileDialog.getExistingDirectory(
self.widget,
"Save file",
"{0}".format(QDir.homePath()),
QFileDialog.ShowDirsOnly
)
if path:
print('Save as: {0}'.format(path))
transfer_task.destination = path
self.accepted_fts.append(transfer_id)
ym.accept_transfer(self.cybuddy.yahoo_id, transfer_id)
if url.host() == 'decline':
self._javascript('file_decline',
self.cybuddy.yahoo_id,
transfer_id)
ym.decline_transfer(self.cybuddy.yahoo_id, transfer_id)
if url.host() == 'cancel-send':
self._javascript('cancel_send',
self.cybuddy.yahoo_id,
transfer_id)
ym.cancel_transfer(self.cybuddy.yahoo_id, transfer_id)
if url.host() == 'open-file' or url.host() == 'open-dir':
QDesktopServices.openUrl(QUrl(url.path()))
if url.host() == 'decline-web':
yahoo_id = url.path()[1:]
self._javascript('webcam_declined', yahoo_id)
ym.deny_webcam_request(yahoo_id)
if url.host() == 'accept-web':
yahoo_id = url.path()[1:]
self._javascript('webcam_accepted', yahoo_id)
ym.accept_webcam_request(yahoo_id)
self.view_webcam()
def _get_link_from_status(self):
sep = ''
statusmsg = self.cybuddy.status.message
if statusmsg:
sep = ' - '
if 'http://' in statusmsg:
href = statusmsg[statusmsg.index('http://'):].split(' ', 1)[0]
statusmsg = statusmsg.replace(href, '').lstrip().rstrip()
text = '<font color="#8C8C8C">{0}<a href="{1}">{2}</a></font>'.format(sep, href, statusmsg)
return [href, text]
elif 'https://' in statusmsg:
href = statusmsg[statusmsg.index('https://'):].split(' ', 1)[0]
statusmsg = statusmsg.replace(href, '').lstrip().rstrip()
text = '<font color="#8C8C8C">{0}<a href="{1}">{2}</a></font>'.format(sep, href, statusmsg)
return [href, text]
elif 'www.' in statusmsg:
href = statusmsg[statusmsg.index('www.'):].split(' ', 1)[0]
statusmsg = statusmsg.replace(href, '').lstrip().rstrip()
text = '<font color="#8C8C8C">{0}<a href="{1}">{2}</a></font>'.format(sep, href, statusmsg)
return [href, text]
else:
text = '<font color="#8C8C8C">{0}{1}</font>'.format(sep, self.cybuddy.status.message)
return ['', text]
def _update_buddy(self):
self.widget.contactName.setText(self.cybuddy.display_name)
self._javascript('update_buddy_name', self.cybuddy.display_name)
def _update_status(self):
if self.cybuddy.status.online:
if self.cybuddy.status.idle_time:
self.widget.contactStatus.setPixmap(QPixmap(":status/resources/user-away.png"))
if self.sender().__class__.__name__ == 'CyBuddy':
self._notify_status('is Idle.')
elif self.cybuddy.status.code == YAHOO_STATUS_BUSY:
self.widget.contactStatus.setPixmap(QPixmap(":status/resources/user-busy.png"))
if self.sender().__class__.__name__ == 'CyBuddy':
self._notify_status('is Busy.')
else:
self.widget.contactStatus.setPixmap(QPixmap(":status/resources/user-online.png"))
if self.sender().__class__.__name__ == 'CyBuddy':
self._notify_status('is now Available.')
statusMsg = self._get_link_from_status()
self.widget.contactStatusText.setText(statusMsg[1])
self.widget.contactStatusText.setToolTip(statusMsg[0])
else:
self.widget.contactStatus.setPixmap(QPixmap(":status/resources/user-offline.png"))
self.widget.contactStatusText.setText('')
self.widget.contactStatusText.setToolTip('')
if self.sender().__class__.__name__ == 'CyBuddy':
self._notify_status('is now Offline.')
def _notify_status(self, message):
self._javascript('status_updated',
'<span class="buddy_items">{0}</span> {1}'
.format(self.cybuddy.display_name, message),
util.pixmap_to_base64(self.widget.contactStatus.pixmap()),
datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
def _update_avatar(self):
self.widget.hisAvatar.setPixmap(self.cybuddy.avatar.image)
def _writing_message(self, e):
if e.key() == Qt.Key_Return or e.key() == Qt.Key_Enter:
self.widget.sendButton.click()
ym.send_typing(self.cybuddy.yahoo_id, False)
self.killTimer(self.typingTimer)
self.typingTimer = None
else:
QTextEdit.keyPressEvent(self.widget.textEdit, e)
if e.key() > Qt.Key_0 and e.key() < Qt.Key_Z:
if self.typingTimer:
self.killTimer(self.typingTimer)
else:
ym.send_typing(self.cybuddy.yahoo_id, True)
self.typingTimer = self.startTimer(5000)
def _typing(self, cyemussa, tn):
sender = tn.sender
if not sender:
# we are typing this from somewhere else
sender = self.app.me.yahoo_id
if not sender == self.app.me.yahoo_id and not sender == self.cybuddy.yahoo_id:
return
if tn.status:
self._javascript('start_typing', sender)
else:
self._javascript('stop_typing')
def _add_info(self, text, pixmap=None):
image = None
if pixmap:
image = util.pixmap_to_base64(pixmap)
# text = util.sanitize_html(text)
if image:
self._javascript('add_info', text, image)
else:
self._javascript('add_info', text)
def _send_message(self):
raw_msg = self.widget.textEdit.toPlainText()
timestamp = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
self.widget.textEdit.setDocument(QTextDocument())
ym.send_message(self.cybuddy.yahoo_id, str(raw_msg))
self._javascript('message_out', self.app.me.yahoo_id, self._text_to_emotes(raw_msg), timestamp)
def _send_file(self):
files = QFileDialog.getOpenFileNames(
self.widget,
"Select one or more files to send",
QDir.homePath(),
"Any file (*.*)"
)
ftinfo = cyemussa.CyFileTransferInfo()
ftinfo.sender = self.app.me.yahoo_id
ftinfo.receiver = self.cybuddy.yahoo_id
ftinfo.host = ym.ft_host
ftinfo.transfer_id = util.string_to_base64(
util.string_to_md5(util.random_string(25))
)
transfer_task = FileTransferTask()
transfer_task.transfer_info = ftinfo
files_absolute = []
sizes = []
for path in files:
f = cyemussa.CyFile()
f.filename = QFile(path).fileName()
f.filesize = QFile(path).size()
transfer_task.files.append(f)
files_absolute.append(QFileInfo(f.filename).fileName())
sizes.append(f.filesize)
self.transfer_tasks[transfer_task.transfer_info.transfer_id] = transfer_task
thumbnail = None
if len(files) == 1 and QImageReader.imageFormat(files[0]):
icon = util.scalePixmapAspectFill(QPixmap(files[0]), QSize(32, 32))
icon_base64 = util.pixmap_to_base64(icon, 'JPG')
thumbnail = icon_base64
else:
icon = QFileIconProvider().icon(QFileInfo(files[0]))
icon_base64 = util.pixmap_to_base64(icon.pixmap(QSize(32, 32)))
self._javascript(
'file_out',
ftinfo.receiver,
json.dumps(files_absolute),
json.dumps(sizes),
ftinfo.transfer_id,
icon_base64
)
ym.send_files(
self.cybuddy.yahoo_id,
transfer_task.transfer_info.transfer_id,
files,
thumbnail
)
def _text_to_emotes(self, text):
words = text.split()
for i, w in enumerate(words):
for emo in emotes:
pattern = re.compile(re.escape(emo), re.IGNORECASE)
word = pattern.sub('<img src="{0}" alt="{1}" />'.format(emotes[emo], emo), w)
if not word == w: # a replacement was made, skip to the next word
words[i] = word
break
text = ' '.join(words)
return text
def _download_progress(self, transfer_id, filename, progress):
self._javascript(
'file_progress',
transfer_id,
QFileInfo(filename).fileName(),
progress
)
def _download_finished(self, transfer_id):
transfer_task = self.transfer_tasks[transfer_id]
if len(transfer_task.files):
ym.accept_transfer_next_file(self.cybuddy.yahoo_id, transfer_id)
self._download_progress(transfer_id, transfer_task.files[0], 0.0)
else:
del self.transfer_tasks[transfer_id]
if QFileInfo(transfer_task.destination).isDir():
action = 'open-dir'
else:
action = 'open-file'
self._javascript(
'transfer_finished',
transfer_id,
transfer_task.transfer_info.sender,
len(transfer_task.paths),
action,
transfer_task.destination
)
def timerEvent(self, event):
ym.send_typing(self.cybuddy.yahoo_id, False)
if self.typingTimer:
self.killTimer(self.typingTimer)
self.typingTimer = None
def close(self):
# called by parent when the chat is closing
ym.unregister_callback(cb.EMUSSA_CALLBACK_TYPING_NOTIFY, self._typing)
self.cybuddy.update.disconnect(self._update_buddy)
if self.typingTimer:
ym.send_typing(self.cybuddy.yahoo_id, False)
def add_buddy(self):
self.add_buddy_wizard = AddBuddyWizard(self.app, self.cybuddy.yahoo_id)
def receive_message(self, cymessage):
message = util.yahoo_rich_to_html(cymessage.message)
timestamp = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
if not cymessage.sender:
sender = self.app.me.yahoo_id
else:
sender = self.cybuddy.display_name
if cymessage.offline:
message = '(offline) {0}'.format(message)
if cymessage.timestamp:
timestamp = datetime.datetime.fromtimestamp(int(cymessage.timestamp)).strftime('%Y-%m-%d %H:%M:%S')
self._javascript('message_in', sender, self._text_to_emotes(util.sanitize_html(message)), timestamp)
def receive_file_transfer(self, cyfiletransfer):
files = []
sizes = []
ftinfo = cyemussa.CyFileTransferInfo()
ftinfo.transfer_id = cyfiletransfer.transfer_id
ftinfo.sender = cyfiletransfer.sender
transfer_task = FileTransferTask()
transfer_task.transfer_info = ftinfo
for f in cyfiletransfer.files:
transfer_task.files.append(cyemussa.CyFile(f))
files.append(f.filename)
sizes.append(f.filesize)
self.transfer_tasks[cyfiletransfer.transfer_id] = transfer_task
if not cyfiletransfer.thumbnail:
icon = QFileIconProvider().icon(QFileInfo(files[0]))
icon_base64 = util.pixmap_to_base64(icon.pixmap(QSize(32, 32)))
else:
icon_base64 = cyfiletransfer.thumbnail
self._javascript(
'file_in',
cyfiletransfer.sender,
json.dumps(files),
json.dumps(sizes),
cyfiletransfer.transfer_id,
icon_base64
)
def receive_file_info(self, cyftinfo):
if cyftinfo.transfer_id in self.accepted_fts:
ym.accept_file(
cyftinfo.sender, cyftinfo.receiver, cyftinfo.filename,
cyftinfo.transfer_id, cyftinfo.transfer_type, cyftinfo.host,
cyftinfo.relay_id
)
transfer_task = self.transfer_tasks[cyftinfo.transfer_id]
finfo = QFileInfo(transfer_task.destination)
if finfo.isDir():
path = '{0}/{1}'.format(transfer_task.destination, cyftinfo.filename)
else:
path = '{0}'.format(transfer_task.destination)
# remove the current file from left files list and download it
transfer_task.remove_file(cyftinfo.filename)
transfer_task.paths.append(path)
self._download_progress(cyftinfo.transfer_id, cyftinfo.filename, 0.0)
self.downloader = FileDownloader(transfer_task, path)
self.downloader.progress.connect(self._download_progress)
self.downloader.finished.connect(self._download_finished)
self.downloader.download(cyftinfo.get_download_url())
def transfer_cancelled(self, cyfiletransfer):
self._javascript(
'file_cancel',
cyfiletransfer.sender,
cyfiletransfer.transfer_id
)
def transfer_accepted(self, cyfiletransfer):
transfer_task = self.transfer_tasks[cyfiletransfer.transfer_id]
ym.send_file_info(
self.cybuddy.yahoo_id,
cyfiletransfer.transfer_id,
QFileInfo(transfer_task.files[0].filename).fileName()
)
def transfer_rejected(self, cyfiletransfer):
self._javascript(
'file_rejected',
cyfiletransfer.sender,
cyfiletransfer.transfer_id
)
def transfer_upload(self, cyftinfo):
transfer_task = self.transfer_tasks[cyftinfo.transfer_id]
if not cyftinfo.relay_id:
# this is an already started multi-file transfer
# which now requires us to upload the next file
transfer_task.files.pop(0)
if len(transfer_task.files):
filepath = transfer_task.files[0].filename
self.uploader = FileUploader(transfer_task)
self.uploader.progress.connect(self._download_progress)
self.uploader.finished.connect(self._download_finished)
ym.send_file_info(
self.cybuddy.yahoo_id,
cyftinfo.transfer_id,
QFileInfo(filepath).fileName()
)
# self.uploader.upload(
# transfer_task.transfer_info.get_download_url(),
# filepath
# )
return
filepath = transfer_task.files[0].filename
transfer_task.transfer_info.relay_id = cyftinfo.relay_id
self.uploader = FileUploader(transfer_task)
self.uploader.progress.connect(self._download_progress)
self.uploader.finished.connect(self._download_finished)
self.uploader.upload(
transfer_task.transfer_info.get_download_url(),
filepath
)
def webcam_invite(self, webcam_notification):
self._javascript(
'webcam_invite',
webcam_notification.sender
)
def view_webcam(self):
ym.send_webcam_request(self.cybuddy.yahoo_id)
self.webcam_viewer = WebcamViewer(self)
def receive_audible(self, audible):
self.audible = audible
self.manager = QNetworkAccessManager()
self.manager.finished.connect(self.post_audible)
self.req = QNetworkRequest(QUrl(self.audible.url))
self.req.setRawHeader('User-agent', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US)')
self.reply = self.manager.get(self.req)
def post_audible(self, swfdata):
tmpfile = QTemporaryFile('XXXXXX.swf', self.widget)
if tmpfile.open():
tmpfile.write(swfdata.readAll())
tmpfile.close()
timestamp = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
self._javascript('audible_in', self.audible.sender, tmpfile.fileName(), self.audible.message, timestamp)
import sys
sys.path.append('..')
from file_downloader import FileDownloader
if __name__=="__main__":
FD = FileDownloader('../../conf/global_var.json')
batch = [(u'704FCD3C4D673DE631BF985CFEABCB121A4EB7CF1FD135E1C98CFA02268FA035', u'https://s3.amazonaws.com/memex-images/full/bdb9cdde29d5a29411fe7a7dc1ca501a37303431.jpg', {u'_type': u'escorts', u'_timestamp': 1464761131138, u'_index': u'memex-domains_2016.01', u'_score': 0.0, u'fields': {u'obj_parent': [u'2E7904456709725B2B1F3108683D6895FCE0BDF8BFA29E34FD799FF8AAE4CFE5'], u'timestamp': [u'2016-01-26T12:31:11'], u'obj_original_url': [u'http://469-978-3183.escortphonelist.com/images/escort-phone-list.jpg'], u'obj_stored_url': [u'https://s3.amazonaws.com/memex-images/full/bdb9cdde29d5a29411fe7a7dc1ca501a37303431.jpg']}, u'_id': u'704FCD3C4D673DE631BF985CFEABCB121A4EB7CF1FD135E1C98CFA02268FA035'}), (u'3F98B492B2BA98676202271F85D2E3EF0D1E02B9A1C706E76D53C8AAA374494B', u'https://s3.amazonaws.com/memex-images/full/7f667480e65f17e2bec4c3da58bd407d06c4276a.jpg', {u'_type': u'escorts', u'_timestamp': 1464761131138, u'_index': u'memex-domains_2016.01', u'_score': 0.0, u'fields': {u'obj_parent': [u'44D3364D05EF7544A4B8ED36738FAFC9F56651D29D76C930DB73588A5221434B'], u'timestamp': [u'2016-01-26T12:32:20'], u'obj_original_url': [u'http://images.eroticmugshots.com/cities/63/large/21490623-2.jpg'], u'obj_stored_url': [u'https://s3.amazonaws.com/memex-images/full/7f667480e65f17e2bec4c3da58bd407d06c4276a.jpg']}, u'_id': u'3F98B492B2BA98676202271F85D2E3EF0D1E02B9A1C706E76D53C8AAA374494B'}), (u'701DA3E5D25EB3329D710C7181ED6DA9D60D081711822EDA056D29F808E39A95', u'https://s3.amazonaws.com/memex-images/full/219e1c7ba41154f5dada28d1a0bddac280900f0d.jpg', {u'_type': u'escorts', u'_timestamp': 1464761131138, u'_index': u'memex-domains_2016.01', u'_score': 0.0, u'fields': {u'obj_parent': [u'7FE5C4C07A16FFFC22E359BD067D15C816F04BD433950C95767CE81DDB441923'], u'timestamp': [u'2016-01-26T12:32:11'], u'obj_original_url': [u'http://images.eroticmugshots.com/cities/13/thumbnail/34348493-1.jpg'], u'obj_stored_url': [u'https://s3.amazonaws.com/memex-images/full/219e1c7ba41154f5dada28d1a0bddac280900f0d.jpg']}, u'_id': u'701DA3E5D25EB3329D710C7181ED6DA9D60D081711822EDA056D29F808E39A95'}), (u'6D7387E477A206C965EF1535BB83E004147531CE99F9AA14D2B3FD6FC94B22F1', u'https://s3.amazonaws.com/memex-images/full/173446f78bf85d8baf94456c4918e3941e72ee80.jpg', {u'_type': u'escorts', u'_timestamp': 1464761131138, u'_index': u'memex-domains_2016.01', u'_score': 0.0, u'fields': {u'obj_parent': [u'1E61CBB6E97B170EF81926D38E0CFBCD4F31763C1461051447713CF4601E8E7D'], u'timestamp': [u'2016-01-26T12:32:16'], u'obj_original_url': [u'http://images.escortsincollege.com/cities/126/large/10593977-3.jpg'], u'obj_stored_url': [u'https://s3.amazonaws.com/memex-images/full/173446f78bf85d8baf94456c4918e3941e72ee80.jpg']}, u'_id': u'6D7387E477A206C965EF1535BB83E004147531CE99F9AA14D2B3FD6FC94B22F1'}), (u'5B369C1A8E426BC5A5A44FDCAC9D9CE378094967AFEAC184A49A5BAF22E62EF5', u'https://s3.amazonaws.com/roxyimages/5c76b19d1548351973868f9ff97d9c0a52f918cb.jpg', {u'_type': u'escorts', u'_timestamp': 1464761131889, u'_index': u'memex-domains_2016.01', u'_score': 0.0, u'fields': {u'obj_parent': [u'51FEA8D56B4AECCDEFB4469A08DDECD29413334D6575F43B6268CA026E85E586'], u'timestamp': [u'2015-12-17T23:06:11'], u'obj_original_url': [u'http://images4.backpage.com/imager/u/large/125355586/3ef7f93df039c1fa1b6dadfa6c482745.jpg'], u'obj_stored_url': [u'https://s3.amazonaws.com/roxyimages/5c76b19d1548351973868f9ff97d9c0a52f918cb.jpg']}, u'_id': u'5B369C1A8E426BC5A5A44FDCAC9D9CE378094967AFEAC184A49A5BAF22E62EF5'}), (u'0A775A8BDE19A6EA640636F02FD451056D5FDA2300844DCC0AFBC652C02BA37B', u'https://s3.amazonaws.com/roxyimages/3f844e64033e87ca67cfde1154bec0578ee6e2cb.jpg', {u'_type': u'escorts', u'_timestamp': 1464761131889, u'_index': u'memex-domains_2016.01', u'_score': 0.0, u'fields': {u'obj_parent': [u'BD7FB7967E393CEFA88185132FC3106C5DF162F6F63A21AAD6632170F9EDC858'], u'timestamp': [u'2015-12-30T05:58:09'], u'obj_original_url': [u'http://images2.backpage.com/imager/u/large/230328867/01bf2acb8bdf85b6df93aeeb2394c11b.jpg'], u'obj_stored_url': [u'https://s3.amazonaws.com/roxyimages/3f844e64033e87ca67cfde1154bec0578ee6e2cb.jpg']}, u'_id': u'0A775A8BDE19A6EA640636F02FD451056D5FDA2300844DCC0AFBC652C02BA37B'}), (u'CAFB17AB66382E6834B1AB6ED854750054188789532F80DE99B7253B811BDB1F', u'https://s3.amazonaws.com/roxyimages/54f226755c7371c1651c5b0920ca563b89e03860.jpg', {u'_type': u'escorts', u'_timestamp': 1464761131889, u'_index': u'memex-domains_2016.01', u'_score': 0.0, u'fields': {u'obj_parent': [u'1383B4D668C03367999D645EC4418A450AC5FF5D19EE0681E39CCB4E7FA00642'], u'timestamp': [u'2015-12-16T23:58:17'], u'obj_original_url': [u'http://images4.backpage.com/imager/u/large/125277948/f58eca086858632543b39fcd0d2636fd.jpg'], u'obj_stored_url': [u'https://s3.amazonaws.com/roxyimages/54f226755c7371c1651c5b0920ca563b89e03860.jpg']}, u'_id': u'CAFB17AB66382E6834B1AB6ED854750054188789532F80DE99B7253B811BDB1F'}), (u'C0BABA15173709DA7A56207670578312CA30A7E94EDBF68A8140D8D5AA3C20FC', u'https://s3.amazonaws.com/memex-images/full/2d7348b6561c9329f8f8841dc929738ada17e95c.jpg', {u'_type': u'escorts', u'_timestamp': 1464761131973, u'_index': u'memex-domains_2016.01', u'_score': 0.0, u'fields': {u'obj_parent': [u'60C23254F4EADC5B927316AF6596BFC81CBE4187AE00419F7EAB20424F34ED91'], u'timestamp': [u'2016-01-26T13:15:21'], u'obj_original_url': [u'http://img13.asexyservice.com/4/p/200/56a6f1147f975rfRkwq_jZw8Dfd2a72kQGmO6PRwRaWKjXWdp4_200.jpg'], u'obj_stored_url': [u'https://s3.amazonaws.com/memex-images/full/2d7348b6561c9329f8f8841dc929738ada17e95c.jpg']}, u'_id': u'C0BABA15173709DA7A56207670578312CA30A7E94EDBF68A8140D8D5AA3C20FC'}), (u'6687736C0D7E5702DFBA1B94EC8F75E17CE956C5649C4354F5CDD828AEB226D6', u'https://s3.amazonaws.com/memex-images/full/229d071eafac66b81b47c7e7da041c6c91fc9ecf.jpg', {u'_type': u'escorts', u'_timestamp': 1464761131973, u'_index': u'memex-domains_2016.01', u'_score': 0.0, u'fields': {u'obj_parent': [u'5B193E4B99B3C64CA403D0EF4231D8E58554360103DBF069E09C9308B12FEFFC'], u'timestamp': [u'2016-01-26T13:15:18'], u'obj_original_url': [u'http://img7.asexyservice.com/Z/A/200/56a6f114be85a8267wNaP8K32cfsGA_emZr_DrkTMg1mZC84AZ_200.jpg'], u'obj_stored_url': [u'https://s3.amazonaws.com/memex-images/full/229d071eafac66b81b47c7e7da041c6c91fc9ecf.jpg']}, u'_id': u'6687736C0D7E5702DFBA1B94EC8F75E17CE956C5649C4354F5CDD828AEB226D6'}), (u'70520066B7DB04EF9B01858AFC652E973A2098B6039DBDB014063696B0480B82', u'https://s3.amazonaws.com/roxyimages/9bb2b24ccc451467da2e7ae89782b2d9ede6b0c0.jpg', {u'_type': u'escorts', u'_timestamp': 1464761132012, u'_index': u'memex-domains_2016.01', u'_score': 0.0, u'fields': {u'obj_parent': [u'CADCDF5FE92191882A3786FC2A80D3C55F4429F8C35DAA2BB68EC0554EDEEBB2'], u'timestamp': [u'2015-11-17T06:36:19'], u'obj_original_url': [u'http://images2.backpage.com/imager/u/large/222290798/8fd6fdf9abf273e7b235105742f54403.jpg'], u'obj_stored_url': [u'https://s3.amazonaws.com/roxyimages/9bb2b24ccc451467da2e7ae89782b2d9ede6b0c0.jpg']}, u'_id': u'70520066B7DB04EF9B01858AFC652E973A2098B6039DBDB014063696B0480B82'})]
readable_images = FD.download_images(batch)
print readable_images
from bs4 import BeautifulSoup
from requests.exceptions import RequestException
import requests
from file_downloader import FileDownloader
fileDownLoader = FileDownloader()
for page_index in range(1, 110):
with open('io-data/log.txt', 'a') as log_file:
try:
with open('io-data/t-shirt-links.txt', 'a') as t_shirt_links:
t_shirt_links.write(
'Starting to process page {0}\n'.format(page_index))
headers = {
'user-agent':
'user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36'
}
try:
pages_with_data = requests.get(
'https://www.gearbest.com/men-s-t-shirts-c_11926/{0}.html?page_size=120'
.format(str(page_index)),
headers=headers)
# r = requests.get('https://www.threadless.com/catalog/type,guys/style,tees/page,{0}'.format(page_index))
except RequestException as e:
print('Error in page {0}'.format(str(page_index)))
links_parser = BeautifulSoup(pages_with_data.text)
results = links_parser.find_all('a',
{'class', 'gbGoodsItem_thumb'})
def read_excel_file(self):
workbook = load_workbook(self.file_dir +
'GBP-SBP-resource-centre-250917.xlsx')
first_sheet = workbook.get_sheet_names()[0]
worksheet = workbook.get_sheet_by_name(first_sheet)
for i, row in enumerate(worksheet.iter_rows()):
if i == 0:
continue
if self.es.check_document_exists(index="green_bond",
doc="report",
id=i):
print "Document " + str(i) + " exists!"
doc = self.es.find_document(index="green_bond",
doc="report",
id=i)
if row[4].hyperlink is not None and row[
4].internal_value == doc["_source"][
"external_review_report"]:
try:
link = row[4].hyperlink.target
file_downloader = FileDownloader()
file_downloader.download_file(link)
text = self.read_pdf_file(self.file_dir +
"temp_file.pdf")
gbp = GBP(row[0].internal_value, row[1].internal_value,
row[2].internal_value, row[3].internal_value,
text, row[5].internal_value)
self.es.update_document(index="green_bond",
doc="report",
id=i,
data=json.dumps(gbp.__dict__))
print "Update document " + str(i) + " succesfully!"
except:
print "Hyperlink error! Document " + str(i)
pass
if row[3].hyperlink is not None and row[
3].internal_value == doc["_source"][
"external_review_form"]:
try:
link = row[3].hyperlink.target
file_downloader = FileDownloader()
file_downloader.download_file(link)
text = self.read_pdf_file(self.file_dir +
"temp_file.pdf")
gbp = GBP(
doc["_source"]["green_bond_issuer"],
doc["_source"]["country"],
doc["_source"]["market_information_template"],
text, doc["_source"]["external_review_report"],
doc["_source"]["external_link"])
self.es.update_document(index="green_bond",
doc="report",
id=i,
data=json.dumps(gbp.__dict__))
print "Update document " + str(i) + " succesfully!"
except:
print "Hyperlink error! Document " + str(i)
pass
continue
if row[4].hyperlink is None:
gbp = GBP(row[0].internal_value, row[1].internal_value,
row[2].internal_value, row[3].internal_value,
row[4].internal_value, row[5].internal_value)
self.es.add_document(i, json.dumps(gbp.__dict__))
print "Adding document " + str(i) + " succesfully!"
else:
try:
link = row[4].hyperlink.target
file_downloader = FileDownloader()
file_downloader.download_file(link)
self.text = self.read_pdf_file(self.file_dir +
"temp_file.pdf")
gbp = GBP(row[0].internal_value, row[1].internal_value,
row[2].internal_value, row[3].internal_value,
self.text, row[5].internal_value)
self.es.add_document(i, json.dumps(gbp.__dict__))
print "Adding document " + str(i) + " succesfully!"
except:
print "Hyperlink error! Document " + str(i)
pass
class App(tk.Tk):
bot = None
driver = None
meta_data = None
download_queue_video = None
download_queue_captions = None
external_exercise_queue = None
file_downloader = None
def __init__(self, title, *args, **kwargs):
tk.Tk.__init__(self, *args, **kwargs)
self.title(title)
# Adjust size
# self.geometry("400x200")
self.maxsize(800, 600)
self.minsize(800, 470)
# Create left and right frames
top_frame = self.createTopFrame()
bottom_frame = self.createBottomFrame()
# Create frames on top frame
self.createInputFrame(top_frame)
self.createOutputFrame(top_frame)
# Create frames on right frame
# self.createStatusFrame(bottom_frame)
self.createFileDownloaderFrame(bottom_frame)
###################################################################################################################
"""" Frame Creation Functions """
###################################################################################################################
def createTopFrame(self):
top_frame = tk.Frame(self, width=650, height=200, bg='grey')
top_frame.grid(row=0,
column=0,
padx=10,
pady=5,
sticky=tk.N + tk.W + tk.E)
return top_frame
def createBottomFrame(self):
bottom_frame = tk.Frame(self, width=650, height=400, bg='grey')
bottom_frame.grid(row=1,
column=0,
padx=10,
pady=5,
sticky=tk.N + tk.W + tk.E)
return bottom_frame
def createInputFrame(self, root):
labelframe = ttk.LabelFrame(root, text="Input", width=600, height=300)
labelframe.pack(fill="x", expand="yes")
course_link_label = tk.Label(labelframe, text="Course Link:")
self.course_link_entry = ttk.Entry(labelframe, width=100)
self.load_button = ttk.Button(labelframe, text="Load", width=8)
self.load_button.config(command=self.loadButtonAction)
self.input_status_label = tk.Label(labelframe, text="", fg="green")
# self.get_video_check_var = tk.IntVar(value=1)
# self.get_reading_check_var = tk.IntVar(value=1)
# self.get_quiz_check_var = tk.IntVar(value=1)
# self.get_graded_check_var = tk.IntVar(value=1)
# checkboxes_frame = tk.Frame(labelframe)
# get_video_check_box = tk.Checkbutton(checkboxes_frame, text="Video", variable=self.get_video_check_var)
# get_reading_check_box = tk.Checkbutton(checkboxes_frame, text="Reading", variable=self.get_reading_check_var)
# get_quiz_check_box = tk.Checkbutton(checkboxes_frame, text="Quiz", variable=self.get_quiz_check_var)
# get_graded_check_box = tk.Checkbutton(checkboxes_frame, text="Peer Graded Assignment", variable=self.get_graded_check_var)
padding_y = 3
course_link_label.grid(row=0,
column=0,
padx=5,
pady=padding_y + 2,
sticky=tk.S + tk.N)
self.course_link_entry.grid(row=0,
column=1,
padx=5,
pady=padding_y + 2,
sticky=tk.S + tk.N)
self.load_button.grid(row=0,
column=2,
padx=5,
pady=padding_y + 2,
sticky=tk.S + tk.N)
self.input_status_label.grid(row=1,
column=1,
padx=5,
pady=padding_y - 3,
sticky=tk.W + tk.S + tk.N)
# Checkboxes Frame
# checkboxes_frame.grid(row=1, column=1, pady=padding_y + 2, sticky=tk.S + tk.N + tk.W)
# get_video_check_box.grid(row=0, column=0, padx=5, sticky=tk.S + tk.N + tk.W)
# get_reading_check_box.grid(row=0, column=1, padx=5, sticky=tk.S + tk.N + tk.W)
# get_quiz_check_box.grid(row=0, column=2, padx=5, sticky=tk.S + tk.N + tk.W)
# get_graded_check_box.grid(row=0, column=3, padx=5, sticky=tk.S + tk.N + tk.W)
def createOutputFrame(self, root):
labelframe = ttk.LabelFrame(root, text="Output", width=600, height=300)
labelframe.pack(fill="x", expand="yes")
output_folder_label = tk.Label(labelframe, text="Output:")
self.output_folder_var = tk.StringVar()
output_folder_entry = ttk.Entry(labelframe,
textvariable=self.output_folder_var,
width=100)
def browseFolder():
folder_selected = filedialog.askdirectory()
if folder_selected != "":
self.output_folder_var.set(folder_selected)
browse_button = ttk.Button(labelframe,
text="Browse",
width=8,
command=browseFolder)
# browse_button.config(command=browseFolder)
self.get_video_check_var = tk.IntVar(value=1)
self.get_reading_check_var = tk.IntVar(value=1)
self.get_quiz_check_var = tk.IntVar(value=1)
self.get_graded_check_var = tk.IntVar(value=1)
self.get_external_check_var = tk.IntVar(value=1)
checkboxes_frame = tk.Frame(labelframe)
get_video_check_box = ttk.Checkbutton(
checkboxes_frame, text="Video", variable=self.get_video_check_var)
get_reading_check_box = ttk.Checkbutton(
checkboxes_frame,
text="Reading",
variable=self.get_reading_check_var)
get_quiz_check_box = ttk.Checkbutton(checkboxes_frame,
text="Quiz",
variable=self.get_quiz_check_var)
get_graded_check_box = ttk.Checkbutton(
checkboxes_frame,
text="Peer Graded",
variable=self.get_graded_check_var)
get_external_check_box = ttk.Checkbutton(
checkboxes_frame,
text="External Exercise",
variable=self.get_external_check_var)
# Button Group
buttons_frame = tk.Frame(labelframe)
self.download_all_button = ttk.Button(
buttons_frame,
text="Start Download",
command=self.downloadAllButtonAction)
# Menu Button
self.more_menu_button = ttk.Menubutton(buttons_frame, text='More')
self.more_menu_button.menu = tk.Menu(self.more_menu_button, tearoff=0)
self.more_menu_button['menu'] = self.more_menu_button.menu
self.more_menu_button.menu.add_command(label="Scrape",
command=self.scrapeButtonAction)
self.more_menu_button.menu.add_command(
label="Download Video", command=self.downloadVideoButtonAction)
self.more_menu_button.menu.add_command(
label="Download External Exercise",
state=tk.DISABLED,
command=self.downloadExternalButtonAction)
self.more_menu_button.menu.add_command(
label="Download Resource",
command=self.downloadResourceButtonAction)
self.more_menu_button.menu.add_command(
label="Download Images", command=self.downloadImageButtonAction)
self.more_menu_button.menu.add_command(
label="Download Attachments",
command=self.downloadAttachmentButtonAction)
# self.scrape_button = tk.Button(buttons_frame, text="Scrape", state=tk.DISABLED, width=8, command=self.scrapeButtonAction)
# self.download_video_button = tk.Button(buttons_frame, text="Download Video", state=tk.DISABLED, command=self.downloadVideoButtonAction)
# self.download_external_button = tk.Button(buttons_frame, text="External Exercise", state=tk.DISABLED, command=self.downloadExternalButtonAction)
# self.download_resource_button = tk.Button(buttons_frame, text="Resource", command=self.downloadResourceButtonAction)
# self.download_image_button = tk.Button(buttons_frame, text="Images", command=self.downloadImageButtonAction)
# self.download_attachment_button = tk.Button(buttons_frame, text="Attachments", command=self.downloadAttachmentButtonAction)
# Status
self.output_status_label = tk.Label(labelframe, text="", fg="green")
padding_y = 3
output_folder_label.grid(row=0,
column=0,
padx=5,
pady=padding_y + 2,
sticky=tk.S + tk.N)
output_folder_entry.grid(row=0,
column=1,
padx=5,
pady=padding_y + 2,
sticky=tk.S + tk.N)
browse_button.grid(row=0,
column=2,
ipadx=5,
padx=5,
pady=padding_y + 2,
sticky=tk.S + tk.N)
checkboxes_frame.grid(row=1,
column=1,
pady=padding_y + 2,
sticky=tk.S + tk.N + tk.W)
buttons_frame.grid(row=2,
column=1,
padx=5,
pady=padding_y,
sticky=tk.W)
self.output_status_label.grid(row=3,
column=1,
padx=5,
pady=0,
sticky=tk.W)
# Checkboxes Frame
get_video_check_box.grid(row=0,
column=0,
padx=5,
sticky=tk.S + tk.N + tk.W)
get_reading_check_box.grid(row=0,
column=1,
padx=5,
sticky=tk.S + tk.N + tk.W)
get_quiz_check_box.grid(row=0,
column=2,
padx=5,
sticky=tk.S + tk.N + tk.W)
get_graded_check_box.grid(row=0,
column=3,
padx=5,
sticky=tk.S + tk.N + tk.W)
get_external_check_box.grid(row=0,
column=4,
padx=5,
sticky=tk.S + tk.N + tk.W)
# Buttons Frame
self.download_all_button.grid(row=0,
column=0,
ipadx="5",
padx=5,
pady=padding_y,
sticky=tk.W)
self.more_menu_button.grid(row=0,
column=1,
padx=5,
pady=padding_y,
sticky=tk.W)
# self.scrape_button.grid(row=0, column=1, padx=5, pady=padding_y, sticky=tk.N)
# self.download_video_button.grid(row=0, column=2, padx=5, pady=padding_y, sticky=tk.W)
# self.download_external_button.grid(row=0, column=3, padx=5, pady=padding_y, sticky=tk.W)
# self.download_resource_button.grid(row=0, column=4, padx=5, pady=padding_y, sticky=tk.W)
# self.download_image_button.grid(row=0, column=5, padx=5, pady=padding_y, sticky=tk.W)
# self.download_attachment_button.grid(row=0, column=6, padx=5, pady=padding_y, sticky=tk.W)
def createStatusFrame(self, root):
labelframe = ttk.LabelFrame(root, text="Status", width=600, height=100)
labelframe.pack(fill="x", expand="yes")
self.status_label_line_1 = tk.Label(labelframe, text="Downloading")
self.status_label_line_2 = tk.Label(labelframe, text="Data")
padding_y = 0
self.status_label_line_1.grid(row=0,
column=0,
padx=5,
pady=padding_y,
sticky=tk.W)
self.status_label_line_2.grid(row=1,
column=0,
padx=5,
pady=padding_y,
sticky=tk.W)
def createFileDownloaderFrame(self, root):
labelframe = ttk.LabelFrame(root,
text="File Downloader",
width=600,
height=500)
labelframe.pack(fill="x", expand="yes")
self.week_label = tk.Label(labelframe, text="", fg="green")
self.topic_label = tk.Label(labelframe, text="", fg="green")
filename_label = tk.Label(labelframe, text="Filename: ")
self.filename_val_label = tk.Label(labelframe, text="")
url_label = tk.Label(labelframe, text="Url: ")
self.url_val_label = tk.Label(labelframe, text="")
output_label = tk.Label(labelframe, text="Output: ")
self.output_val_label = tk.Label(labelframe, text="")
downloaded_label = tk.Label(labelframe, text="Downloaded: ")
self.downloaded_val_label = tk.Label(labelframe, text="")
size_label = tk.Label(labelframe, text="Size: ")
self.size_val_label = tk.Label(labelframe, text="")
speed_label = tk.Label(labelframe, text="Speed: ")
self.speed_val_label = tk.Label(labelframe, text="")
time_label = tk.Label(labelframe, text="Time left: ")
self.time_val_label = tk.Label(labelframe, text="")
total_label = tk.Label(labelframe, text="Total: ")
self.total_val_label = tk.Label(labelframe, text="")
# Progress bar widget
self.progress_bar = ttk.Progressbar(labelframe,
length=100,
orient=tk.HORIZONTAL,
mode='determinate')
self.total_progress_bar = ttk.Progressbar(labelframe,
length=100,
orient=tk.HORIZONTAL,
mode='determinate')
# Status Bar
self.status_bar_label = tk.Label(labelframe, text="Ready")
# Button Group
button_group_frame = tk.Frame(labelframe)
self.pause_resume_btn = ttk.Button(
button_group_frame,
text="Pause",
width=8,
state=tk.DISABLED,
command=self.pauseDownloadButtonAction)
self.skip_btn = ttk.Button(button_group_frame,
text="Skip",
width=8,
state=tk.DISABLED,
command=self.skipDownloadButtonAction)
self.cancel_btn = ttk.Button(button_group_frame,
text="Cancel",
width=8,
state=tk.DISABLED,
command=self.cancelDownloadButtonAction)
padding_y = 0
self.week_label.grid(row=0,
column=0,
padx=5,
pady=padding_y,
sticky=tk.W)
self.topic_label.grid(row=0,
column=1,
columnspan=3,
padx=5,
pady=padding_y,
sticky=tk.W)
filename_label.grid(row=1,
column=0,
padx=5,
pady=padding_y,
sticky=tk.W)
self.filename_val_label.grid(row=1,
column=1,
columnspan=3,
padx=5,
pady=padding_y,
sticky=tk.W)
url_label.grid(row=2, column=0, padx=5, pady=padding_y, sticky=tk.W)
self.url_val_label.grid(row=2,
column=1,
columnspan=3,
padx=5,
pady=padding_y,
sticky=tk.W)
output_label.grid(row=3, column=0, padx=5, pady=padding_y, sticky=tk.W)
self.output_val_label.grid(row=3,
column=1,
columnspan=3,
padx=5,
pady=padding_y,
sticky=tk.W)
downloaded_label.grid(row=4,
column=0,
padx=5,
pady=padding_y,
sticky=tk.W)
self.downloaded_val_label.grid(row=4,
column=1,
padx=5,
pady=padding_y,
sticky=tk.W)
size_label.grid(row=4, column=2, padx=5, pady=padding_y, sticky=tk.W)
self.size_val_label.grid(row=4,
column=3,
padx=5,
pady=padding_y,
sticky=tk.W)
speed_label.grid(row=5, column=0, padx=5, pady=padding_y, sticky=tk.W)
self.speed_val_label.grid(row=5,
column=1,
padx=5,
pady=padding_y,
sticky=tk.W)
time_label.grid(row=5, column=2, padx=5, pady=padding_y, sticky=tk.W)
self.time_val_label.grid(row=5,
column=3,
padx=5,
pady=padding_y,
sticky=tk.W)
self.progress_bar.grid(row=6,
column=0,
columnspan=4,
padx=5,
pady=padding_y + 8,
sticky=tk.W + tk.E)
total_label.grid(row=7, column=0, padx=5, pady=padding_y, sticky=tk.W)
self.total_val_label.grid(row=7,
column=1,
padx=5,
pady=padding_y,
sticky=tk.W)
self.total_progress_bar.grid(row=8,
column=0,
columnspan=4,
padx=5,
pady=padding_y + 3,
sticky=tk.W + tk.E)
self.status_bar_label.grid(row=9,
column=0,
columnspan=3,
padx=5,
pady=padding_y + 3,
sticky=tk.W)
# Button Group Frame
button_group_frame.grid(row=9,
column=3,
padx=5,
pady=padding_y + 3,
sticky=tk.E)
self.pause_resume_btn.grid(row=0,
column=0,
ipadx="5",
padx=5,
pady=padding_y,
sticky=tk.E)
self.skip_btn.grid(row=0,
column=1,
ipadx="5",
padx=5,
pady=padding_y,
sticky=tk.E)
self.cancel_btn.grid(row=0,
column=2,
ipadx="5",
padx=5,
pady=padding_y,
sticky=tk.E)
labelframe.columnconfigure(1, weight=1)
labelframe.columnconfigure(3, weight=2)
###################################################################################################################
"""" Button Handler Functions """
###################################################################################################################
def loadButtonAction(self):
course_link = self.getCourseLink()
if course_link == "":
messagebox.showinfo(title="Information",
message="Please enter course link")
return
if self.driver == None:
self.driver = Driver("main")
if self.bot == None:
self.bot = Bot(self.driver, start_week=1, gui=self)
self.bot.setCourseUrl(course_link)
download_topics = self.getDownloadTopics()
self.bot.setDownloadTopics(download_topics)
Thread(target=self.runLoadMetaThread).start()
def scrapeButtonAction(self):
output_folder = self.getOutputFolder()
if output_folder == "":
messagebox.showinfo(title="Information",
message="Please choose output folder")
return
if self.bot:
self.bot.setOutputRoot(output_folder)
else:
messagebox.showinfo(title="Information", message="Bot not loaded!")
download_topics = self.getDownloadTopics()
self.bot.setDownloadTopics(download_topics)
if self.meta_data:
Thread(target=self.runDownloadHtmlAndGetVideoQueue).start()
else:
messagebox.showinfo(
title="Information",
message="Meta Data not loaded. Please load again")
def downloadVideoButtonAction(self):
if self.getOutputFolder() == "":
messagebox.showinfo(title="Information",
message="Please choose output folder")
return
if not self.download_queue_video:
messagebox.showinfo(title="Information",
message="Download Queue is Empty!")
return
Thread(target=self.runVideoDownloader).start()
def downloadExternalButtonAction(self):
output_folder = self.getOutputFolder()
if output_folder == "":
messagebox.showinfo(title="Information",
message="Please choose output folder")
return
if not self.external_exercise_queue:
messagebox.showinfo(title="Information",
message="External Exercise Queue is Empty!")
return
Thread(target=self.runExternalExerciseDownloader).start()
def downloadImageButtonAction(self):
output_folder = self.getOutputFolder()
if output_folder == "":
messagebox.showinfo(title="Information",
message="Please choose output folder")
return
if self.bot == None:
self.bot = Bot(self.driver, gui=self)
self.bot.setOutputRoot(output_folder)
Thread(target=self.runImageDownloader).start()
def downloadAttachmentButtonAction(self):
output_folder = self.getOutputFolder()
if output_folder == "":
messagebox.showinfo(title="Information",
message="Please choose output folder")
return
if self.bot == None:
self.bot = Bot(self.driver, gui=self)
self.bot.setOutputRoot(output_folder)
Thread(target=self.runAttachmentDownloader).start()
def downloadResourceButtonAction(self):
course_link = self.getCourseLink()
if course_link == "":
messagebox.showinfo(title="Information",
message="Please enter course link")
return
output_folder = self.getOutputFolder()
if output_folder == "":
messagebox.showinfo(title="Information",
message="Please choose output folder")
return
if self.driver == None:
self.driver = Driver("main")
if self.bot == None:
self.bot = Bot(self.driver, course_url=course_link, gui=self)
self.bot.setOutputRoot(output_folder)
# self.disableSkipButton(False)
# self.disablePauseButton(False)
# self.disableCancelButton(False)
# self.disableDownloadVideoButton(True)
Thread(target=self.runResourceDownloader).start()
# self.downloadStatusLoop()
def downloadAllButtonAction(self):
course_link = self.getCourseLink()
if course_link == "":
messagebox.showinfo(title="Information",
message="Please enter course link")
return
output_folder = self.getOutputFolder()
if output_folder == "":
messagebox.showinfo(title="Information",
message="Please choose output folder")
return
if self.driver == None:
self.driver = Driver("main")
if self.bot == None:
self.bot = Bot(self.driver, start_week=1, gui=self)
self.bot.setCourseUrl(course_link)
self.bot.setOutputRoot(output_folder)
download_topics = self.getDownloadTopics()
self.bot.setDownloadTopics(download_topics)
if self.bot == None:
self.bot = Bot(self.driver, gui=self)
self.bot.setOutputRoot(output_folder)
Thread(target=self.runAllDownloader).start()
def pauseDownloadButtonAction(self):
if self.file_downloader:
if self.file_downloader.pause():
self.pause_resume_btn.config(text="Resume")
self.pause_resume_btn.config(
command=self.resumeDownloadButtonAction)
def resumeDownloadButtonAction(self):
if self.file_downloader:
if self.file_downloader.resume():
self.pause_resume_btn.config(text="Pause")
self.pause_resume_btn.config(
command=self.pauseDownloadButtonAction)
def skipDownloadButtonAction(self):
if self.file_downloader:
self.file_downloader.stop()
def cancelDownloadButtonAction(self):
if self.file_downloader:
if self.file_downloader.stopQueue():
self.disablePauseButton(True)
self.disableCancelButton(True)
self.disableSkipButton(True)
self.resetFileDownloaderInfo()
self.disableDownloadVideoButton(False)
###################################################################################################################
"""" Getter Functions """
###################################################################################################################
def getCourseLink(self):
return self.course_link_entry.get()
def getOutputFolder(self):
return os.path.normpath(self.output_folder_var.get()
) if self.output_folder_var.get() != "" else ""
def getDownloadTopics(self):
get_video = True if self.get_video_check_var.get() else False
get_reading = True if self.get_reading_check_var.get() else False
get_quiz = True if self.get_quiz_check_var.get() else False
get_graded_assignment = True if self.get_graded_check_var.get(
) else False
get_external_exercise = True if self.get_external_check_var.get(
) else False
return get_video, get_reading, get_quiz, get_graded_assignment, get_external_exercise
###################################################################################################################
"""" Thread Functions """
###################################################################################################################
def runLoadMetaThread(self, silent=False):
# Disable Buttons
self.disableIOButtons()
self.disableDownloadButtons()
self.setInputStatus("Loading coursera...", color="grey")
self.meta_data = self.bot.loadMeta()
if self.meta_data:
print(self.meta_data)
item_count = 0
for week in self.meta_data:
for topic in week['topics']:
item_count += len(topic['items'])
self.setInputStatus("Loaded {} weeks with {} topics..".format(
len(self.meta_data), item_count))
if not silent:
messagebox.showinfo(title="Information",
message="Meta data loaded!")
# Enable Buttons
self.disableIOButtons(False)
return True
else:
messagebox.showwarning("Warning", "Canceled")
# Enable Buttons
self.disableIOButtons(False)
return False
def runDownloadHtmlAndGetVideoQueue(self, silent=False):
self.setOutputStatus("Scraping...", color="blue")
# Disable Buttons
self.disableIOButtons()
self.disableDownloadButtons()
result = self.bot.downloadHtmlAndGetVideoQueue(self.meta_data)
self.download_queue_video, self.download_queue_captions, self.external_exercise_queue = result
print(self.download_queue_captions)
print(self.download_queue_video)
self.setOutputStatus(
"Downloaded HTMLs & Loaded {} video(s), {} external exercise in the queue"
.format(len(self.download_queue_video),
len(self.external_exercise_queue)),
color="green")
# Enable Buttons
self.disableIOButtons(False)
if not silent:
messagebox.showinfo(
title="Information",
message="HTML Downloaded and Video Download Queue Generated")
def runVideoDownloader(self, silent=False):
self.downloadCaptions()
self.downloadVideos()
if not silent:
self.showMessageDialog("Video Download Complete!")
self.setFileDownloaderText(week="Complete",
topic="Video Download Complete!",
color="green")
self.disableIOButtons(False)
self.disableDownloadButtons()
utils.log("Video Download Complete!")
def runResourceDownloader(self, silent=False):
# Disable Buttons
self.disableIOButtons()
self.disableDownloadButtons()
self.bot.downloadResources()
if not silent:
messagebox.showinfo(title="Information",
message="Resource Download Complete!")
# Enable Buttons
self.disableIOButtons(False)
def runExternalExerciseDownloader(self, silent=False):
# Disable Buttons
self.disableIOButtons()
self.disableDownloadButtons()
self.bot.downloadExternalExercise()
if not silent:
messagebox.showinfo(title="Information",
message="External Exercise Download Complete!")
# Enable Buttons
self.disableIOButtons(False)
def runImageDownloader(self, silent=False):
# Disable Buttons
self.disableIOButtons()
self.disableDownloadButtons()
self.bot.downloadImages()
if not silent:
messagebox.showinfo(title="Information",
message="Image(s) Download Complete!")
# Enable Buttons
self.disableIOButtons(False)
def runAttachmentDownloader(self, silent=False):
# Disable Buttons
self.disableIOButtons()
self.disableDownloadButtons()
self.bot.downloadAttachments()
if not silent:
messagebox.showinfo(title="Information",
message="Attachment(s) Download Complete!")
# Enable Buttons
self.disableIOButtons(False)
def runAllDownloader(self):
# Disable Buttons
self.disableIOButtons()
self.disableDownloadButtons()
total_steps = 8
current_step = 1
self.setStatusBarText("Step {} / {} : Loading metadata".format(
current_step, total_steps),
color="blue")
meta_downloaded = self.runLoadMetaThread(silent=True)
if not meta_downloaded:
self.setStatusBarText("Ready")
return
current_step += 1
self.setStatusBarText("Step {} / {} : Scrapping".format(
current_step, total_steps),
color="blue")
self.runDownloadHtmlAndGetVideoQueue(silent=True)
current_step += 1
self.setStatusBarText("Step {} / {} : Downloading Resource(s)".format(
current_step, total_steps),
color="blue")
self.runResourceDownloader(silent=True)
current_step += 1
self.setStatusBarText(
"Step {} / {} : Downloading External Exercise(s)".format(
current_step, total_steps),
color="blue")
self.runExternalExerciseDownloader(silent=True)
current_step += 1
self.setStatusBarText("Step {} / {} : Downloading Image(s)".format(
current_step, total_steps),
color="blue")
self.runImageDownloader(silent=True)
current_step += 1
self.setStatusBarText(
"Step {} / {} : Downloading Attachment(s)".format(
current_step, total_steps),
color="blue")
self.runAttachmentDownloader(silent=True)
current_step += 1
self.setStatusBarText("Step {} / {} : Downloading Caption(s)".format(
current_step, total_steps),
color="blue")
if not self.download_queue_video:
print("Video Download Queue is Empty!")
else:
self.downloadCaptions()
current_step += 1
self.setStatusBarText("Step {} / {} : Downloading Video(s)".format(
current_step, total_steps),
color="blue")
self.downloadVideos()
self.setStatusBarText("Ready")
self.showMessageDialog("All Download Complete!")
self.setFileDownloaderText(week="Complete",
topic="All Download Complete!",
color="green")
self.disableIOButtons(False)
utils.log("All Download Complete!")
# Other Thread Functions
def downloadCaptions(self):
# Disable Buttons
self.disableIOButtons()
self.disableDownloadButtons(False)
root = self.getOutputFolder()
if self.file_downloader == None:
self.file_downloader = FileDownloader(root)
self.file_downloader.attachGUI(self)
self.file_downloader.loadQueueFromList(self.download_queue_captions)
self.file_downloader.startDownloadGui()
def downloadVideos(self):
# Download Videos
self.file_downloader.loadQueueFromList(self.download_queue_video)
self.file_downloader.startDownloadGui()
###################################################################################################################
"""" Looper Functions """
###################################################################################################################
def downloadStatusLoop(self):
if self.file_downloader:
total_files, self.current_download_no, self.current_download_item, download_info = self.file_downloader.getDownloadInfo(
)
if download_info:
self.progress_bar['value'] = download_info['progress']
# self.total_progress_bar['value'] = 1
self.after(100, self.downloadStatusLoop)
###################################################################################################################
"""" GUI Functions """
###################################################################################################################
def setScrapProgress(self, download_info):
item = download_info['item']
progress = download_info['progress']
current_no = download_info['current_no']
total_files = download_info['total_files']
week = item['path'].split("\\")[0]
topic_text = "Topic: {}".format(item['path'].split("\\")[1])
filename = item['filename']
url = item['url']
output = download_info['full_path']
eta = download_info['eta']
speed = download_info['speed']
dl_size = download_info['dl_size']
file_size = download_info['total_size']
self.setFileDownloaderInfo(week=week,
topic=topic_text,
filename=filename,
url=url,
output=output,
eta=eta,
speed=speed,
dl_size=dl_size,
file_size=file_size,
progress=progress,
current_no=current_no,
total_files=total_files)
def setVideoDownloadProgress(self, download_info):
item = download_info['item']
progress = download_info['progress']
current_no = download_info['current_no']
total_files = download_info['total_files']
week = item['path'].split("\\")[0]
topic_text = "Topic: {}".format(item['path'].split("\\")[1])
filename = item['filename']
url = item['url']
output = download_info['full_path']
eta = download_info['eta']
speed = download_info['speed']
dl_size = download_info['dl_size']
file_size = download_info['total_size']
self.setFileDownloaderInfo(week=week,
topic=topic_text,
filename=filename,
url=url,
output=output,
eta=eta,
speed=speed,
dl_size=dl_size,
file_size=file_size,
progress=progress,
current_no=current_no,
total_files=total_files)
def setFileDownloaderInfo(self,
week=None,
topic=None,
filename=None,
url=None,
output=None,
eta=None,
speed=None,
dl_size=None,
file_size=None,
progress=None,
current_no=0,
total_files=0):
total_text = ""
total_progress = 0
if current_no > 0 and total_files > 0:
prev_progress = (current_no - 1) / total_files * 100
single_progress = 1 / total_files * 100
total_progress = prev_progress + (single_progress * progress / 100)
total_text = "{} of {} ( {:.2f} % )".format(
current_no, total_files, total_progress)
if week is not None: self.week_label.config(text=week, fg="blue")
if topic is not None:
self.topic_label.config(text=self.fitText(topic, 70), fg="blue")
if filename is not None:
self.filename_val_label.config(text=self.fitText(filename))
if url is not None: self.url_val_label.config(text=self.fitText(url))
if output is not None:
self.output_val_label.config(text=self.fitText(output))
if eta is not None: self.time_val_label.config(text=eta)
if speed is not None: self.speed_val_label.config(text=speed)
if dl_size is not None:
downloaded_text = "{} ( {:.2f} % )".format(
dl_size, progress) if dl_size != "" else ""
self.downloaded_val_label.config(text=downloaded_text)
if file_size is not None: self.size_val_label.config(text=file_size)
if total_text is not None: self.total_val_label.config(text=total_text)
if progress is not None: self.progress_bar['value'] = progress
if total_progress is not None:
self.total_progress_bar['value'] = total_progress
def setFileDownloaderText(self, week=None, topic=None, color="black"):
if week is not None:
self.week_label.config(text=week, fg=color)
if topic is not None:
self.topic_label.config(text=self.fitText(topic, 70), fg=color)
def resetFileDownloaderInfo(self):
self.setFileDownloaderInfo(week="",
topic="",
filename="",
url="",
output="",
eta="",
speed="",
dl_size="",
file_size="",
progress=0,
current_no=0,
total_files=0)
def setStatusBarText(self, text, color="black"):
self.status_bar_label.config(text=text, fg=color)
# Button Disable
def disableButton(self, button, val=True):
if val:
button.config(state="disabled")
else:
button.config(state="normal")
def disableButtons(self, *argv, val=True):
for button in argv:
if val:
button.config(state="disabled")
else:
button.config(state="normal")
def disableDownloadVideoButton(self, val):
if val:
self.download_video_button.config(state="disabled")
else:
self.download_video_button.config(state="normal")
def disablePauseButton(self, val):
if val:
self.pause_resume_btn.config(state="disabled")
else:
self.pause_resume_btn.config(state="normal")
def disableSkipButton(self, val):
if val:
self.skip_btn.config(state="disabled")
else:
self.skip_btn.config(state="normal")
def disableCancelButton(self, val):
if val:
self.cancel_btn.config(state="disabled")
else:
self.cancel_btn.config(state="normal")
def disableIOButtons(self, val=True):
self.disableButtons(self.load_button,
self.download_all_button,
self.more_menu_button,
val=val)
def disableDownloadButtons(self, val=True):
self.disableButtons(self.pause_resume_btn,
self.skip_btn,
self.cancel_btn,
val=val)
# Show Dialogs
def showDownloadCompleteDialog(self, msg=None):
if msg is None:
msg = "Download complete!"
messagebox.showinfo(title="Information", message=msg)
self.disableIOButtons(False)
self.disableDownloadButtons(True)
def showMessageDialog(self, msg, title="Information"):
messagebox.showinfo(title=title, message=msg)
def askOkCancelDialog(self, msg, title="Information"):
return messagebox.askokcancel(title=title, message=msg)
# Set Status
def setInputStatus(self, text, color="green"):
self.input_status_label.config(text=text, fg=color)
def setOutputStatus(self, text, color="green"):
self.output_status_label.config(text=text, fg=color)
###################################################################################################################
"""" Utility Functions """
###################################################################################################################
def fitText(self, text, width=100):
if text and width and len(text) > width:
text = text[:width - 3] + '...'
return text
###################################################################################################################
"""" Other Functions """
###################################################################################################################
def destroy(self):
super().destroy()
if self.driver is not None:
print("Closing Browser...")
self.driver.closeBrowser()
print("Closed Browser")
