def receive_file_info(self, cyftinfo): if cyftinfo.transfer_id in self.accepted_fts: ym.accept_file( cyftinfo.sender, cyftinfo.receiver, cyftinfo.filename, cyftinfo.transfer_id, cyftinfo.transfer_type, cyftinfo.host, cyftinfo.relay_id ) transfer_task = self.transfer_tasks[cyftinfo.transfer_id] finfo = QFileInfo(transfer_task.destination) if finfo.isDir(): path = '{0}/{1}'.format(transfer_task.destination, cyftinfo.filename) else: path = '{0}'.format(transfer_task.destination) # remove the current file from left files list and download it transfer_task.remove_file(cyftinfo.filename) transfer_task.paths.append(path) self._download_progress(cyftinfo.transfer_id, cyftinfo.filename, 0.0) self.downloader = FileDownloader(transfer_task, path) self.downloader.progress.connect(self._download_progress) self.downloader.finished.connect(self._download_finished) self.downloader.download(cyftinfo.get_download_url())
def run(dst_path, datasetconfig, downloadData, clean_dst=False, force_download=True, overwrite_current_date=None): try: print('\n> processing', datasetconfig["where_to_download"]["identifier"]) dst_dataset_path = os.path.join( dst_path, datasetconfig["where_to_download"]["identifier"]) if os.path.exists(dst_dataset_path) and clean_dst: shutil.rmtree(dst_dataset_path) # download print('File Downloads:') fs = FileDownloader(dst_dataset_path) data = fs.process(datasetconfig, downloadData, force=force_download, current_datetime=overwrite_current_date) print(data) except Exception as e: print('exception in run', e) exc_type, exc_value, exc_traceback = sys.exc_info() lines = traceback.format_exception(exc_type, exc_value, exc_traceback) print(''.join(lines))
def test_multi_thread_download(self): URL = 'http://olps.cgtransport.org/Esewa/img/ESEWA2.jpg' resource_size = 24963 with tempfile.TemporaryDirectory() as temp_dir: logger = self.create_logger('test_multi_thread_download') download_file = os.path.join(temp_dir, 'download_file') downloader = FileDownloader(URL, download_file, logger, 4) downloader.download() self.assertEqual(os.stat(download_file).st_size, resource_size)
def downloadCaptions(self): # Disable Buttons self.disableIOButtons() self.disableDownloadButtons(False) root = self.getOutputFolder() if self.file_downloader == None: self.file_downloader = FileDownloader(root) self.file_downloader.attachGUI(self) self.file_downloader.loadQueueFromList(self.download_queue_captions) self.file_downloader.startDownloadGui()
def test_setUp(self): FILE_NAME = 'temp_file' FILE_SIZE = 4 * 1000 * 1000 SPEED_LIMIT = 256 * 1000 * 1 HOST = "0.0.0.0" PORT = 10000 with tempfile.TemporaryDirectory() as temp_dir: logger = self.create_logger('test_simple_server_send') FILE_SIZE = self.create_dummy_file( FILE_SIZE, os.path.join(temp_dir, FILE_NAME)) self.assertTrue(os.path.exists(os.path.join(temp_dir, FILE_NAME))) server = FileServer(HOST, PORT, logger, temp_dir, SPEED_LIMIT) server.start() PORT = server.sock.getsockname()[1] URL = f"http://{HOST}:{PORT}/{FILE_NAME}" downloader_single_thread = FileDownloader(URL, "simple_download", logger) downloader_single_thread.download() downloader_multi_thread = FileDownloader(URL, "multi_thread_download", logger, threads=4) downloader_multi_thread.download() server.stop() self.assertEqual(os.stat("simple_download").st_size, FILE_SIZE) self.assertEqual( os.stat("multi_thread_download").st_size, FILE_SIZE)
def download_from_url(url, output_directory, filename=None, use_cache=True): """Download file from a url and put it under output_directory. :param url: Url that gives response. :type url: str :param output_directory: Directory to put the diagram. :type output_directory: str :param filename: Optional filename for downloaded file. :type filename: str :param use_cache: If there is a cached copy of the file already in the output directory, do not refetch it (True) or force refecth it (False). :type use_cache: bool :returns: File path if success to download, else None :rtype: str """ if filename is None: filename = get_filename(url) LOGGER.info('Download file %s from %s' % (filename, url)) file_path = os.path.join(output_directory, filename) if os.path.exists(file_path) and use_cache: LOGGER.info('File %s exists, not downloading' % file_path) return file_path # Set Proxy in webpage proxy = get_proxy() network_manager = QNetworkAccessManager() if not proxy is None: network_manager.setProxy(proxy) # Download Process # noinspection PyTypeChecker downloader = FileDownloader(network_manager, url, file_path) try: result = downloader.download() except IOError as ex: raise DownloadException(ex) if result[0] is not True: _, error_message = result raise DownloadException(error_message) if os.path.exists(file_path): return file_path else: return None
def download_from_url(url, output_directory, filename=None, use_cache=True): """Download file from a url and put it under output_directory. :param url: Url that gives response. :type url: str :param output_directory: Directory to put the diagram. :type output_directory: str :param filename: Optional filename for downloaded file. :type filename: str :param use_cache: If there is a cached copy of the file already in the output directory, do not refetch it (True) or force refecth it (False). :type use_cache: bool :returns: File path if success to download, else None :rtype: str """ if filename is None: filename = get_filename(url) LOGGER.info('Download file %s from %s' % (filename, url)) file_path = os.path.join(output_directory, filename) if os.path.exists(file_path) and use_cache: LOGGER.info('File %s exists, not downloading' % file_path) return file_path # Set Proxy in webpage proxy = get_proxy() network_manager = QNetworkAccessManager() if not proxy is None: network_manager.setProxy(proxy) # Download Process # noinspection PyTypeChecker downloader = FileDownloader(network_manager, url, file_path) try: result = downloader.download() except IOError as ex: raise DownloadException(ex) if result[0] is not True: _, error_message = result raise DownloadException(error_message) if os.path.exists(file_path): return file_path else: return None
def process_wget(input_data): data = validate_wget(input_data) if data: url = data[0] if data[1]: extension = data[1] # todo: а куда скачивать? file_downloader = FileDownloader(url, extension) file_downloader.download_files() else: # todo : download with url only pass # print(data) # print('op : ' + str(data[0])) # print('url : ' + data[1]) # print('extension : ' + str(data[2])) pass
def start_a_job(self): current_job = None for job in self.job_list: if current_job == None and not job["isQueued"] and not job[ "isFinished"]: self.current_active_jobz += 1 job["isQueued"] = True current_job = job print(f"Current Job is:{current_job}") if current_job != None: message = current_job["message"] downloader = FileDownloader(message, self) # Start the image process by downloading the image downloader.get_file_path()
from file_downloader import FileDownloader with open('io-data/download-logger.txt', 'a') as log_file: fileDownLoader = FileDownloader(file_with_urls='t-shirt-links.txt', logger=log_file) # fileDownLoader = FileDownloader(file_with_urls='1.txt') fileDownLoader.download_files()
import logging import argparse from file_downloader import FileDownloader #URL1 = 'http://olps.cgtransport.org/Esewa/img/ESEWA2.jpg' if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--url') parser.add_argument('--save_as', type=str, default='download_file') parser.add_argument('--threads', type=int, default=1) parser.add_argument('--verbose', '-v', action='store_true') args = parser.parse_args() if args.verbose: log_level = logging.DEBUG else: log_level = logging.WARNING logger = logging.getLogger('file_downloader_logger') logger.setLevel(log_level) console_handler = logging.StreamHandler() console_handler.setLevel(log_level) formatter = logging.Formatter('%(asctime)s - %(message)s') console_handler.setFormatter(formatter) logger.addHandler(console_handler) downloader = FileDownloader(args.url, args.save_as, logger, args.threads) downloader.download()
def main(self): ''' :return: ''' # Always do these steps regardless of option # Parse variables files to extract relevant sample information # Identify samples to upload to BaseSpace samples_to_upload = identify_samples() # Load and parse out variables from variables files associated with each sample all_variables = load_all_variables(samples_to_upload, os.getcwd()) # Identify the worksheet number which will be used as the project name in BaseSpace worksheet = identify_worksheet(all_variables) # Pair samples- DNA sample is key, RNA sample to look up- if No RNA sample, it is None sample_pairs = create_sample_pairs(all_variables) # Write out sample pairs to log file for checking if needed log.warning(f"sample pairs are {sample_pairs}") # Locate the fastqs associated with all samples all_fastqs = locate_all_fastqs(samples_to_upload, os.getcwd()) # Create a project in BaseSpace- will not create if it already exists, but will still return project id upload = FileUpload(self.authentication_token, worksheet, samples_to_upload, all_fastqs) project = upload.create_basespace_project() log.info(f"Project {worksheet} created") log.warning(f"Project id for project name {worksheet} is {project}") # If whole pipeline required then upload fastq files if not args.tst170 and not args.smp2 and not args.dl_files: # Upload fastq files print(f"uploading fastq files for all samples") upload.upload_files() # Create launch app object for TST170 app launch_tst = LaunchApp(self.authentication_token, worksheet, project, app_name, app_version, sample_pairs) # If resuming from TST170 required or full pipeline- launch the TST170 app if not args.smp2 and not args.dl_files: # Launch TST170 application for each pair in turn # IMPORTANT NOTE: Only processes paired data tst_170 = launch_tst.launch_tst170_pairs() # Dump data to file with open(os.path.join(os.getcwd(), "tst_170.json"), 'w') as t: json.dump(tst_170, t) # If resuming from SMP2v3 launch load in required TST170 data from file elif args.smp2: try: with open(os.path.join(os.getcwd(), "tst_170.json")) as ts: tst_170 = json.load(ts) except FileNotFoundError: raise FileNotFoundError( f"Could not find file tst_170.json. Cannot resume pipeline from SMP2 step." f"Please delete TST170 analysis in BaseSpace and resume pipeline from" f"TST170 stage.") # If resuming from SMP2v3 required, resuming from TST170 required or full pipeline- launch the SMP2 app if not args.dl_files: # Create launch app object for SMP2 v3 if not just downloading files- poll TST170 and when complete # launch SMP2 launch_smp = LaunchApp(self.authentication_token, worksheet, project, smp2_app_name, smp2_app_version, sample_pairs, tst_170) # Poll the tst 170 appsessions until completion, then launch smp2 app smp_appsession = launch_smp.poll_tst170_launch_smp2() # Dump data to file with open(os.path.join(os.getcwd(), "smp.json"), 'w') as s: json.dump(smp_appsession, s) # If downloading files from a completed SMP2 app required # Create a LaunchApp object for smp2 app if flag to only download files is set- allows for polling of SMP2 if args.dl_files: # Load data in required smp2 data from file try: with open(os.path.join(os.getcwd(), "smp.json")) as sm: smp = json.load(sm) except FileNotFoundError: raise FileNotFoundError( f"Could not find file smp.json. Cannot resume pipeline from download step." f"Please delete SMP2 analysis in BaseSpace and resume pipeline from" f"SMP2 stage.") launch_smp = LaunchApp(self.authentication_token, worksheet, project, smp2_app_name, smp2_app_version, sample_pairs, None, smp) # None as tst170 app data not required # Poll the smp appsessions until completion smp_appresults = launch_smp.poll_smp2() # Download all required files- every step requires this file_download = FileDownloader(self.authentication_token, smp_appresults, worksheet) file_download.download_files()
from flask.ext.sqlalchemy import SQLAlchemy import gevent from gevent import monkey # patches stdlib (including socket and ssl modules) to cooperate with other greenlets monkey.patch_all() app = Flask(__name__) app.config.from_object('config') db = SQLAlchemy(app) import logging from logging.handlers import RotatingFileHandler file_handler = RotatingFileHandler('tmp/downloads.log', 'a', 1 * 1024 * 1024, 10) # file_handler.setFormatter( logging.Formatter( '%(asctime)s %(levelname)s: %(message)s [in %(pathname)s:%(lineno)d]')) app.logger.setLevel(logging.DEBUG) file_handler.setLevel(logging.DEBUG) app.logger.addHandler(file_handler) app.logger.info('===startup downloader===') log = app.logger.info log2 = app.logger.debug from downloads import file_manager from file_downloader import FileDownloader Downloads = FileDownloader()
from PySide2.QtCore import QObject, Signal, Property, QUrl from PySide2.QtGui import QGuiApplication, QIcon from PySide2.QtQml import QQmlApplicationEngine from file_downloader import FileDownloader from program_settings import SettingsManager import resources import sys app = QGuiApplication(sys.argv) app.setWindowIcon(QIcon(':/mi.png')) engine = QQmlApplicationEngine() settings_man = SettingsManager() fd = FileDownloader() engine.rootContext().setContextProperty("filedownloader", fd) engine.rootContext().setContextProperty("sm", settings_man) engine.load(QUrl.fromLocalFile(':/Main.qml')) if not engine.rootObjects(): sys.exit(-1) sys.exit(app.exec_())
class ChatWidget(QWidget): def __init__(self, parent, cybuddy): super(ChatWidget, self).__init__() self.widget = uic.loadUi('ui/chatwidget.ui') self.parent_window = parent self.app = parent.app self.cybuddy = cybuddy self.typingTimer = None self.is_ready = False self.queue = [] self.accepted_fts = [] # accepted file transfers self.transfer_tasks = {} self.webcam_viewer = None defaultSettings = QWebSettings.globalSettings() defaultSettings.setAttribute(QWebSettings.JavascriptEnabled, True) defaultSettings.setAttribute(QWebSettings.PluginsEnabled, True) defaultSettings.setAttribute(QWebSettings.DeveloperExtrasEnabled, True) ym.register_callback(cb.EMUSSA_CALLBACK_TYPING_NOTIFY, self._typing) self.widget.textEdit.keyPressEvent = self._writing_message self.widget.sendButton.clicked.connect(self._send_message) self.widget.filetransfer_btn.clicked.connect(self._send_file) self.widget.myAvatar.setPixmap(self.app.me.avatar.image) self.widget.messagesView.setUrl(QUrl('ui/resources/html/chat/index.html')) self.widget.messagesView.loadFinished.connect(self._document_ready) self.widget.messagesView.linkClicked.connect(self._link_clicked) self.widget.messagesView.page().setLinkDelegationPolicy(QWebPage.DelegateAllLinks) self.cybuddy.update.connect(self._update_buddy) self.cybuddy.update_status.connect(self._update_status) self.cybuddy.update_avatar.connect(self._update_avatar) self.widget.addUserBtn.clicked.connect(self.add_buddy) self._update_buddy() self._update_status() self._update_avatar() if cybuddy.yahoo_id in self.app.buddylist.buddy_items: self.widget.addUserBtn.setHidden(True) self.widget.ignoreUserBtn.setHidden(True) def _javascript(self, function, *args): # if the document is not ready, wait a while until we start calling JS functions on it if not self.is_ready: self.queue.append([function, args]) arguments_list = [] for arg in args: jsarg = str(arg).replace("'", "\\'") arguments_list.append("'" + jsarg + "'") jscode = function + "(" + ", ".join(arguments_list) + ")" self.widget.messagesView.page().mainFrame().evaluateJavaScript(jscode) def _document_ready(self): self.is_ready = True if self.app.me.status.code == YAHOO_STATUS_INVISIBLE: pixmap = QPixmap(":status/resources/user-invisible.png") self._add_info('You appear offline to <span class="buddyname">{0}</span>'.format( self.cybuddy.display_name) ) elif not self.cybuddy.status.online: pixmap = QPixmap(":status/resources/user-offline.png") self._add_info('<span class="buddyname">{0}</span> seems to be offline and will' 'receive your messages next time when he/she logs in.' .format(self.cybuddy.display_name), pixmap) for task in self.queue: self._javascript(task[0], *task[1]) self.queue = [] self._javascript('show_timestamps', 'true') def _link_clicked(self, url): if url.scheme() == 'cyuf': transfer_id = QFileInfo(url.path()).fileName() if url.host() == 'save': transfer_task = self.transfer_tasks[transfer_id] path = None if len(transfer_task.files) == 1: path = QFileDialog.getSaveFileName( self.widget, "Save file", "{0}/{1}".format(QDir.homePath(), transfer_task.files[0].filename), "Any file (*.*)" ) else: path = QFileDialog.getExistingDirectory( self.widget, "Save file", "{0}".format(QDir.homePath()), QFileDialog.ShowDirsOnly ) if path: print('Save as: {0}'.format(path)) transfer_task.destination = path self.accepted_fts.append(transfer_id) ym.accept_transfer(self.cybuddy.yahoo_id, transfer_id) if url.host() == 'decline': self._javascript('file_decline', self.cybuddy.yahoo_id, transfer_id) ym.decline_transfer(self.cybuddy.yahoo_id, transfer_id) if url.host() == 'cancel-send': self._javascript('cancel_send', self.cybuddy.yahoo_id, transfer_id) ym.cancel_transfer(self.cybuddy.yahoo_id, transfer_id) if url.host() == 'open-file' or url.host() == 'open-dir': QDesktopServices.openUrl(QUrl(url.path())) if url.host() == 'decline-web': yahoo_id = url.path()[1:] self._javascript('webcam_declined', yahoo_id) ym.deny_webcam_request(yahoo_id) if url.host() == 'accept-web': yahoo_id = url.path()[1:] self._javascript('webcam_accepted', yahoo_id) ym.accept_webcam_request(yahoo_id) self.view_webcam() def _get_link_from_status(self): sep = '' statusmsg = self.cybuddy.status.message if statusmsg: sep = ' - ' if 'http://' in statusmsg: href = statusmsg[statusmsg.index('http://'):].split(' ', 1)[0] statusmsg = statusmsg.replace(href, '').lstrip().rstrip() text = '<font color="#8C8C8C">{0}<a href="{1}">{2}</a></font>'.format(sep, href, statusmsg) return [href, text] elif 'https://' in statusmsg: href = statusmsg[statusmsg.index('https://'):].split(' ', 1)[0] statusmsg = statusmsg.replace(href, '').lstrip().rstrip() text = '<font color="#8C8C8C">{0}<a href="{1}">{2}</a></font>'.format(sep, href, statusmsg) return [href, text] elif 'www.' in statusmsg: href = statusmsg[statusmsg.index('www.'):].split(' ', 1)[0] statusmsg = statusmsg.replace(href, '').lstrip().rstrip() text = '<font color="#8C8C8C">{0}<a href="{1}">{2}</a></font>'.format(sep, href, statusmsg) return [href, text] else: text = '<font color="#8C8C8C">{0}{1}</font>'.format(sep, self.cybuddy.status.message) return ['', text] def _update_buddy(self): self.widget.contactName.setText(self.cybuddy.display_name) self._javascript('update_buddy_name', self.cybuddy.display_name) def _update_status(self): if self.cybuddy.status.online: if self.cybuddy.status.idle_time: self.widget.contactStatus.setPixmap(QPixmap(":status/resources/user-away.png")) if self.sender().__class__.__name__ == 'CyBuddy': self._notify_status('is Idle.') elif self.cybuddy.status.code == YAHOO_STATUS_BUSY: self.widget.contactStatus.setPixmap(QPixmap(":status/resources/user-busy.png")) if self.sender().__class__.__name__ == 'CyBuddy': self._notify_status('is Busy.') else: self.widget.contactStatus.setPixmap(QPixmap(":status/resources/user-online.png")) if self.sender().__class__.__name__ == 'CyBuddy': self._notify_status('is now Available.') statusMsg = self._get_link_from_status() self.widget.contactStatusText.setText(statusMsg[1]) self.widget.contactStatusText.setToolTip(statusMsg[0]) else: self.widget.contactStatus.setPixmap(QPixmap(":status/resources/user-offline.png")) self.widget.contactStatusText.setText('') self.widget.contactStatusText.setToolTip('') if self.sender().__class__.__name__ == 'CyBuddy': self._notify_status('is now Offline.') def _notify_status(self, message): self._javascript('status_updated', '<span class="buddy_items">{0}</span> {1}' .format(self.cybuddy.display_name, message), util.pixmap_to_base64(self.widget.contactStatus.pixmap()), datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')) def _update_avatar(self): self.widget.hisAvatar.setPixmap(self.cybuddy.avatar.image) def _writing_message(self, e): if e.key() == Qt.Key_Return or e.key() == Qt.Key_Enter: self.widget.sendButton.click() ym.send_typing(self.cybuddy.yahoo_id, False) self.killTimer(self.typingTimer) self.typingTimer = None else: QTextEdit.keyPressEvent(self.widget.textEdit, e) if e.key() > Qt.Key_0 and e.key() < Qt.Key_Z: if self.typingTimer: self.killTimer(self.typingTimer) else: ym.send_typing(self.cybuddy.yahoo_id, True) self.typingTimer = self.startTimer(5000) def _typing(self, cyemussa, tn): sender = tn.sender if not sender: # we are typing this from somewhere else sender = self.app.me.yahoo_id if not sender == self.app.me.yahoo_id and not sender == self.cybuddy.yahoo_id: return if tn.status: self._javascript('start_typing', sender) else: self._javascript('stop_typing') def _add_info(self, text, pixmap=None): image = None if pixmap: image = util.pixmap_to_base64(pixmap) # text = util.sanitize_html(text) if image: self._javascript('add_info', text, image) else: self._javascript('add_info', text) def _send_message(self): raw_msg = self.widget.textEdit.toPlainText() timestamp = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') self.widget.textEdit.setDocument(QTextDocument()) ym.send_message(self.cybuddy.yahoo_id, str(raw_msg)) self._javascript('message_out', self.app.me.yahoo_id, self._text_to_emotes(raw_msg), timestamp) def _send_file(self): files = QFileDialog.getOpenFileNames( self.widget, "Select one or more files to send", QDir.homePath(), "Any file (*.*)" ) ftinfo = cyemussa.CyFileTransferInfo() ftinfo.sender = self.app.me.yahoo_id ftinfo.receiver = self.cybuddy.yahoo_id ftinfo.host = ym.ft_host ftinfo.transfer_id = util.string_to_base64( util.string_to_md5(util.random_string(25)) ) transfer_task = FileTransferTask() transfer_task.transfer_info = ftinfo files_absolute = [] sizes = [] for path in files: f = cyemussa.CyFile() f.filename = QFile(path).fileName() f.filesize = QFile(path).size() transfer_task.files.append(f) files_absolute.append(QFileInfo(f.filename).fileName()) sizes.append(f.filesize) self.transfer_tasks[transfer_task.transfer_info.transfer_id] = transfer_task thumbnail = None if len(files) == 1 and QImageReader.imageFormat(files[0]): icon = util.scalePixmapAspectFill(QPixmap(files[0]), QSize(32, 32)) icon_base64 = util.pixmap_to_base64(icon, 'JPG') thumbnail = icon_base64 else: icon = QFileIconProvider().icon(QFileInfo(files[0])) icon_base64 = util.pixmap_to_base64(icon.pixmap(QSize(32, 32))) self._javascript( 'file_out', ftinfo.receiver, json.dumps(files_absolute), json.dumps(sizes), ftinfo.transfer_id, icon_base64 ) ym.send_files( self.cybuddy.yahoo_id, transfer_task.transfer_info.transfer_id, files, thumbnail ) def _text_to_emotes(self, text): words = text.split() for i, w in enumerate(words): for emo in emotes: pattern = re.compile(re.escape(emo), re.IGNORECASE) word = pattern.sub('<img src="{0}" alt="{1}" />'.format(emotes[emo], emo), w) if not word == w: # a replacement was made, skip to the next word words[i] = word break text = ' '.join(words) return text def _download_progress(self, transfer_id, filename, progress): self._javascript( 'file_progress', transfer_id, QFileInfo(filename).fileName(), progress ) def _download_finished(self, transfer_id): transfer_task = self.transfer_tasks[transfer_id] if len(transfer_task.files): ym.accept_transfer_next_file(self.cybuddy.yahoo_id, transfer_id) self._download_progress(transfer_id, transfer_task.files[0], 0.0) else: del self.transfer_tasks[transfer_id] if QFileInfo(transfer_task.destination).isDir(): action = 'open-dir' else: action = 'open-file' self._javascript( 'transfer_finished', transfer_id, transfer_task.transfer_info.sender, len(transfer_task.paths), action, transfer_task.destination ) def timerEvent(self, event): ym.send_typing(self.cybuddy.yahoo_id, False) if self.typingTimer: self.killTimer(self.typingTimer) self.typingTimer = None def close(self): # called by parent when the chat is closing ym.unregister_callback(cb.EMUSSA_CALLBACK_TYPING_NOTIFY, self._typing) self.cybuddy.update.disconnect(self._update_buddy) if self.typingTimer: ym.send_typing(self.cybuddy.yahoo_id, False) def add_buddy(self): self.add_buddy_wizard = AddBuddyWizard(self.app, self.cybuddy.yahoo_id) def receive_message(self, cymessage): message = util.yahoo_rich_to_html(cymessage.message) timestamp = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') if not cymessage.sender: sender = self.app.me.yahoo_id else: sender = self.cybuddy.display_name if cymessage.offline: message = '(offline) {0}'.format(message) if cymessage.timestamp: timestamp = datetime.datetime.fromtimestamp(int(cymessage.timestamp)).strftime('%Y-%m-%d %H:%M:%S') self._javascript('message_in', sender, self._text_to_emotes(util.sanitize_html(message)), timestamp) def receive_file_transfer(self, cyfiletransfer): files = [] sizes = [] ftinfo = cyemussa.CyFileTransferInfo() ftinfo.transfer_id = cyfiletransfer.transfer_id ftinfo.sender = cyfiletransfer.sender transfer_task = FileTransferTask() transfer_task.transfer_info = ftinfo for f in cyfiletransfer.files: transfer_task.files.append(cyemussa.CyFile(f)) files.append(f.filename) sizes.append(f.filesize) self.transfer_tasks[cyfiletransfer.transfer_id] = transfer_task if not cyfiletransfer.thumbnail: icon = QFileIconProvider().icon(QFileInfo(files[0])) icon_base64 = util.pixmap_to_base64(icon.pixmap(QSize(32, 32))) else: icon_base64 = cyfiletransfer.thumbnail self._javascript( 'file_in', cyfiletransfer.sender, json.dumps(files), json.dumps(sizes), cyfiletransfer.transfer_id, icon_base64 ) def receive_file_info(self, cyftinfo): if cyftinfo.transfer_id in self.accepted_fts: ym.accept_file( cyftinfo.sender, cyftinfo.receiver, cyftinfo.filename, cyftinfo.transfer_id, cyftinfo.transfer_type, cyftinfo.host, cyftinfo.relay_id ) transfer_task = self.transfer_tasks[cyftinfo.transfer_id] finfo = QFileInfo(transfer_task.destination) if finfo.isDir(): path = '{0}/{1}'.format(transfer_task.destination, cyftinfo.filename) else: path = '{0}'.format(transfer_task.destination) # remove the current file from left files list and download it transfer_task.remove_file(cyftinfo.filename) transfer_task.paths.append(path) self._download_progress(cyftinfo.transfer_id, cyftinfo.filename, 0.0) self.downloader = FileDownloader(transfer_task, path) self.downloader.progress.connect(self._download_progress) self.downloader.finished.connect(self._download_finished) self.downloader.download(cyftinfo.get_download_url()) def transfer_cancelled(self, cyfiletransfer): self._javascript( 'file_cancel', cyfiletransfer.sender, cyfiletransfer.transfer_id ) def transfer_accepted(self, cyfiletransfer): transfer_task = self.transfer_tasks[cyfiletransfer.transfer_id] ym.send_file_info( self.cybuddy.yahoo_id, cyfiletransfer.transfer_id, QFileInfo(transfer_task.files[0].filename).fileName() ) def transfer_rejected(self, cyfiletransfer): self._javascript( 'file_rejected', cyfiletransfer.sender, cyfiletransfer.transfer_id ) def transfer_upload(self, cyftinfo): transfer_task = self.transfer_tasks[cyftinfo.transfer_id] if not cyftinfo.relay_id: # this is an already started multi-file transfer # which now requires us to upload the next file transfer_task.files.pop(0) if len(transfer_task.files): filepath = transfer_task.files[0].filename self.uploader = FileUploader(transfer_task) self.uploader.progress.connect(self._download_progress) self.uploader.finished.connect(self._download_finished) ym.send_file_info( self.cybuddy.yahoo_id, cyftinfo.transfer_id, QFileInfo(filepath).fileName() ) # self.uploader.upload( # transfer_task.transfer_info.get_download_url(), # filepath # ) return filepath = transfer_task.files[0].filename transfer_task.transfer_info.relay_id = cyftinfo.relay_id self.uploader = FileUploader(transfer_task) self.uploader.progress.connect(self._download_progress) self.uploader.finished.connect(self._download_finished) self.uploader.upload( transfer_task.transfer_info.get_download_url(), filepath ) def webcam_invite(self, webcam_notification): self._javascript( 'webcam_invite', webcam_notification.sender ) def view_webcam(self): ym.send_webcam_request(self.cybuddy.yahoo_id) self.webcam_viewer = WebcamViewer(self) def receive_audible(self, audible): self.audible = audible self.manager = QNetworkAccessManager() self.manager.finished.connect(self.post_audible) self.req = QNetworkRequest(QUrl(self.audible.url)) self.req.setRawHeader('User-agent', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US)') self.reply = self.manager.get(self.req) def post_audible(self, swfdata): tmpfile = QTemporaryFile('XXXXXX.swf', self.widget) if tmpfile.open(): tmpfile.write(swfdata.readAll()) tmpfile.close() timestamp = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') self._javascript('audible_in', self.audible.sender, tmpfile.fileName(), self.audible.message, timestamp)
import sys sys.path.append('..') from file_downloader import FileDownloader if __name__=="__main__": FD = FileDownloader('../../conf/global_var.json') batch = [(u'704FCD3C4D673DE631BF985CFEABCB121A4EB7CF1FD135E1C98CFA02268FA035', u'https://s3.amazonaws.com/memex-images/full/bdb9cdde29d5a29411fe7a7dc1ca501a37303431.jpg', {u'_type': u'escorts', u'_timestamp': 1464761131138, u'_index': u'memex-domains_2016.01', u'_score': 0.0, u'fields': {u'obj_parent': [u'2E7904456709725B2B1F3108683D6895FCE0BDF8BFA29E34FD799FF8AAE4CFE5'], u'timestamp': [u'2016-01-26T12:31:11'], u'obj_original_url': [u'http://469-978-3183.escortphonelist.com/images/escort-phone-list.jpg'], u'obj_stored_url': [u'https://s3.amazonaws.com/memex-images/full/bdb9cdde29d5a29411fe7a7dc1ca501a37303431.jpg']}, u'_id': u'704FCD3C4D673DE631BF985CFEABCB121A4EB7CF1FD135E1C98CFA02268FA035'}), (u'3F98B492B2BA98676202271F85D2E3EF0D1E02B9A1C706E76D53C8AAA374494B', u'https://s3.amazonaws.com/memex-images/full/7f667480e65f17e2bec4c3da58bd407d06c4276a.jpg', {u'_type': u'escorts', u'_timestamp': 1464761131138, u'_index': u'memex-domains_2016.01', u'_score': 0.0, u'fields': {u'obj_parent': [u'44D3364D05EF7544A4B8ED36738FAFC9F56651D29D76C930DB73588A5221434B'], u'timestamp': [u'2016-01-26T12:32:20'], u'obj_original_url': [u'http://images.eroticmugshots.com/cities/63/large/21490623-2.jpg'], u'obj_stored_url': [u'https://s3.amazonaws.com/memex-images/full/7f667480e65f17e2bec4c3da58bd407d06c4276a.jpg']}, u'_id': u'3F98B492B2BA98676202271F85D2E3EF0D1E02B9A1C706E76D53C8AAA374494B'}), (u'701DA3E5D25EB3329D710C7181ED6DA9D60D081711822EDA056D29F808E39A95', u'https://s3.amazonaws.com/memex-images/full/219e1c7ba41154f5dada28d1a0bddac280900f0d.jpg', {u'_type': u'escorts', u'_timestamp': 1464761131138, u'_index': u'memex-domains_2016.01', u'_score': 0.0, u'fields': {u'obj_parent': [u'7FE5C4C07A16FFFC22E359BD067D15C816F04BD433950C95767CE81DDB441923'], u'timestamp': [u'2016-01-26T12:32:11'], u'obj_original_url': [u'http://images.eroticmugshots.com/cities/13/thumbnail/34348493-1.jpg'], u'obj_stored_url': [u'https://s3.amazonaws.com/memex-images/full/219e1c7ba41154f5dada28d1a0bddac280900f0d.jpg']}, u'_id': u'701DA3E5D25EB3329D710C7181ED6DA9D60D081711822EDA056D29F808E39A95'}), (u'6D7387E477A206C965EF1535BB83E004147531CE99F9AA14D2B3FD6FC94B22F1', u'https://s3.amazonaws.com/memex-images/full/173446f78bf85d8baf94456c4918e3941e72ee80.jpg', {u'_type': u'escorts', u'_timestamp': 1464761131138, u'_index': u'memex-domains_2016.01', u'_score': 0.0, u'fields': {u'obj_parent': [u'1E61CBB6E97B170EF81926D38E0CFBCD4F31763C1461051447713CF4601E8E7D'], u'timestamp': [u'2016-01-26T12:32:16'], u'obj_original_url': [u'http://images.escortsincollege.com/cities/126/large/10593977-3.jpg'], u'obj_stored_url': [u'https://s3.amazonaws.com/memex-images/full/173446f78bf85d8baf94456c4918e3941e72ee80.jpg']}, u'_id': u'6D7387E477A206C965EF1535BB83E004147531CE99F9AA14D2B3FD6FC94B22F1'}), (u'5B369C1A8E426BC5A5A44FDCAC9D9CE378094967AFEAC184A49A5BAF22E62EF5', u'https://s3.amazonaws.com/roxyimages/5c76b19d1548351973868f9ff97d9c0a52f918cb.jpg', {u'_type': u'escorts', u'_timestamp': 1464761131889, u'_index': u'memex-domains_2016.01', u'_score': 0.0, u'fields': {u'obj_parent': [u'51FEA8D56B4AECCDEFB4469A08DDECD29413334D6575F43B6268CA026E85E586'], u'timestamp': [u'2015-12-17T23:06:11'], u'obj_original_url': [u'http://images4.backpage.com/imager/u/large/125355586/3ef7f93df039c1fa1b6dadfa6c482745.jpg'], u'obj_stored_url': [u'https://s3.amazonaws.com/roxyimages/5c76b19d1548351973868f9ff97d9c0a52f918cb.jpg']}, u'_id': u'5B369C1A8E426BC5A5A44FDCAC9D9CE378094967AFEAC184A49A5BAF22E62EF5'}), (u'0A775A8BDE19A6EA640636F02FD451056D5FDA2300844DCC0AFBC652C02BA37B', u'https://s3.amazonaws.com/roxyimages/3f844e64033e87ca67cfde1154bec0578ee6e2cb.jpg', {u'_type': u'escorts', u'_timestamp': 1464761131889, u'_index': u'memex-domains_2016.01', u'_score': 0.0, u'fields': {u'obj_parent': [u'BD7FB7967E393CEFA88185132FC3106C5DF162F6F63A21AAD6632170F9EDC858'], u'timestamp': [u'2015-12-30T05:58:09'], u'obj_original_url': [u'http://images2.backpage.com/imager/u/large/230328867/01bf2acb8bdf85b6df93aeeb2394c11b.jpg'], u'obj_stored_url': [u'https://s3.amazonaws.com/roxyimages/3f844e64033e87ca67cfde1154bec0578ee6e2cb.jpg']}, u'_id': u'0A775A8BDE19A6EA640636F02FD451056D5FDA2300844DCC0AFBC652C02BA37B'}), (u'CAFB17AB66382E6834B1AB6ED854750054188789532F80DE99B7253B811BDB1F', u'https://s3.amazonaws.com/roxyimages/54f226755c7371c1651c5b0920ca563b89e03860.jpg', {u'_type': u'escorts', u'_timestamp': 1464761131889, u'_index': u'memex-domains_2016.01', u'_score': 0.0, u'fields': {u'obj_parent': [u'1383B4D668C03367999D645EC4418A450AC5FF5D19EE0681E39CCB4E7FA00642'], u'timestamp': [u'2015-12-16T23:58:17'], u'obj_original_url': [u'http://images4.backpage.com/imager/u/large/125277948/f58eca086858632543b39fcd0d2636fd.jpg'], u'obj_stored_url': [u'https://s3.amazonaws.com/roxyimages/54f226755c7371c1651c5b0920ca563b89e03860.jpg']}, u'_id': u'CAFB17AB66382E6834B1AB6ED854750054188789532F80DE99B7253B811BDB1F'}), (u'C0BABA15173709DA7A56207670578312CA30A7E94EDBF68A8140D8D5AA3C20FC', u'https://s3.amazonaws.com/memex-images/full/2d7348b6561c9329f8f8841dc929738ada17e95c.jpg', {u'_type': u'escorts', u'_timestamp': 1464761131973, u'_index': u'memex-domains_2016.01', u'_score': 0.0, u'fields': {u'obj_parent': [u'60C23254F4EADC5B927316AF6596BFC81CBE4187AE00419F7EAB20424F34ED91'], u'timestamp': [u'2016-01-26T13:15:21'], u'obj_original_url': [u'http://img13.asexyservice.com/4/p/200/56a6f1147f975rfRkwq_jZw8Dfd2a72kQGmO6PRwRaWKjXWdp4_200.jpg'], u'obj_stored_url': [u'https://s3.amazonaws.com/memex-images/full/2d7348b6561c9329f8f8841dc929738ada17e95c.jpg']}, u'_id': u'C0BABA15173709DA7A56207670578312CA30A7E94EDBF68A8140D8D5AA3C20FC'}), (u'6687736C0D7E5702DFBA1B94EC8F75E17CE956C5649C4354F5CDD828AEB226D6', u'https://s3.amazonaws.com/memex-images/full/229d071eafac66b81b47c7e7da041c6c91fc9ecf.jpg', {u'_type': u'escorts', u'_timestamp': 1464761131973, u'_index': u'memex-domains_2016.01', u'_score': 0.0, u'fields': {u'obj_parent': [u'5B193E4B99B3C64CA403D0EF4231D8E58554360103DBF069E09C9308B12FEFFC'], u'timestamp': [u'2016-01-26T13:15:18'], u'obj_original_url': [u'http://img7.asexyservice.com/Z/A/200/56a6f114be85a8267wNaP8K32cfsGA_emZr_DrkTMg1mZC84AZ_200.jpg'], u'obj_stored_url': [u'https://s3.amazonaws.com/memex-images/full/229d071eafac66b81b47c7e7da041c6c91fc9ecf.jpg']}, u'_id': u'6687736C0D7E5702DFBA1B94EC8F75E17CE956C5649C4354F5CDD828AEB226D6'}), (u'70520066B7DB04EF9B01858AFC652E973A2098B6039DBDB014063696B0480B82', u'https://s3.amazonaws.com/roxyimages/9bb2b24ccc451467da2e7ae89782b2d9ede6b0c0.jpg', {u'_type': u'escorts', u'_timestamp': 1464761132012, u'_index': u'memex-domains_2016.01', u'_score': 0.0, u'fields': {u'obj_parent': [u'CADCDF5FE92191882A3786FC2A80D3C55F4429F8C35DAA2BB68EC0554EDEEBB2'], u'timestamp': [u'2015-11-17T06:36:19'], u'obj_original_url': [u'http://images2.backpage.com/imager/u/large/222290798/8fd6fdf9abf273e7b235105742f54403.jpg'], u'obj_stored_url': [u'https://s3.amazonaws.com/roxyimages/9bb2b24ccc451467da2e7ae89782b2d9ede6b0c0.jpg']}, u'_id': u'70520066B7DB04EF9B01858AFC652E973A2098B6039DBDB014063696B0480B82'})] readable_images = FD.download_images(batch) print readable_images
from bs4 import BeautifulSoup from requests.exceptions import RequestException import requests from file_downloader import FileDownloader fileDownLoader = FileDownloader() for page_index in range(1, 110): with open('io-data/log.txt', 'a') as log_file: try: with open('io-data/t-shirt-links.txt', 'a') as t_shirt_links: t_shirt_links.write( 'Starting to process page {0}\n'.format(page_index)) headers = { 'user-agent': 'user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36' } try: pages_with_data = requests.get( 'https://www.gearbest.com/men-s-t-shirts-c_11926/{0}.html?page_size=120' .format(str(page_index)), headers=headers) # r = requests.get('https://www.threadless.com/catalog/type,guys/style,tees/page,{0}'.format(page_index)) except RequestException as e: print('Error in page {0}'.format(str(page_index))) links_parser = BeautifulSoup(pages_with_data.text) results = links_parser.find_all('a', {'class', 'gbGoodsItem_thumb'})
def read_excel_file(self): workbook = load_workbook(self.file_dir + 'GBP-SBP-resource-centre-250917.xlsx') first_sheet = workbook.get_sheet_names()[0] worksheet = workbook.get_sheet_by_name(first_sheet) for i, row in enumerate(worksheet.iter_rows()): if i == 0: continue if self.es.check_document_exists(index="green_bond", doc="report", id=i): print "Document " + str(i) + " exists!" doc = self.es.find_document(index="green_bond", doc="report", id=i) if row[4].hyperlink is not None and row[ 4].internal_value == doc["_source"][ "external_review_report"]: try: link = row[4].hyperlink.target file_downloader = FileDownloader() file_downloader.download_file(link) text = self.read_pdf_file(self.file_dir + "temp_file.pdf") gbp = GBP(row[0].internal_value, row[1].internal_value, row[2].internal_value, row[3].internal_value, text, row[5].internal_value) self.es.update_document(index="green_bond", doc="report", id=i, data=json.dumps(gbp.__dict__)) print "Update document " + str(i) + " succesfully!" except: print "Hyperlink error! Document " + str(i) pass if row[3].hyperlink is not None and row[ 3].internal_value == doc["_source"][ "external_review_form"]: try: link = row[3].hyperlink.target file_downloader = FileDownloader() file_downloader.download_file(link) text = self.read_pdf_file(self.file_dir + "temp_file.pdf") gbp = GBP( doc["_source"]["green_bond_issuer"], doc["_source"]["country"], doc["_source"]["market_information_template"], text, doc["_source"]["external_review_report"], doc["_source"]["external_link"]) self.es.update_document(index="green_bond", doc="report", id=i, data=json.dumps(gbp.__dict__)) print "Update document " + str(i) + " succesfully!" except: print "Hyperlink error! Document " + str(i) pass continue if row[4].hyperlink is None: gbp = GBP(row[0].internal_value, row[1].internal_value, row[2].internal_value, row[3].internal_value, row[4].internal_value, row[5].internal_value) self.es.add_document(i, json.dumps(gbp.__dict__)) print "Adding document " + str(i) + " succesfully!" else: try: link = row[4].hyperlink.target file_downloader = FileDownloader() file_downloader.download_file(link) self.text = self.read_pdf_file(self.file_dir + "temp_file.pdf") gbp = GBP(row[0].internal_value, row[1].internal_value, row[2].internal_value, row[3].internal_value, self.text, row[5].internal_value) self.es.add_document(i, json.dumps(gbp.__dict__)) print "Adding document " + str(i) + " succesfully!" except: print "Hyperlink error! Document " + str(i) pass
class App(tk.Tk): bot = None driver = None meta_data = None download_queue_video = None download_queue_captions = None external_exercise_queue = None file_downloader = None def __init__(self, title, *args, **kwargs): tk.Tk.__init__(self, *args, **kwargs) self.title(title) # Adjust size # self.geometry("400x200") self.maxsize(800, 600) self.minsize(800, 470) # Create left and right frames top_frame = self.createTopFrame() bottom_frame = self.createBottomFrame() # Create frames on top frame self.createInputFrame(top_frame) self.createOutputFrame(top_frame) # Create frames on right frame # self.createStatusFrame(bottom_frame) self.createFileDownloaderFrame(bottom_frame) ################################################################################################################### """" Frame Creation Functions """ ################################################################################################################### def createTopFrame(self): top_frame = tk.Frame(self, width=650, height=200, bg='grey') top_frame.grid(row=0, column=0, padx=10, pady=5, sticky=tk.N + tk.W + tk.E) return top_frame def createBottomFrame(self): bottom_frame = tk.Frame(self, width=650, height=400, bg='grey') bottom_frame.grid(row=1, column=0, padx=10, pady=5, sticky=tk.N + tk.W + tk.E) return bottom_frame def createInputFrame(self, root): labelframe = ttk.LabelFrame(root, text="Input", width=600, height=300) labelframe.pack(fill="x", expand="yes") course_link_label = tk.Label(labelframe, text="Course Link:") self.course_link_entry = ttk.Entry(labelframe, width=100) self.load_button = ttk.Button(labelframe, text="Load", width=8) self.load_button.config(command=self.loadButtonAction) self.input_status_label = tk.Label(labelframe, text="", fg="green") # self.get_video_check_var = tk.IntVar(value=1) # self.get_reading_check_var = tk.IntVar(value=1) # self.get_quiz_check_var = tk.IntVar(value=1) # self.get_graded_check_var = tk.IntVar(value=1) # checkboxes_frame = tk.Frame(labelframe) # get_video_check_box = tk.Checkbutton(checkboxes_frame, text="Video", variable=self.get_video_check_var) # get_reading_check_box = tk.Checkbutton(checkboxes_frame, text="Reading", variable=self.get_reading_check_var) # get_quiz_check_box = tk.Checkbutton(checkboxes_frame, text="Quiz", variable=self.get_quiz_check_var) # get_graded_check_box = tk.Checkbutton(checkboxes_frame, text="Peer Graded Assignment", variable=self.get_graded_check_var) padding_y = 3 course_link_label.grid(row=0, column=0, padx=5, pady=padding_y + 2, sticky=tk.S + tk.N) self.course_link_entry.grid(row=0, column=1, padx=5, pady=padding_y + 2, sticky=tk.S + tk.N) self.load_button.grid(row=0, column=2, padx=5, pady=padding_y + 2, sticky=tk.S + tk.N) self.input_status_label.grid(row=1, column=1, padx=5, pady=padding_y - 3, sticky=tk.W + tk.S + tk.N) # Checkboxes Frame # checkboxes_frame.grid(row=1, column=1, pady=padding_y + 2, sticky=tk.S + tk.N + tk.W) # get_video_check_box.grid(row=0, column=0, padx=5, sticky=tk.S + tk.N + tk.W) # get_reading_check_box.grid(row=0, column=1, padx=5, sticky=tk.S + tk.N + tk.W) # get_quiz_check_box.grid(row=0, column=2, padx=5, sticky=tk.S + tk.N + tk.W) # get_graded_check_box.grid(row=0, column=3, padx=5, sticky=tk.S + tk.N + tk.W) def createOutputFrame(self, root): labelframe = ttk.LabelFrame(root, text="Output", width=600, height=300) labelframe.pack(fill="x", expand="yes") output_folder_label = tk.Label(labelframe, text="Output:") self.output_folder_var = tk.StringVar() output_folder_entry = ttk.Entry(labelframe, textvariable=self.output_folder_var, width=100) def browseFolder(): folder_selected = filedialog.askdirectory() if folder_selected != "": self.output_folder_var.set(folder_selected) browse_button = ttk.Button(labelframe, text="Browse", width=8, command=browseFolder) # browse_button.config(command=browseFolder) self.get_video_check_var = tk.IntVar(value=1) self.get_reading_check_var = tk.IntVar(value=1) self.get_quiz_check_var = tk.IntVar(value=1) self.get_graded_check_var = tk.IntVar(value=1) self.get_external_check_var = tk.IntVar(value=1) checkboxes_frame = tk.Frame(labelframe) get_video_check_box = ttk.Checkbutton( checkboxes_frame, text="Video", variable=self.get_video_check_var) get_reading_check_box = ttk.Checkbutton( checkboxes_frame, text="Reading", variable=self.get_reading_check_var) get_quiz_check_box = ttk.Checkbutton(checkboxes_frame, text="Quiz", variable=self.get_quiz_check_var) get_graded_check_box = ttk.Checkbutton( checkboxes_frame, text="Peer Graded", variable=self.get_graded_check_var) get_external_check_box = ttk.Checkbutton( checkboxes_frame, text="External Exercise", variable=self.get_external_check_var) # Button Group buttons_frame = tk.Frame(labelframe) self.download_all_button = ttk.Button( buttons_frame, text="Start Download", command=self.downloadAllButtonAction) # Menu Button self.more_menu_button = ttk.Menubutton(buttons_frame, text='More') self.more_menu_button.menu = tk.Menu(self.more_menu_button, tearoff=0) self.more_menu_button['menu'] = self.more_menu_button.menu self.more_menu_button.menu.add_command(label="Scrape", command=self.scrapeButtonAction) self.more_menu_button.menu.add_command( label="Download Video", command=self.downloadVideoButtonAction) self.more_menu_button.menu.add_command( label="Download External Exercise", state=tk.DISABLED, command=self.downloadExternalButtonAction) self.more_menu_button.menu.add_command( label="Download Resource", command=self.downloadResourceButtonAction) self.more_menu_button.menu.add_command( label="Download Images", command=self.downloadImageButtonAction) self.more_menu_button.menu.add_command( label="Download Attachments", command=self.downloadAttachmentButtonAction) # self.scrape_button = tk.Button(buttons_frame, text="Scrape", state=tk.DISABLED, width=8, command=self.scrapeButtonAction) # self.download_video_button = tk.Button(buttons_frame, text="Download Video", state=tk.DISABLED, command=self.downloadVideoButtonAction) # self.download_external_button = tk.Button(buttons_frame, text="External Exercise", state=tk.DISABLED, command=self.downloadExternalButtonAction) # self.download_resource_button = tk.Button(buttons_frame, text="Resource", command=self.downloadResourceButtonAction) # self.download_image_button = tk.Button(buttons_frame, text="Images", command=self.downloadImageButtonAction) # self.download_attachment_button = tk.Button(buttons_frame, text="Attachments", command=self.downloadAttachmentButtonAction) # Status self.output_status_label = tk.Label(labelframe, text="", fg="green") padding_y = 3 output_folder_label.grid(row=0, column=0, padx=5, pady=padding_y + 2, sticky=tk.S + tk.N) output_folder_entry.grid(row=0, column=1, padx=5, pady=padding_y + 2, sticky=tk.S + tk.N) browse_button.grid(row=0, column=2, ipadx=5, padx=5, pady=padding_y + 2, sticky=tk.S + tk.N) checkboxes_frame.grid(row=1, column=1, pady=padding_y + 2, sticky=tk.S + tk.N + tk.W) buttons_frame.grid(row=2, column=1, padx=5, pady=padding_y, sticky=tk.W) self.output_status_label.grid(row=3, column=1, padx=5, pady=0, sticky=tk.W) # Checkboxes Frame get_video_check_box.grid(row=0, column=0, padx=5, sticky=tk.S + tk.N + tk.W) get_reading_check_box.grid(row=0, column=1, padx=5, sticky=tk.S + tk.N + tk.W) get_quiz_check_box.grid(row=0, column=2, padx=5, sticky=tk.S + tk.N + tk.W) get_graded_check_box.grid(row=0, column=3, padx=5, sticky=tk.S + tk.N + tk.W) get_external_check_box.grid(row=0, column=4, padx=5, sticky=tk.S + tk.N + tk.W) # Buttons Frame self.download_all_button.grid(row=0, column=0, ipadx="5", padx=5, pady=padding_y, sticky=tk.W) self.more_menu_button.grid(row=0, column=1, padx=5, pady=padding_y, sticky=tk.W) # self.scrape_button.grid(row=0, column=1, padx=5, pady=padding_y, sticky=tk.N) # self.download_video_button.grid(row=0, column=2, padx=5, pady=padding_y, sticky=tk.W) # self.download_external_button.grid(row=0, column=3, padx=5, pady=padding_y, sticky=tk.W) # self.download_resource_button.grid(row=0, column=4, padx=5, pady=padding_y, sticky=tk.W) # self.download_image_button.grid(row=0, column=5, padx=5, pady=padding_y, sticky=tk.W) # self.download_attachment_button.grid(row=0, column=6, padx=5, pady=padding_y, sticky=tk.W) def createStatusFrame(self, root): labelframe = ttk.LabelFrame(root, text="Status", width=600, height=100) labelframe.pack(fill="x", expand="yes") self.status_label_line_1 = tk.Label(labelframe, text="Downloading") self.status_label_line_2 = tk.Label(labelframe, text="Data") padding_y = 0 self.status_label_line_1.grid(row=0, column=0, padx=5, pady=padding_y, sticky=tk.W) self.status_label_line_2.grid(row=1, column=0, padx=5, pady=padding_y, sticky=tk.W) def createFileDownloaderFrame(self, root): labelframe = ttk.LabelFrame(root, text="File Downloader", width=600, height=500) labelframe.pack(fill="x", expand="yes") self.week_label = tk.Label(labelframe, text="", fg="green") self.topic_label = tk.Label(labelframe, text="", fg="green") filename_label = tk.Label(labelframe, text="Filename: ") self.filename_val_label = tk.Label(labelframe, text="") url_label = tk.Label(labelframe, text="Url: ") self.url_val_label = tk.Label(labelframe, text="") output_label = tk.Label(labelframe, text="Output: ") self.output_val_label = tk.Label(labelframe, text="") downloaded_label = tk.Label(labelframe, text="Downloaded: ") self.downloaded_val_label = tk.Label(labelframe, text="") size_label = tk.Label(labelframe, text="Size: ") self.size_val_label = tk.Label(labelframe, text="") speed_label = tk.Label(labelframe, text="Speed: ") self.speed_val_label = tk.Label(labelframe, text="") time_label = tk.Label(labelframe, text="Time left: ") self.time_val_label = tk.Label(labelframe, text="") total_label = tk.Label(labelframe, text="Total: ") self.total_val_label = tk.Label(labelframe, text="") # Progress bar widget self.progress_bar = ttk.Progressbar(labelframe, length=100, orient=tk.HORIZONTAL, mode='determinate') self.total_progress_bar = ttk.Progressbar(labelframe, length=100, orient=tk.HORIZONTAL, mode='determinate') # Status Bar self.status_bar_label = tk.Label(labelframe, text="Ready") # Button Group button_group_frame = tk.Frame(labelframe) self.pause_resume_btn = ttk.Button( button_group_frame, text="Pause", width=8, state=tk.DISABLED, command=self.pauseDownloadButtonAction) self.skip_btn = ttk.Button(button_group_frame, text="Skip", width=8, state=tk.DISABLED, command=self.skipDownloadButtonAction) self.cancel_btn = ttk.Button(button_group_frame, text="Cancel", width=8, state=tk.DISABLED, command=self.cancelDownloadButtonAction) padding_y = 0 self.week_label.grid(row=0, column=0, padx=5, pady=padding_y, sticky=tk.W) self.topic_label.grid(row=0, column=1, columnspan=3, padx=5, pady=padding_y, sticky=tk.W) filename_label.grid(row=1, column=0, padx=5, pady=padding_y, sticky=tk.W) self.filename_val_label.grid(row=1, column=1, columnspan=3, padx=5, pady=padding_y, sticky=tk.W) url_label.grid(row=2, column=0, padx=5, pady=padding_y, sticky=tk.W) self.url_val_label.grid(row=2, column=1, columnspan=3, padx=5, pady=padding_y, sticky=tk.W) output_label.grid(row=3, column=0, padx=5, pady=padding_y, sticky=tk.W) self.output_val_label.grid(row=3, column=1, columnspan=3, padx=5, pady=padding_y, sticky=tk.W) downloaded_label.grid(row=4, column=0, padx=5, pady=padding_y, sticky=tk.W) self.downloaded_val_label.grid(row=4, column=1, padx=5, pady=padding_y, sticky=tk.W) size_label.grid(row=4, column=2, padx=5, pady=padding_y, sticky=tk.W) self.size_val_label.grid(row=4, column=3, padx=5, pady=padding_y, sticky=tk.W) speed_label.grid(row=5, column=0, padx=5, pady=padding_y, sticky=tk.W) self.speed_val_label.grid(row=5, column=1, padx=5, pady=padding_y, sticky=tk.W) time_label.grid(row=5, column=2, padx=5, pady=padding_y, sticky=tk.W) self.time_val_label.grid(row=5, column=3, padx=5, pady=padding_y, sticky=tk.W) self.progress_bar.grid(row=6, column=0, columnspan=4, padx=5, pady=padding_y + 8, sticky=tk.W + tk.E) total_label.grid(row=7, column=0, padx=5, pady=padding_y, sticky=tk.W) self.total_val_label.grid(row=7, column=1, padx=5, pady=padding_y, sticky=tk.W) self.total_progress_bar.grid(row=8, column=0, columnspan=4, padx=5, pady=padding_y + 3, sticky=tk.W + tk.E) self.status_bar_label.grid(row=9, column=0, columnspan=3, padx=5, pady=padding_y + 3, sticky=tk.W) # Button Group Frame button_group_frame.grid(row=9, column=3, padx=5, pady=padding_y + 3, sticky=tk.E) self.pause_resume_btn.grid(row=0, column=0, ipadx="5", padx=5, pady=padding_y, sticky=tk.E) self.skip_btn.grid(row=0, column=1, ipadx="5", padx=5, pady=padding_y, sticky=tk.E) self.cancel_btn.grid(row=0, column=2, ipadx="5", padx=5, pady=padding_y, sticky=tk.E) labelframe.columnconfigure(1, weight=1) labelframe.columnconfigure(3, weight=2) ################################################################################################################### """" Button Handler Functions """ ################################################################################################################### def loadButtonAction(self): course_link = self.getCourseLink() if course_link == "": messagebox.showinfo(title="Information", message="Please enter course link") return if self.driver == None: self.driver = Driver("main") if self.bot == None: self.bot = Bot(self.driver, start_week=1, gui=self) self.bot.setCourseUrl(course_link) download_topics = self.getDownloadTopics() self.bot.setDownloadTopics(download_topics) Thread(target=self.runLoadMetaThread).start() def scrapeButtonAction(self): output_folder = self.getOutputFolder() if output_folder == "": messagebox.showinfo(title="Information", message="Please choose output folder") return if self.bot: self.bot.setOutputRoot(output_folder) else: messagebox.showinfo(title="Information", message="Bot not loaded!") download_topics = self.getDownloadTopics() self.bot.setDownloadTopics(download_topics) if self.meta_data: Thread(target=self.runDownloadHtmlAndGetVideoQueue).start() else: messagebox.showinfo( title="Information", message="Meta Data not loaded. Please load again") def downloadVideoButtonAction(self): if self.getOutputFolder() == "": messagebox.showinfo(title="Information", message="Please choose output folder") return if not self.download_queue_video: messagebox.showinfo(title="Information", message="Download Queue is Empty!") return Thread(target=self.runVideoDownloader).start() def downloadExternalButtonAction(self): output_folder = self.getOutputFolder() if output_folder == "": messagebox.showinfo(title="Information", message="Please choose output folder") return if not self.external_exercise_queue: messagebox.showinfo(title="Information", message="External Exercise Queue is Empty!") return Thread(target=self.runExternalExerciseDownloader).start() def downloadImageButtonAction(self): output_folder = self.getOutputFolder() if output_folder == "": messagebox.showinfo(title="Information", message="Please choose output folder") return if self.bot == None: self.bot = Bot(self.driver, gui=self) self.bot.setOutputRoot(output_folder) Thread(target=self.runImageDownloader).start() def downloadAttachmentButtonAction(self): output_folder = self.getOutputFolder() if output_folder == "": messagebox.showinfo(title="Information", message="Please choose output folder") return if self.bot == None: self.bot = Bot(self.driver, gui=self) self.bot.setOutputRoot(output_folder) Thread(target=self.runAttachmentDownloader).start() def downloadResourceButtonAction(self): course_link = self.getCourseLink() if course_link == "": messagebox.showinfo(title="Information", message="Please enter course link") return output_folder = self.getOutputFolder() if output_folder == "": messagebox.showinfo(title="Information", message="Please choose output folder") return if self.driver == None: self.driver = Driver("main") if self.bot == None: self.bot = Bot(self.driver, course_url=course_link, gui=self) self.bot.setOutputRoot(output_folder) # self.disableSkipButton(False) # self.disablePauseButton(False) # self.disableCancelButton(False) # self.disableDownloadVideoButton(True) Thread(target=self.runResourceDownloader).start() # self.downloadStatusLoop() def downloadAllButtonAction(self): course_link = self.getCourseLink() if course_link == "": messagebox.showinfo(title="Information", message="Please enter course link") return output_folder = self.getOutputFolder() if output_folder == "": messagebox.showinfo(title="Information", message="Please choose output folder") return if self.driver == None: self.driver = Driver("main") if self.bot == None: self.bot = Bot(self.driver, start_week=1, gui=self) self.bot.setCourseUrl(course_link) self.bot.setOutputRoot(output_folder) download_topics = self.getDownloadTopics() self.bot.setDownloadTopics(download_topics) if self.bot == None: self.bot = Bot(self.driver, gui=self) self.bot.setOutputRoot(output_folder) Thread(target=self.runAllDownloader).start() def pauseDownloadButtonAction(self): if self.file_downloader: if self.file_downloader.pause(): self.pause_resume_btn.config(text="Resume") self.pause_resume_btn.config( command=self.resumeDownloadButtonAction) def resumeDownloadButtonAction(self): if self.file_downloader: if self.file_downloader.resume(): self.pause_resume_btn.config(text="Pause") self.pause_resume_btn.config( command=self.pauseDownloadButtonAction) def skipDownloadButtonAction(self): if self.file_downloader: self.file_downloader.stop() def cancelDownloadButtonAction(self): if self.file_downloader: if self.file_downloader.stopQueue(): self.disablePauseButton(True) self.disableCancelButton(True) self.disableSkipButton(True) self.resetFileDownloaderInfo() self.disableDownloadVideoButton(False) ################################################################################################################### """" Getter Functions """ ################################################################################################################### def getCourseLink(self): return self.course_link_entry.get() def getOutputFolder(self): return os.path.normpath(self.output_folder_var.get() ) if self.output_folder_var.get() != "" else "" def getDownloadTopics(self): get_video = True if self.get_video_check_var.get() else False get_reading = True if self.get_reading_check_var.get() else False get_quiz = True if self.get_quiz_check_var.get() else False get_graded_assignment = True if self.get_graded_check_var.get( ) else False get_external_exercise = True if self.get_external_check_var.get( ) else False return get_video, get_reading, get_quiz, get_graded_assignment, get_external_exercise ################################################################################################################### """" Thread Functions """ ################################################################################################################### def runLoadMetaThread(self, silent=False): # Disable Buttons self.disableIOButtons() self.disableDownloadButtons() self.setInputStatus("Loading coursera...", color="grey") self.meta_data = self.bot.loadMeta() if self.meta_data: print(self.meta_data) item_count = 0 for week in self.meta_data: for topic in week['topics']: item_count += len(topic['items']) self.setInputStatus("Loaded {} weeks with {} topics..".format( len(self.meta_data), item_count)) if not silent: messagebox.showinfo(title="Information", message="Meta data loaded!") # Enable Buttons self.disableIOButtons(False) return True else: messagebox.showwarning("Warning", "Canceled") # Enable Buttons self.disableIOButtons(False) return False def runDownloadHtmlAndGetVideoQueue(self, silent=False): self.setOutputStatus("Scraping...", color="blue") # Disable Buttons self.disableIOButtons() self.disableDownloadButtons() result = self.bot.downloadHtmlAndGetVideoQueue(self.meta_data) self.download_queue_video, self.download_queue_captions, self.external_exercise_queue = result print(self.download_queue_captions) print(self.download_queue_video) self.setOutputStatus( "Downloaded HTMLs & Loaded {} video(s), {} external exercise in the queue" .format(len(self.download_queue_video), len(self.external_exercise_queue)), color="green") # Enable Buttons self.disableIOButtons(False) if not silent: messagebox.showinfo( title="Information", message="HTML Downloaded and Video Download Queue Generated") def runVideoDownloader(self, silent=False): self.downloadCaptions() self.downloadVideos() if not silent: self.showMessageDialog("Video Download Complete!") self.setFileDownloaderText(week="Complete", topic="Video Download Complete!", color="green") self.disableIOButtons(False) self.disableDownloadButtons() utils.log("Video Download Complete!") def runResourceDownloader(self, silent=False): # Disable Buttons self.disableIOButtons() self.disableDownloadButtons() self.bot.downloadResources() if not silent: messagebox.showinfo(title="Information", message="Resource Download Complete!") # Enable Buttons self.disableIOButtons(False) def runExternalExerciseDownloader(self, silent=False): # Disable Buttons self.disableIOButtons() self.disableDownloadButtons() self.bot.downloadExternalExercise() if not silent: messagebox.showinfo(title="Information", message="External Exercise Download Complete!") # Enable Buttons self.disableIOButtons(False) def runImageDownloader(self, silent=False): # Disable Buttons self.disableIOButtons() self.disableDownloadButtons() self.bot.downloadImages() if not silent: messagebox.showinfo(title="Information", message="Image(s) Download Complete!") # Enable Buttons self.disableIOButtons(False) def runAttachmentDownloader(self, silent=False): # Disable Buttons self.disableIOButtons() self.disableDownloadButtons() self.bot.downloadAttachments() if not silent: messagebox.showinfo(title="Information", message="Attachment(s) Download Complete!") # Enable Buttons self.disableIOButtons(False) def runAllDownloader(self): # Disable Buttons self.disableIOButtons() self.disableDownloadButtons() total_steps = 8 current_step = 1 self.setStatusBarText("Step {} / {} : Loading metadata".format( current_step, total_steps), color="blue") meta_downloaded = self.runLoadMetaThread(silent=True) if not meta_downloaded: self.setStatusBarText("Ready") return current_step += 1 self.setStatusBarText("Step {} / {} : Scrapping".format( current_step, total_steps), color="blue") self.runDownloadHtmlAndGetVideoQueue(silent=True) current_step += 1 self.setStatusBarText("Step {} / {} : Downloading Resource(s)".format( current_step, total_steps), color="blue") self.runResourceDownloader(silent=True) current_step += 1 self.setStatusBarText( "Step {} / {} : Downloading External Exercise(s)".format( current_step, total_steps), color="blue") self.runExternalExerciseDownloader(silent=True) current_step += 1 self.setStatusBarText("Step {} / {} : Downloading Image(s)".format( current_step, total_steps), color="blue") self.runImageDownloader(silent=True) current_step += 1 self.setStatusBarText( "Step {} / {} : Downloading Attachment(s)".format( current_step, total_steps), color="blue") self.runAttachmentDownloader(silent=True) current_step += 1 self.setStatusBarText("Step {} / {} : Downloading Caption(s)".format( current_step, total_steps), color="blue") if not self.download_queue_video: print("Video Download Queue is Empty!") else: self.downloadCaptions() current_step += 1 self.setStatusBarText("Step {} / {} : Downloading Video(s)".format( current_step, total_steps), color="blue") self.downloadVideos() self.setStatusBarText("Ready") self.showMessageDialog("All Download Complete!") self.setFileDownloaderText(week="Complete", topic="All Download Complete!", color="green") self.disableIOButtons(False) utils.log("All Download Complete!") # Other Thread Functions def downloadCaptions(self): # Disable Buttons self.disableIOButtons() self.disableDownloadButtons(False) root = self.getOutputFolder() if self.file_downloader == None: self.file_downloader = FileDownloader(root) self.file_downloader.attachGUI(self) self.file_downloader.loadQueueFromList(self.download_queue_captions) self.file_downloader.startDownloadGui() def downloadVideos(self): # Download Videos self.file_downloader.loadQueueFromList(self.download_queue_video) self.file_downloader.startDownloadGui() ################################################################################################################### """" Looper Functions """ ################################################################################################################### def downloadStatusLoop(self): if self.file_downloader: total_files, self.current_download_no, self.current_download_item, download_info = self.file_downloader.getDownloadInfo( ) if download_info: self.progress_bar['value'] = download_info['progress'] # self.total_progress_bar['value'] = 1 self.after(100, self.downloadStatusLoop) ################################################################################################################### """" GUI Functions """ ################################################################################################################### def setScrapProgress(self, download_info): item = download_info['item'] progress = download_info['progress'] current_no = download_info['current_no'] total_files = download_info['total_files'] week = item['path'].split("\\")[0] topic_text = "Topic: {}".format(item['path'].split("\\")[1]) filename = item['filename'] url = item['url'] output = download_info['full_path'] eta = download_info['eta'] speed = download_info['speed'] dl_size = download_info['dl_size'] file_size = download_info['total_size'] self.setFileDownloaderInfo(week=week, topic=topic_text, filename=filename, url=url, output=output, eta=eta, speed=speed, dl_size=dl_size, file_size=file_size, progress=progress, current_no=current_no, total_files=total_files) def setVideoDownloadProgress(self, download_info): item = download_info['item'] progress = download_info['progress'] current_no = download_info['current_no'] total_files = download_info['total_files'] week = item['path'].split("\\")[0] topic_text = "Topic: {}".format(item['path'].split("\\")[1]) filename = item['filename'] url = item['url'] output = download_info['full_path'] eta = download_info['eta'] speed = download_info['speed'] dl_size = download_info['dl_size'] file_size = download_info['total_size'] self.setFileDownloaderInfo(week=week, topic=topic_text, filename=filename, url=url, output=output, eta=eta, speed=speed, dl_size=dl_size, file_size=file_size, progress=progress, current_no=current_no, total_files=total_files) def setFileDownloaderInfo(self, week=None, topic=None, filename=None, url=None, output=None, eta=None, speed=None, dl_size=None, file_size=None, progress=None, current_no=0, total_files=0): total_text = "" total_progress = 0 if current_no > 0 and total_files > 0: prev_progress = (current_no - 1) / total_files * 100 single_progress = 1 / total_files * 100 total_progress = prev_progress + (single_progress * progress / 100) total_text = "{} of {} ( {:.2f} % )".format( current_no, total_files, total_progress) if week is not None: self.week_label.config(text=week, fg="blue") if topic is not None: self.topic_label.config(text=self.fitText(topic, 70), fg="blue") if filename is not None: self.filename_val_label.config(text=self.fitText(filename)) if url is not None: self.url_val_label.config(text=self.fitText(url)) if output is not None: self.output_val_label.config(text=self.fitText(output)) if eta is not None: self.time_val_label.config(text=eta) if speed is not None: self.speed_val_label.config(text=speed) if dl_size is not None: downloaded_text = "{} ( {:.2f} % )".format( dl_size, progress) if dl_size != "" else "" self.downloaded_val_label.config(text=downloaded_text) if file_size is not None: self.size_val_label.config(text=file_size) if total_text is not None: self.total_val_label.config(text=total_text) if progress is not None: self.progress_bar['value'] = progress if total_progress is not None: self.total_progress_bar['value'] = total_progress def setFileDownloaderText(self, week=None, topic=None, color="black"): if week is not None: self.week_label.config(text=week, fg=color) if topic is not None: self.topic_label.config(text=self.fitText(topic, 70), fg=color) def resetFileDownloaderInfo(self): self.setFileDownloaderInfo(week="", topic="", filename="", url="", output="", eta="", speed="", dl_size="", file_size="", progress=0, current_no=0, total_files=0) def setStatusBarText(self, text, color="black"): self.status_bar_label.config(text=text, fg=color) # Button Disable def disableButton(self, button, val=True): if val: button.config(state="disabled") else: button.config(state="normal") def disableButtons(self, *argv, val=True): for button in argv: if val: button.config(state="disabled") else: button.config(state="normal") def disableDownloadVideoButton(self, val): if val: self.download_video_button.config(state="disabled") else: self.download_video_button.config(state="normal") def disablePauseButton(self, val): if val: self.pause_resume_btn.config(state="disabled") else: self.pause_resume_btn.config(state="normal") def disableSkipButton(self, val): if val: self.skip_btn.config(state="disabled") else: self.skip_btn.config(state="normal") def disableCancelButton(self, val): if val: self.cancel_btn.config(state="disabled") else: self.cancel_btn.config(state="normal") def disableIOButtons(self, val=True): self.disableButtons(self.load_button, self.download_all_button, self.more_menu_button, val=val) def disableDownloadButtons(self, val=True): self.disableButtons(self.pause_resume_btn, self.skip_btn, self.cancel_btn, val=val) # Show Dialogs def showDownloadCompleteDialog(self, msg=None): if msg is None: msg = "Download complete!" messagebox.showinfo(title="Information", message=msg) self.disableIOButtons(False) self.disableDownloadButtons(True) def showMessageDialog(self, msg, title="Information"): messagebox.showinfo(title=title, message=msg) def askOkCancelDialog(self, msg, title="Information"): return messagebox.askokcancel(title=title, message=msg) # Set Status def setInputStatus(self, text, color="green"): self.input_status_label.config(text=text, fg=color) def setOutputStatus(self, text, color="green"): self.output_status_label.config(text=text, fg=color) ################################################################################################################### """" Utility Functions """ ################################################################################################################### def fitText(self, text, width=100): if text and width and len(text) > width: text = text[:width - 3] + '...' return text ################################################################################################################### """" Other Functions """ ################################################################################################################### def destroy(self): super().destroy() if self.driver is not None: print("Closing Browser...") self.driver.closeBrowser() print("Closed Browser")