def __crawljobs(self, keyword, city=u'全国'): page_num = 1 url = u'http://www.lagou.com/jobs/positionAjax.json?px=default&city=%s&needAddtionalResult=false' % (city,) page_size = sys.maxint errors = [] while page_num < page_size: data = u'first=false&pn=%d&kd=%s' % (page_num, keyword) cache, content = Downloader.get_html(url, data, city) if content: try: result = json.loads(content) page_no = result['content']['pageNo'] if page_no > 0: print '{kv}: {page_num}'.format(kv=keyword, page_num=page_num) page_num += 1 if not cache: # 每个网络请求间隔一秒 sleep(0.1) else: # 为空,说明最后一页 break except Exception, e: page_num += 1 # 发生异常, 删除缓存文件 Downloader.remove_file(url, data, city) errors.append((1, url, data, city, e)) else: # 请求结果为空 errors.append((2, url, data, city, e))
def applyGloablBlackList(cfg_params): removeBList = cfg_params.get("GRID.remove_default_blacklist", 0 ) blackAnaOps = None if int(removeBList) == 0: try: blacklist = Downloader("http://cmsdoc.cern.ch/cms/LCG/crab/config/") result = blacklist.config("site_black_list.conf") except: msg = "ERROR talking to cmsdoc\n%s"%str(sys.exc_info()[1]) common.logger.info(msg) common.logger.info("WARNING: Could not download default site black list") result = [] if result : blackAnaOps = result.strip() common.logger.debug("Enforced black list: %s "%blackAnaOps) else: common.logger.info("WARNING: Skipping default black list!") if blackAnaOps: blackUser = cfg_params.get("GRID.se_black_list", '' ) if blackUser: cfg_params['GRID.se_black_list'] = blackUser + ',' + blackAnaOps else: cfg_params['GRID.se_black_list'] = blackAnaOps msg = "PSN black list: %s" % cfg_params['GRID.se_black_list'] common.logger.info(msg)
def configure(self, cfg_params): self.cfg_params = cfg_params self.jobtypeName = cfg_params.get('CRAB.jobtype','') self.schedulerName = cfg_params.get('CRAB.scheduler','') Scheduler.configure(self,cfg_params) #self.proxyValid=0 #self.dontCheckProxy=int(cfg_params.get("GRID.dont_check_proxy",0)) self.space_token = cfg_params.get("USER.space_token",None) self.proxyServer= 'myproxy.cern.ch' #self.group = cfg_params.get("GRID.group", None) #self.role = cfg_params.get("GRID.role", None) removeBList = cfg_params.get("GRID.remove_default_blacklist", 0 ) blackAnaOps = None if int(removeBList) == 0: blacklist = Downloader("http://cmsdoc.cern.ch/cms/LCG/crab/config/") result = blacklist.config("site_black_list.conf") if result != None: blackAnaOps = result common.logger.debug("Enforced black list: %s "%blackAnaOps) else: common.logger.info("WARNING: Skipping default black list!") self.EDG_ce_black_list = None if cfg_params.has_key('GRID.ce_black_list') and cfg_params['GRID.ce_black_list']: self.EDG_ce_black_list = cfg_params.get('GRID.ce_black_list') if int(removeBList) == 0 and blackAnaOps: self.EDG_ce_black_list += ",%s"%blackAnaOps elif int(removeBList) == 0 and blackAnaOps: self.EDG_ce_black_list = blackAnaOps if self.EDG_ce_black_list: self.EDG_ce_black_list = str(self.EDG_ce_black_list).split(',') self.EDG_ce_white_list = cfg_params.get('GRID.ce_white_list',None) if (self.EDG_ce_white_list): self.EDG_ce_white_list = str(self.EDG_ce_white_list).split(',') self.VO = cfg_params.get('GRID.virtual_organization','cms') self.EDG_clock_time = cfg_params.get('GRID.max_wall_clock_time',None) # Default minimum CPU time to >= 130 minutes self.EDG_cpu_time = cfg_params.get('GRID.max_cpu_time', '130') ## Add EDG_WL_LOCATION to the python path #if not self.CRAB_useServer and not self.CRAB_serverName: # if not os.environ.has_key('EDG_WL_LOCATION'): # msg = "Error: the EDG_WL_LOCATION variable is not set." # raise CrabException(msg) # path = os.environ['EDG_WL_LOCATION'] # libPath=os.path.join(path, "lib") # sys.path.append(libPath) # libPath=os.path.join(path, "lib", "python") # sys.path.append(libPath) self.checkProxy() return
def runProgram(youtube_search_query, text_to_find): dl = Downloader() celeb_videos = Search.search(youtube_search_query) links = [Caption.YOUTUBE_LINK + x for x in celeb_videos] print links count = 0 for videoid in celeb_videos: cap = Caption(videoid) results = cap.find(text_to_find) for result in results: dl.download(result, text_to_find + str(count)) count += 1
def manageCredential(self): """ Prepare configuration and Call credential API """ common.logger.info("Registering credential to the server : %s"%self.server_name) try: myproxyserver = Downloader("http://cmsdoc.cern.ch/cms/LCG/crab/config/").config("myproxy_server.conf") myproxyserver = myproxyserver.strip() if myproxyserver is None: raise CrabException("myproxy_server.conf retrieved but empty") except Exception, e: common.logger.info("Problem setting myproxy server endpoint: using myproxy.cern.ch") common.logger.debug(e) myproxyserver = 'myproxy.cern.ch'
def insert_jobs(self): errors = [] types = self.get_types() for keyword in types: page_num = 1 city = u'全国' url = u'http://www.lagou.com/jobs/positionAjax.json?px=default&city=%s&needAddtionalResult=false' % (city,) page_size = sys.maxint while page_num < page_size: data = u'first=false&pn=%d&kd=%s' % (page_num, keyword) cache, content = Downloader.get_html(url, data, city) if content: try: result = json.loads(content) pos_result = result['content']['positionResult'] jobs = pos_result['result'] if len(jobs) > 0: self.db.insert_jobs(jobs) print data else: # 为空说明是最后一页 errors.append('result empty {0} {1}'.format(url, data, content)) break page_num += 1 except Exception, e: page_num += 1 # 发生异常 errors.append((1, url, data, city, e)) else: # 请求结果为空 errors.append((2, url, data, city, e))
def craw(self, root_url): # todo: 层数设置? page_count = self.page_count UrlManager.add_new_url(root_url) try: while UrlManager.has_new_url(): todo_url = UrlManager.get_new_url() try: print u"\nNO: %d 正在检测链接:%s" % (page_count, todo_url) except: print u"出错", todo_url # todo: 换成 contiune 是不是更好? # page_status, bad_links = Downloader.test(todo_url) # print u"\n结果:%s" % page_status # todo: 加入状态码的说明 state, content = Downloader.download(todo_url) if state: prase_state, new_urls = Parser.parse(todo_url, content) if prase_state: UrlManager.add_new_urls(new_urls) else: Outputer.collect_data(new_urls) else: Outputer.collect_data(content) page_count += 1 # except: # print u"页面爬取失败" # UrlManager.add_wrong_url(todo_url) # todo:测试代码 if page_count == 5000: self.page_count = page_count break print UrlManager.num_new_url() finally: Outputer.output_txt(self.page_count)
def infomation_crew(self): # 一条一条取 total_count = 20000 i = 0 while i < total_count: i += 1 # 去除诗词的详情页url info = self.db.select_unanalyzed_infomation() if info is not None: # 下载分析 url = info['url'] html = Downloader.get_html(url, 'infomation') if html: content = Analyzer.get_info_detail(html) if content: self.db.update_infomationurl(url, content) print '%d/%d %s %s' % (i, total_count, info['title'], url) else: self.db.insert_error('analyze_info_detail_error', 7, 'reason', url) else: self.db.insert_error('download_info_detail_error', 6, 'reason', url) else: # 没有了 return
def _createXMLcommand( self, taskUName, cmdSpec="status", rng="all", newTaskAddIns=False, flavour="analysis", type="fullySpecified", jobs="-1", ): xmlString = "" cfile = minidom.Document() ver = common.prog_version_str node = cfile.createElement("TaskCommand") node.setAttribute("Task", str(taskUName)) node.setAttribute("Subject", str(self.userSubj)) node.setAttribute("Command", str(cmdSpec)) node.setAttribute("Range", str(rng)) node.setAttribute("TotJob", str(jobs)) node.setAttribute("Scheduler", str(self.cfg_params["CRAB.scheduler"])) node.setAttribute("Flavour", str(flavour)) node.setAttribute("Type", str(type)) node.setAttribute("ClientVersion", str(ver)) ## Only Temporary. it should be at Server level removeBList = self.cfg_params.get("GRID.remove_default_blacklist", 0) blackAnaOps = "None.Nowhere" if int(removeBList) == 0: blacklist = Downloader("http://cmsdoc.cern.ch/cms/LCG/crab/config/") result = blacklist.config("site_black_list.conf") if result != None: blackAnaOps = result common.logger.debug("Enforced black list: %s " % str(blacklist)) else: common.logger.info("WARNING: Skipping default black list!") try: myproxyserver = Downloader("http://cmsdoc.cern.ch/cms/LCG/crab/config/").config("myproxy_server.conf") myproxyserver = myproxyserver.strip() if myproxyserver is None: raise CrabException("myproxy_server.conf retrieved but empty") except Exception, e: common.logger.info("Problem setting myproxy server endpoint: using myproxy.cern.ch") common.logger.debug(e) myproxyserver = "myproxy.cern.ch"
def __init__(self, serverName): import string self.serverName = string.lower(serverName) common.logger.debug('Calling ServerConfig '+self.serverName) url ='http://cmsdoc.cern.ch/cms/LCG/crab/config/' self.downloader = Downloader(url)
def rb_configure(self, RB): url ='http://cmsdoc.cern.ch/cms/LCG/crab/config/' from Downloader import Downloader import httplib common.logger.debug('Downloading config files for WMS: '+url) ## 25-Jun-2009 SL: patch to use Cream enabled WMS if ( self.cfg_params.get('GRID.use_cream',None) ): RB='CREAM' if not RB: return None rb_param_file = None configFileName = 'glite_wms_'+str(RB)+'.conf' results = Downloader(url) try: gliteConfig = results.filePath(configFileName) except httplib.HTTPException, ex: raise CrabException( "Problem getting RB config file: %s, reason:"%(configFileName, ex) )
def poem_list_crew(self): for i in range(1, 6): url = 'http://www.haoshiwen.org/type.php?x=%d' % i content = Downloader.get_html(url, 'poemlist') if content: page_count = Analyzer.get_page_count(content) # 分析 for j in range(1, page_count + 1): page_url = 'http://www.haoshiwen.org/type.php?x=%d&page=%d' % (i, j) # 入库 self.db.insert_url(page_url, 1) # 判断是否分析过 if self.db.url_analyzed(page_url): pass else: content = Downloader.get_html(page_url, 'poemlist') if content: # 分析诗的列表 poems = Analyzer.get_poems_from_list_page(content) if poems: # 入库 self.db.insert_urls(poems, 2) self.db.update_url(page_url) print '%d %d/%d: %s' % (i, j, page_count, page_url) else: if Analyzer.check_poem_list_last_page(content): # 最后一页 break else: print u'分析失败' self.db.insert_error('analyze_poem_list_error', 3, 'reason', page_url) # 错误入库:analyze_poem_list_error else: print u'获取页面诗词列表错误' self.db.insert_error('get_poem_list_error', 2, 'reason', page_url) # 错误入库:get_poem_list_error else: print u'分析首页失败' self.db.insert_error('analyze_poem_list_first_page_error', 1, 'reason', page_url)
def parseAnswerServer(self, updateInfo): updateDict = parse_qs(updateInfo, strict_parsing=True) # updateDict has: mustUpdate, urlUser, newVersionUser, urlAdmin, # newVersionAdmin updates = dict() # updates has: mustUpdate, userFile, adminFile mustUpdate = int(updateDict['mustUpdate'][0]) updates['mustUpdate'] = mustUpdate downloader = Downloader() location = "../../persistent/downloads" if mustUpdate == 0: logger.info('No update available') elif (mustUpdate & UPDATE_ADMIN_MODE): adminURL = updateDict['urlAdmin'][0] logger.info('Downloading Admin update: %s' % adminURL) adminFile = downloader.downloadUpdate(location, adminURL) updates['adminFile'] = adminFile logger.info('Administrator update is available called: %s' % adminFile) if DISPLAY_GUI_MESSAGES and not(checkFiles.checkIfAdmin()): root = Tk() root.title('HiSPARC') Message(root, anchor='s', text="Update is available requiring " "administrator rights!\nPlease ask your administrator " "to reboot and install it!").pack(padx=150, pady=100) root.mainloop() elif (mustUpdate & UPDATE_USER_MODE): userURL = updateDict['urlUser'][0] logger.info('Downloading User update: %s' % userURL) userFile = downloader.downloadUpdate(location, userURL) updates['userFile'] = userFile logger.info('User update is available called: %s' % userFile) # Run the update to install it. # First call a batch file so that Python can be closed. os.system(".\\runUserUpdate.bat %s" % userFile) return updates
class Dispatcher(object): """ Dispatcher class to control process of working. """ def __init__(self): create_or_check_path("logs/") fileConfig('config/logger.conf') self.database = Database() self.parser = Parser() self.event_processor = EventProcessor() self.downloader = Downloader() self.new_events_file = None self.new_data = None self.new_event = None def processor(self): """ Process of downloading, parsing and saving information. """ download_status = True while download_status: getting_event_status = self.get_event() if getting_event_status: processing_event_status = self.process_event() if processing_event_status: self.process_data() else: download_status = self.download() def get_event(self): """ Asks Parser for a new event. :return: False if getting event failed else True """ self.new_event = self.parser.get_event(self.new_events_file) return False if self.new_event is None else True def download(self): """ Sends a request to Downloader. :return: False if downloading failed else True """ self.new_events_file = self.downloader.download_archive() return False if self.new_events_file is None else True def process_event(self): """ Asks EventProcessor to get data from event. :return: False if processing event failed else True """ self.new_data = self.event_processor.process_event(self.new_event) return False if self.new_data is None else True def process_data(self): """ Sends data to Database. """ self.database.process_data(self.new_data)
def poem_type_crew(self): url = 'http://www.haoshiwen.org/type.php' content = Downloader.get_html(url, 'poemlist') if content: result = Analyzer.get_poem_types(content) k = 1 for i in result: # 只处理类型 if i[0] == u'类型': for j in i[1]: self.db.insert_poem_types(k, i[0], j[0], j[1]) print '\t' + j[0] k += 1
def __init__(self): create_or_check_path("logs/") fileConfig('config/logger.conf') self.database = Database() self.parser = Parser() self.event_processor = EventProcessor() self.downloader = Downloader() self.new_events_file = None self.new_data = None self.new_event = None
def main(): log = setup_logging("sentinal.log") parser = argparse.ArgumentParser(description='Grabs sentinal images from the ESA stite', epilog='-h for help', formatter_class=argparse.RawTextHelpFormatter) parser.add_argument('-c', action="store_true", help='create db table only', dest='create_db') parser.add_argument('-C', type=str, help='path to the configuration file', dest='configuration_file') parser.add_argument('-f', action="store_true", help='force', dest='force') parser.add_argument('-v', action="store_true", help='run verbosely', dest='verbose') parser.add_argument('-o', action="store_true", help='overwrite data .zip file even if it exists', dest='overwrite') args = parser.parse_args() db = Database(args) if args.create_db: db.createDb(args.verbose) sys.exit(0) config = ConfigManager(args.configuration_file,log) auth = config.getAuth() if not len(auth): log.error('Missing ESA SCIHUB authentication information') sys.exit(7) types = config.getTypes() polygons = config.getPolygons() if len(types) != len(polygons): log.error( 'Incorrect number of polygons and types in configuration file' ) sys.exit(6) downloader = Downloader(args, config, db, log) downloader.getProducts() downloader.downloadProducts()
def configure(self, cfg_params): self.cfg_params = cfg_params self.jobtypeName = cfg_params.get('CRAB.jobtype','') self.schedulerName = cfg_params.get('CRAB.scheduler','') Scheduler.configure(self,cfg_params) self.proxyValid=0 self.dontCheckProxy=int(cfg_params.get("GRID.dont_check_proxy",0)) self.space_token = cfg_params.get("USER.space_token",None) try: self.proxyServer = Downloader("http://cmsdoc.cern.ch/cms/LCG/crab/config/").config("myproxy_server.conf") self.proxyServer = self.proxyServer.strip() if self.proxyServer is None: raise CrabException("myproxy_server.conf retrieved but empty") except Exception, e: common.logger.info("Problem setting myproxy server endpoint: using myproxy.cern.ch") common.logger.debug(e) self.proxyServer= 'myproxy.cern.ch'
def poem_detail_crew(self): # update urls set analyzed = 1 where url = 'http://www.haoshiwen.org/view.php?id=9510' # 一条一条取 total_count = 10 c = 1 while c < total_count: c += 1 continue # 去除诗词的详情页url url = self.db.select_unanalyzed_url(2) if url is not None: url = url['url'] # 判断是否抓取过 if not self.db.poem_exists(url): content = Downloader.get_html(url, 'poem') if content: try: poem = Analyzer.get_poem_detail(content, url) if poem: # 分析成功,入库 poem_content = poem[0] poem_info = poem[1] self.db.insert_poem(poem_content) self.db.insert_infomations(url, 1, poem_info) self.db.update_url(url) print "%d/%d %s %s" % (c, total_count, poem_content['title'], poem_content['url']) else: self.db.insert_error('analyze_poem_detail_error', 5, 'reason', url) except Exception, e: continue else: self.db.insert_error('get_poem_detail_error', 4, 'reason', url) else: self.db.update_url(url) else: count = total_count
def poem_author_crew(self): # 一条一条取 total_count = 20000 i = 0 while i < total_count: i += 1 # 去除诗词的详情页url authors = self.db.select_authors() cou = len(authors) i = 1 for author in authors: url = author[0] if url is not None and url is not '': # 判断是否存在 if not self.db.author_exists(url): html = Downloader.get_html(url, 'author') if html: try: author_info = Analyzer.get_author_detail(html, url) if author_info: author_content = author_info[0] author_infos = author_info[1] self.db.insert_author(author_content) self.db.insert_infomations(url, 2, author_infos) print '%d/%d %s %s' % (i, cou, author_content['name'], url) i += 1 else: self.db.insert_error('analyze_author_detail_error', 9, 'reason', url) except Exception, e: print 'error %s' % (url,) else: self.db.insert_error('download_author_detail_error', 8, 'reason', url) else: # 没有了 return
def poem_type_poem_list_craw(self): types = self.db.get_poem_types() for i in types: j = 1 while True: page_url = i['url'] + "&page=" + str(j) # 入库 self.db.insert_url(page_url, 3) # 判断是否分析过 if self.db.url_analyzed(page_url): j += 1 continue else: content = Downloader.get_html(page_url, 'poemlist') if content: # 分析诗的列表 poems = Analyzer.get_poems_from_list_page(content) if poems: # 入库 self.db.insert_type_poems(i['id'], poems) self.db.update_url(page_url) print '%d: %s' % (j, page_url) j += 1 else: if Analyzer.check_poem_list_last_page(content): # 最后一页 break else: print u'分析失败' self.db.insert_error('analyze_poem_list_error', 3, 'reason', page_url) # 错误入库:analyze_poem_list_error else: print u'获取页面诗词列表错误' self.db.insert_error('get_poem_list_error', 2, 'reason', page_url)
def MainProcess(logger, uperList, saveRootPath, concurrency=3): pp = None try: # -------------------------------------------------------------- # 进行每个 UP 主视频页数的获取 pp = PreProcess(logger=logger, uperList=uperList) pp.ScanLoclInfo(saveRootPath) pp.Process() # -------------------------------------------------------------- # 爬取要下载视频的 url for uper in pp.uperList: logger.info(uper.UserName + " Spider Start···") OneSpiderRetryTimes = 0 # 打算下载的数量,要去网络动态获取的数量进行对比 while ((uper.NeedDownloadFilmCount > len( uper.VideoInfoDic_NetFileName) or len(uper.ErrorUrl_Dic) > 0) and OneSpiderRetryTimes <= 10): # dd = BiliSpider() # GithubDeveloperSpider BiliSpider.start(logger=logger, uper=uper, saveRootPath=saveRootPath, concurrency=concurrency, middleware=middleware) OneSpiderRetryTimes = OneSpiderRetryTimes + 1 logger.info("Try Spider " + uper.UserName + " " + str(OneSpiderRetryTimes) + " times.") RandomSleep() logger.info(uper.UserName + " Spider Done.") if OneSpiderRetryTimes > 10: logger.error(uper.UserName + " Spider Retry " + str(OneSpiderRetryTimes) + "times.") logger.error("Error Url:") for eUrl in uper.ErrorUrl_Dic: logger.error(eUrl) else: # 本地应该原有+准备要下载的 != 网络总数,需要提示 if len(uper.VideoInfoDic_NetFileName) != len( uper.VideoInfoDic_loaclFileName): logger.warn("VideoInfoDic_NetFileName Count: " + str(len(uper.VideoInfoDic_NetFileName)) + " != VideoInfoDic_loaclFileName Count: " + str(len(uper.VideoInfoDic_loaclFileName))) uper.ErrorUrl_Dic.clear() logger.info("Spider All Done.") # -------------------------------------------------------------- logger.info("Start Download" + "----" * 20) # 开始下载 # 先对 local 与 net 的字典进行同步 logger.info("Start Sync Dic") for uper in pp.uperList: iNeedDl = 0 for fileName, oneVideo in zip( uper.VideoInfoDic_loaclFileName.keys(), uper.VideoInfoDic_loaclFileName.values()): # 匹配到对应的项目才进行处理 findList = list( filter(lambda d: d.split('_')[1] == fileName, uper.VideoInfoDic_NetFileName.keys())) if any(findList): uper.VideoInfoDic_NetFileName[ findList[0]].isDownloaded = oneVideo.isDownloaded if oneVideo.isDownloaded == False: iNeedDl = iNeedDl + 1 logger.info(uper.UserName + "NetFile / LocalFile -- NeedDl: " + str(len(uper.VideoInfoDic_NetFileName)) + " / " + str(len(uper.VideoInfoDic_loaclFileName)) + " -- " + str(iNeedDl)) logger.info("End Sync Dic") for uper in pp.uperList: directory = os.path.join(saveRootPath, uper.UserName) for fileName, oneVideo in zip( uper.VideoInfoDic_NetFileName.keys(), uper.VideoInfoDic_NetFileName.values()): if oneVideo.isDownloaded == True: continue DownloadRetryTimes = 0 oneRe = False while oneRe is False and DownloadRetryTimes <= 10: oneRe = Downloader(logger, directory, oneVideo.time, oneVideo.title, oneVideo.url).ProcessOne() DownloadRetryTimes = DownloadRetryTimes + 1 logger.info("Try Download " + str(DownloadRetryTimes) + " times.") RandomSleep() if OneSpiderRetryTimes > 10: logger.error("Retry Download " + str(DownloadRetryTimes) + " times.") logger.error("Error Url: " + oneVideo.url) # 标记下载完成 if oneRe: oneVideo.isDownloaded = True uper.ThisTimeDownloadCount = uper.ThisTimeDownloadCount + 1 except Exception as ex: errInfo = "Catch Exception: " + str(ex) logger.error(errInfo) finally: logger.info("finally" + "----" * 20) for uper in pp.uperList: logger.info("This Time Download: " + uper.UserName + " -- " + str(uper.ThisTimeDownloadCount)) for uper in pp.uperList: for fileName, oneVideo in zip( uper.VideoInfoDic_NetFileName.keys(), uper.VideoInfoDic_NetFileName.values()): if oneVideo.isDownloaded == False: logger.error('Download Fail:' + uper.UserName) logger.error(oneVideo.url) logger.info("This Time Done.")
def download_callback(params): downloader = Downloader(params['url']) downloader.start()
class MainWindow( QtGui.QMainWindow ): ## Constructeur # Le constructeur va creer la fenetre principale en y ajoutant tous les widgets necessaires au programme def __init__( self ): # Appel au constructeur de la classe mere QtGui.QMainWindow.__init__( self ) ########### # Fenetre # ########### ### # Reglages de la fenetre principale ### # Nom de la fenetre self.setWindowTitle( "TVDownloader" ) # Mise en place de son icone self.setWindowIcon( QtGui.QIcon( "ico/TVDownloader.png" ) ) ### # Mise en place des widgets dans la fenetre ### # Widget central qui contiendra tout self.centralWidget = QtGui.QWidget( self ) # # Barre du haut # # Layout horizontal qui contiendra les listes deroulantes self.horizontalLayoutBarreHaut = QtGui.QHBoxLayout() # Liste deroulante pour choisir le site (plugin) self.comboBoxSite = QtGui.QComboBox( self.centralWidget ) self.horizontalLayoutBarreHaut.addWidget( self.comboBoxSite ) # Liste deroulante pour choisir une chaine du site courant self.comboBoxChaine = QtGui.QComboBox( self.centralWidget) self.horizontalLayoutBarreHaut.addWidget( self.comboBoxChaine ) # Liste deroulante pour choisir une emission de la chaine courante self.comboBoxEmission = QtGui.QComboBox( self.centralWidget ) self.horizontalLayoutBarreHaut.addWidget( self.comboBoxEmission ) # # Onglets # # Gestionnaire onglets self.tabWidget = QtGui.QTabWidget( self.centralWidget ) # Onglet Fichiers self.tabFichiers = QtGui.QSplitter( self.centralWidget ) # L'onglet Fichier contient un splitter self.tabWidget.addTab( self.tabFichiers, u"Choix des fichiers" ) # Onglet Telechargements self.tabTelechargements = QtGui.QWidget( self.centralWidget ) self.tabWidget.addTab( self.tabTelechargements, u"Téléchargements" ) # # Liste des fichiers # # Layout de grille qui contient le tableau qui liste les fichiers + boutons self.gridLayoutFichiers = QtGui.QGridLayout( self.tabFichiers ) # Tableau qui contient la liste des fichiers disponibles pour l'emission courante self.tableWidgetFichier = MyQTableWidget( self.tabFichiers ) # Il a 4 colonnes et 0 ligne (pour l'instant) self.tableWidgetFichier.setColumnCount( 3 ) self.tableWidgetFichier.setRowCount( 0 ) # On ajoute les titres self.tableWidgetFichier.setHorizontalHeaderItem( 0, self.tableWidgetFichier.creerItem( "" ) ) self.tableWidgetFichier.setHorizontalHeaderItem( 1, self.tableWidgetFichier.creerItem( "Date" ) ) self.tableWidgetFichier.setHorizontalHeaderItem( 2, self.tableWidgetFichier.creerItem( "Emission" ) ) # On l'ajoute au layout self.gridLayoutFichiers.addWidget( self.tableWidgetFichier, 0, 1, 6, 1 ) # Icones du tableWidget self.iconeFichier = QtGui.QIcon( "ico/gtk-file.svg" ) self.iconeAjoute = QtGui.QIcon( "ico/gtk-add.svg" ) self.iconeTelecharge = QtGui.QIcon( "ico/gtk-apply.svg" ) # Bouton pour ajouter tous les fichiers a la liste des telechargements self.pushButtonToutAjouter = MyQPushButton( self.tabFichiers ) self.pushButtonToutAjouter.setIcon( QtGui.QIcon( "ico/gtk-add.svg" ) ) self.pushButtonToutAjouter.setToolTip( u"Ajouter tous les fichiers à la liste des téléchargements" ) self.gridLayoutFichiers.addWidget( self.pushButtonToutAjouter, 0, 0, 2, 1 ) # Bouton pour rafraichir le plugin courant self.pushButtonRafraichirPlugin = MyQPushButton( self.tabFichiers ) self.pushButtonRafraichirPlugin.setIcon( QtGui.QIcon( "ico/gtk-refresh.svg" ) ) self.pushButtonRafraichirPlugin.setToolTip( "Rafraichir le plugin" ) self.gridLayoutFichiers.addWidget( self.pushButtonRafraichirPlugin, 2, 0, 2, 1 ) # Bouton pour ouvrir la fenetre des preferences du plugin courant self.pushButtonPreferencesPlugin = MyQPushButton( self.tabFichiers ) self.pushButtonPreferencesPlugin.setIcon( QtGui.QIcon( "ico/gtk-preferences.svg" ) ) self.pushButtonPreferencesPlugin.setToolTip( u"Ouvrir les préférences du plugin" ) self.gridLayoutFichiers.addWidget( self.pushButtonPreferencesPlugin, 4, 0, 2, 1 ) # On met en place ce layout sur un widget (pour le splitter) self.widgetFichiers = QtGui.QWidget() self.widgetFichiers.setLayout( self.gridLayoutFichiers ) # # Descriptif des fichiers # # Layout de grille self.gridLayoutDescriptif = QtGui.QGridLayout() # Label pour afficher un logo self.logoFichierDefaut = QtGui.QPixmap() self.logoFichierDefaut.load( "img/gtk-dialog-question.svg" ) self.labelLogo = QtGui.QLabel( self.centralWidget ) self.labelLogo.setPixmap( self.logoFichierDefaut.scaled( QtCore.QSize( 150, 150 ), QtCore.Qt.KeepAspectRatio ) ) self.gridLayoutDescriptif.addWidget( self.labelLogo, 0, 0, 1, 1 ) # Zone de texte pour afficher un descriptif self.plainTextEdit = QtGui.QPlainTextEdit( self.centralWidget ) self.gridLayoutDescriptif.addWidget( self.plainTextEdit, 0, 1, 1, 2 ) # On met en place ce layout sur un widget (pour le splitter) self.widgetDescriptif = QtGui.QWidget() self.widgetDescriptif.setLayout( self.gridLayoutDescriptif ) # Onrientation verticale du splitter self.tabFichiers.setOrientation( QtCore.Qt.Vertical ) # On ajoute les 2 elements au splitter (qui est notre onglet) self.tabFichiers.addWidget( self.widgetFichiers ) self.tabFichiers.addWidget( self.widgetDescriptif ) # # Liste des telechargements # # Layout de grille qui contient le tableau qui liste les fichiers a telecharger + les boutons pour le controller self.gridLayoutTelechargement = QtGui.QGridLayout( self.tabTelechargements ) # Tableau qui contient la liste des fichiers a telecharger self.tableWidgetTelechargement = MyQTableWidget( self.tabTelechargements ) # Il a 5 colonnes et 0 ligne (pour l'instant) self.tableWidgetTelechargement.setColumnCount( 3 ) self.tableWidgetTelechargement.setRowCount( 0 ) # On ajoute le titre des 5 colonnes self.tableWidgetTelechargement.setHorizontalHeaderItem( 0, self.tableWidgetTelechargement.creerItem( "Date" ) ) self.tableWidgetTelechargement.setHorizontalHeaderItem( 1, self.tableWidgetTelechargement.creerItem( "Emission" ) ) self.tableWidgetTelechargement.setHorizontalHeaderItem( 2, self.tableWidgetTelechargement.creerItem( "Etat" ) ) # On l'ajoute au layout self.gridLayoutTelechargement.addWidget( self.tableWidgetTelechargement, 0, 1, 4, 1 ) # Bouton pour monter l'element selectionne tout en haut de la liste self.pushButtonExtremiteMonter = MyQPushButton( self.tabTelechargements ) self.pushButtonExtremiteMonter.setIcon( QtGui.QIcon( "ico/gtk-jump-to-rtl.svg" ) ) self.pushButtonExtremiteMonter.setToolTip( u"Placer l'élément sélectionné tout en haut" ) self.gridLayoutTelechargement.addWidget( self.pushButtonExtremiteMonter, 0, 0, 1, 1 ) # Bouton pour monter l'element selectionne d'un cran dans la liste self.pushButtonMonter = MyQPushButton( self.tabTelechargements ) self.pushButtonMonter.setIcon( QtGui.QIcon( "ico/gtk-go-up.svg" ) ) self.pushButtonMonter.setToolTip( u"Monter l'élément sélectionné" ) self.gridLayoutTelechargement.addWidget( self.pushButtonMonter, 1, 0, 1, 1 ) # Bouton pour descendre l'element selectionne d'un cran dans la liste self.pushButtonDescendre = MyQPushButton( self.tabTelechargements ) self.pushButtonDescendre.setIcon( QtGui.QIcon( "ico/gtk-go-down.svg" ) ) self.pushButtonDescendre.setToolTip( u"Descendre l'élément selectionné" ) self.gridLayoutTelechargement.addWidget( self.pushButtonDescendre, 2, 0, 1, 1 ) # Bouton pour descendre l'element selectionne tout en bas de la liste self.pushButtonExtremiteDescendre = MyQPushButton( self.tabTelechargements ) self.pushButtonExtremiteDescendre.setIcon( QtGui.QIcon( "ico/gtk-jump-to-ltr.svg" ) ) self.pushButtonExtremiteDescendre.setToolTip( u"Placer l'élément sélectionné tout en bas" ) self.gridLayoutTelechargement.addWidget( self.pushButtonExtremiteDescendre, 3, 0, 1, 1 ) # Bouton pour supprimer tous les elements de la liste self.pushButtonToutSupprimer = MyQPushButton( self.tabTelechargements ) self.pushButtonToutSupprimer.setIcon( QtGui.QIcon( "ico/gtk-cancel.svg" ) ) self.pushButtonToutSupprimer.setToolTip( u"Supprimer tous les téléchargements" ) self.gridLayoutTelechargement.addWidget( self.pushButtonToutSupprimer, 0, 2, 1, 1 ) # Bouton pour supprimer de la liste les telechargements termines self.pushButtonNettoyer = MyQPushButton( self.tabTelechargements ) self.pushButtonNettoyer.setIcon( QtGui.QIcon( "ico/gtk-delete-full.svg" ) ) self.pushButtonNettoyer.setToolTip( u"Supprimer les téléchargement terminés" ) self.gridLayoutTelechargement.addWidget( self.pushButtonNettoyer, 1, 2, 1, 1 ) # Bouton pour ouvrir le dossier des telechargements self.pushButtonOuvrirDossierTelechargement = MyQPushButton( self.tabTelechargements ) self.pushButtonOuvrirDossierTelechargement.setIcon( QtGui.QIcon( "ico/gtk-folder.svg" ) ) self.pushButtonOuvrirDossierTelechargement.setToolTip( u"Ouvrir le dossier des téléchargements" ) self.gridLayoutTelechargement.addWidget( self.pushButtonOuvrirDossierTelechargement, 2, 2, 1, 1 ) # # Barre progression de telechargement d'un fichier # self.progressBarTelechargementFichier = QtGui.QProgressBar( self.centralWidget ) self.progressBarTelechargementFichier.setProperty( "value", 0 ) # # Barre de progression de telechargement des fichiers # self.progressBarTelechargement = QtGui.QProgressBar( self.centralWidget ) self.progressBarTelechargement.setProperty( "value", 0 ) # # Boutons du bas pour gerer ajouter/supprimer/lancer telechargements # # Layout horizontal qui contiendra les boutons self.horizontalLayoutBarreBas = QtGui.QHBoxLayout() # Bouton pour lancer les telechargements self.pushButtonLancer = QtGui.QPushButton( QtGui.QIcon( "ico/gtk-media-play-ltr.svg" ), u"Lancer téléchargement", self.centralWidget ) self.horizontalLayoutBarreBas.addWidget( self.pushButtonLancer ) # Bouton pour stopper les telechargements self.pushButtonStop = QtGui.QPushButton( QtGui.QIcon( "ico/gtk-media-stop.svg" ), u"Stopper le téléchargement", self.centralWidget ) self.pushButtonStop.setEnabled( False ) self.horizontalLayoutBarreBas.addWidget( self.pushButtonStop ) ### # Positionnement des differents widgets/layouts sur le layout de grille ### # Layout de grille dans lequel on va placer nos widgets/layouts self.gridLayout = QtGui.QGridLayout( self.centralWidget ) # On ajoute la barre du haut self.gridLayout.addLayout( self.horizontalLayoutBarreHaut, 0, 0, 1, 3 ) # On ajoute le gestionnaire d'onglets self.gridLayout.addWidget( self.tabWidget, 1, 0, 1, 3 ) # On ajoute la barre de progression de telechargement d'un fichier self.gridLayout.addWidget( self.progressBarTelechargementFichier, 2, 0, 1, 3 ) # On ajoute la barre de progression de telechargement des fichiers self.gridLayout.addWidget( self.progressBarTelechargement, 3, 0, 1, 3 ) # On ajoute les boutons ajouter/supprimer/lancer self.gridLayout.addLayout( self.horizontalLayoutBarreBas, 4, 0, 1, 3 ) ### # Mise en place le central widget dans la fenetre ### self.setCentralWidget( self.centralWidget ) ### # Mise en place du menu ### # Menu barre self.menubar = QtGui.QMenuBar( self ) self.menubar.setGeometry( QtCore.QRect( 0, 0, 480, 25 ) ) # Menu Fichier self.menuFichier = QtGui.QMenu( "&Fichier", self.menubar ) self.menubar.addAction( self.menuFichier.menuAction() ) # Action Fichier -> Quitter self.actionQuitter = QtGui.QAction( QtGui.QIcon( "ico/gtk-quit.svg" ), "&Quitter", self ) self.actionQuitter.setIconVisibleInMenu( True ) self.menuFichier.addAction( self.actionQuitter ) # Menu Edition self.menuEdition = QtGui.QMenu( "&Edition", self.menubar ) self.menubar.addAction( self.menuEdition.menuAction() ) # Action Edition -> Mise a jour self.actionMAJ = QtGui.QAction( QtGui.QIcon( "ico/gtk-refresh.svg" ), u"&Mise à jour des plugins", self ) self.actionMAJ.setIconVisibleInMenu( True ) self.menuEdition.addAction( self.actionMAJ ) # Action Edition -> Preferences self.actionPreferences = QtGui.QAction( QtGui.QIcon( "ico/gtk-preferences.svg" ), u"&Préférences", self ) self.actionPreferences.setIconVisibleInMenu( True ) self.menuEdition.addAction( self.actionPreferences ) # Menu Aide self.menuAide = QtGui.QMenu( "&Aide", self.menubar ) self.menubar.addAction( self.menuAide.menuAction() ) # Action Aide -> A propos self.actionAPropos = QtGui.QAction( QtGui.QIcon( "ico/gtk-about.svg" ), u"À p&ropos", self ) self.actionAPropos.setIconVisibleInMenu( True ) self.menuAide.addAction( self.actionAPropos ) # Ajout du menu a l'interface self.setMenuBar( self.menubar ) ### # Signaux provenants de l'interface ### QtCore.QObject.connect( self.tableWidgetFichier, QtCore.SIGNAL( "cellClicked(int,int)" ), self.afficherInformationsFichier ) QtCore.QObject.connect( self.tableWidgetFichier, QtCore.SIGNAL( "cellDoubleClicked(int,int)" ), self.gererTelechargement ) QtCore.QObject.connect( self.pushButtonToutAjouter, QtCore.SIGNAL( "clicked()" ), self.ajouterTousLesFichiers ) QtCore.QObject.connect( self.pushButtonRafraichirPlugin, QtCore.SIGNAL( "clicked()" ), self.rafraichirPlugin ) QtCore.QObject.connect( self.tableWidgetTelechargement, QtCore.SIGNAL( "cellDoubleClicked(int,int)" ), self.supprimerTelechargement ) QtCore.QObject.connect( self.pushButtonExtremiteMonter, QtCore.SIGNAL( "clicked()" ), lambda versLeHaut = True, extremite = True : self.tableWidgetTelechargement.deplacerLigne( versLeHaut, extremite ) ) QtCore.QObject.connect( self.pushButtonMonter, QtCore.SIGNAL( "clicked()" ), lambda versLeHaut = True, extremite = False : self.tableWidgetTelechargement.deplacerLigne( versLeHaut, extremite ) ) QtCore.QObject.connect( self.pushButtonDescendre, QtCore.SIGNAL( "clicked()" ), lambda versLeHaut = False, extremite = False : self.tableWidgetTelechargement.deplacerLigne( versLeHaut, extremite ) ) QtCore.QObject.connect( self.pushButtonExtremiteDescendre, QtCore.SIGNAL( "clicked()" ), lambda versLeHaut = False, extremite = True : self.tableWidgetTelechargement.deplacerLigne( versLeHaut, extremite ) ) QtCore.QObject.connect( self.pushButtonToutSupprimer, QtCore.SIGNAL( "clicked()" ), self.supprimerTousLesTelechargements ) QtCore.QObject.connect( self.pushButtonNettoyer, QtCore.SIGNAL( "clicked()" ), self.nettoyer ) QtCore.QObject.connect( self.pushButtonLancer, QtCore.SIGNAL( "clicked()" ), self.lancerTelechargement ) QtCore.QObject.connect( self.pushButtonStop, QtCore.SIGNAL( "clicked()" ), self.stopperTelechargement ) QtCore.QObject.connect( self.actionQuitter, QtCore.SIGNAL( "triggered()" ), self.close ) ################################################ # Instanciations + initialisation de variables # ################################################ # Fenetre About self.aProposDialog = None # Fenetre des preferences du logiciel self.preferencesDialog = None # Fenetre de mise a jour des plugins self.updateManagerDialog = None # Nom plugin courant self.nomPluginCourant = "" # Liste des fichiers self.listeFichiers = [] # Liste des fichiers a telecharger self.listeFichiersATelecharger = [] # Cache des images descriptive # Clef : urlImage Valeur : image (binaire) self.cacheImage = {} # On intancie le lanceur de signaux self.signaux = Signaux() # On instancie le gestionnaire de preferences self.preferences = Preferences() # On instancie le gestionnaire de preferences des plugins self.preferencesPluginDialog = PreferencePluginDialog( self ) # On instancie le gestionnaire de download self.downloader = Downloader( self.signaux ) # On recupere l'instance de API self.api = API.getInstance() # On instancie le gestionnaire d'historique self.historique = Historique() # On instancie la fenetre d'attente self.fenetreAttenteProgressDialog = FenetreAttenteProgressDialog( self ) # On instancie le gest # # Fenetre de confirmation pour quitter le logiciel # self.quitterMessageBox = QtGui.QMessageBox( self ) self.quitterMessageBox.setWindowTitle( "Fermeture de TVDownloader" ) self.quitterMessageBox.setText( u"Voulez-vous réellement quitter TVDownloader ?" ) self.quitterMessageBox.setInformativeText( u"Votre liste de téléchargement sera perdue" ) self.quitterMessageBox.addButton( "Oui", QtGui.QMessageBox.AcceptRole ) self.quitterMessageBox.addButton( "Non", QtGui.QMessageBox.RejectRole ) ############################################################ # On connecte les signaux des instances precedements crees # ############################################################ QtCore.QObject.connect( self.pushButtonOuvrirDossierTelechargement, QtCore.SIGNAL( "clicked()" ), self.ouvrirRepertoireTelechargement ) QtCore.QObject.connect( self.comboBoxSite, QtCore.SIGNAL( "activated(QString)" ), self.listerChaines ) QtCore.QObject.connect( self.comboBoxChaine, QtCore.SIGNAL( "activated(QString)" ), self.listerEmissions ) QtCore.QObject.connect( self.comboBoxEmission, QtCore.SIGNAL( "activated(QString)" ), self.listerFichiers ) QtCore.QObject.connect( self.pushButtonPreferencesPlugin, QtCore.SIGNAL( "clicked()" ), self.ouvrirPreferencesPlugin ) QtCore.QObject.connect( self.actionPreferences, QtCore.SIGNAL( "triggered()" ), self.ouvrirPreferencesLogiciel ) QtCore.QObject.connect( self.actionMAJ, QtCore.SIGNAL( "triggered()" ), self.ouvrirFenetreMiseAJour ) QtCore.QObject.connect( self.actionAPropos, QtCore.SIGNAL( "triggered()" ), self.ouvrirFenetreAPropos ) QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "debutActualisation(PyQt_PyObject)" ) , self.fenetreAttenteProgressDialog.ouvrirFenetreAttente ) QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "finActualisation()" ) , self.fenetreAttenteProgressDialog.fermerFenetreAttente ) QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "actualiserListesDeroulantes()" ) , self.actualiserListesDeroulantes ) QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "listeChaines(PyQt_PyObject)" ) , self.ajouterChaines ) QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "listeEmissions(PyQt_PyObject)" ) , self.ajouterEmissions ) QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "listeFichiers(PyQt_PyObject)" ) , self.ajouterFichiers ) QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "nouvelleImage(PyQt_PyObject)" ) , self.mettreEnPlaceImage ) QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "debutTelechargement(int)" ) , self.debutTelechargement ) QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "finTelechargement(int)" ) , self.finTelechargement ) QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "finDesTelechargements()" ) , self.activerDesactiverInterface ) QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "pourcentageFichier(int)" ) , self.progressBarTelechargementFichier.setValue ) ######### # Début # ######### # La fenetre prend la dimension qu'elle avait a sa fermeture taille = self.preferences.getPreference( "tailleFenetre" ) self.resize( taille[ 0 ], taille[ 1 ] ) # Si aucun plugin n'est active, on ouvre la fenetre des preferences if( len( self.preferences.getPreference( "pluginsActifs" ) ) == 0 ): self.ouvrirPreferencesLogiciel() # On actualise tous les plugins self.rafraichirTousLesPlugins() ## Methode qui execute les actions necessaires avant de quitter le programme def actionsAvantQuitter( self ): # On sauvegarde les options des plugins self.api.fermeture() # On sauvegarde la taille de la fenetre taille = self.size() self.preferences.setPreference( "tailleFenetre", [ taille.width(), taille.height() ] ) # On sauvegarde les options du logiciel self.preferences.sauvegarderConfiguration() # On sauvegarde l'historique self.historique.sauverHistorique() # On stoppe les telechargements self.stopperTelechargement() ######################################### # Surcharge des methodes de QMainWindow # ######################################### ## Surcharge de la methode appelee lors de la fermeture de la fenetre # Ne doit pas etre appele explicitement # @param evenement Evenement qui a provoque la fermeture def closeEvent( self, evenement ): # On affiche une fenetre pour demander la fermeture si des fichiers sont dans la liste de telechargement if( self.tableWidgetTelechargement.rowCount() > 0 ): # On affiche une fenetre qui demande si on veut quitter retour = self.quitterMessageBox.exec_() # Si on veut quitter if( retour == 0 ): # On execute les actions necessaires self.actionsAvantQuitter() # On accept la fermeture evenement.accept() else: # On refuse la fermeture evenement.ignore() else: # S'il n'y a pas de fichier # On execute les actions necessaires self.actionsAvantQuitter() # On accept la fermeture evenement.accept() ############################################## # Methodes pour remplir les menus deroulants # ############################################## ## Methode qui actualise les listes deroulantes def actualiserListesDeroulantes( self ): # On lance juste l'ajout des sites en se basant sur les plugins actifs self.ajouterSites( self.preferences.getPreference( "pluginsActifs" ) ) ## Methode qui lance le listage des chaines # @param site Nom du plugin/site pour lequel on va lister les chaines def listerChaines( self, site ): def threadListerChaines( self, nomPlugin ): self.signaux.signal( "debutActualisation", nomPlugin ) listeChaines = self.api.getPluginListeChaines( nomPlugin ) self.signaux.signal( "listeChaines", listeChaines ) self.signaux.signal( "finActualisation" ) if( site != "" ): self.nomPluginCourant = qstringToString( site ) threading.Thread( target = threadListerChaines, args = ( self, self.nomPluginCourant ) ).start() # On active (ou pas) le bouton de preference du plugin self.pushButtonPreferencesPlugin.setEnabled( self.api.getPluginListeOptions( self.nomPluginCourant ) != [] ) ## Methode qui lance le listage des emissions # @param chaine Nom de la chaine pour laquelle on va lister les emissions def listerEmissions( self, chaine ): def threadListerEmissions( self, nomPlugin, chaine ): self.signaux.signal( "debutActualisation", nomPlugin ) listeEmissions = self.api.getPluginListeEmissions( nomPlugin, chaine ) self.signaux.signal( "listeEmissions", listeEmissions ) self.signaux.signal( "finActualisation" ) if( chaine != "" ): threading.Thread( target = threadListerEmissions, args = ( self, self.nomPluginCourant, qstringToString( chaine ) ) ).start() ## Methode qui lance le listage des fichiers # @param emission Nom de l'emission pour laquelle on va lister les fichiers def listerFichiers( self, emission ): def threadListerFichiers( self, nomPlugin, emission ): self.signaux.signal( "debutActualisation", nomPlugin ) listeFichiers = self.api.getPluginListeFichiers( nomPlugin, emission ) self.signaux.signal( "listeFichiers", listeFichiers ) self.signaux.signal( "finActualisation" ) if( emission != "" ): threading.Thread( target = threadListerFichiers, args = ( self, self.nomPluginCourant, qstringToString( emission ) ) ).start() ## Methode qui met en place une liste de sites sur l'interface # @param listeSites Liste des sites a mettre en place def ajouterSites( self, listeSites ): # On efface la liste des sites self.comboBoxSite.clear() # On met en place les sites for site in listeSites: self.comboBoxSite.addItem( stringToQstring( site ) ) # On selectionne par defaut celui choisis dans les preference index = self.comboBoxSite.findText( stringToQstring( self.preferences.getPreference( "pluginParDefaut" ) ) ) if( index != -1 ): self.comboBoxSite.setCurrentIndex( index ) # On lance l'ajout des chaines self.listerChaines( self.comboBoxSite.currentText() ) ## Methode qui met en place une liste de chaines sur l'interface # @param listeChaines Liste des chaines a mettre en place def ajouterChaines( self, listeChaines ): # On trie la liste des chaines listeChaines.sort() # On efface la liste des chaines self.comboBoxChaine.clear() # On efface la liste des emissions self.comboBoxEmission.clear() # On efface la liste des fichiers self.tableWidgetFichier.toutSupprimer() # On met en place les chaines for chaine in listeChaines: self.comboBoxChaine.addItem( stringToQstring( chaine ) ) # Si on a juste une seule chaine if( self.comboBoxChaine.count() == 1 ): # On lance l'ajout des emissions self.listerEmissions( self.comboBoxChaine.currentText() ) else: # On ne selectionne pas de chaine self.comboBoxChaine.setCurrentIndex( -1 ) ## Methode qui met en place une liste d'emissions sur l'interface # @param listeEmissions Liste des emissions a mettre en place def ajouterEmissions( self, listeEmissions ): # On trie la liste des emissions listeEmissions.sort() # On efface la liste des emissions self.comboBoxEmission.clear() # On efface la liste des fichiers self.tableWidgetFichier.toutSupprimer() # On met en place la liste des emissions for emission in listeEmissions: self.comboBoxEmission.addItem( stringToQstring( emission ) ) # Si on a juste une seule emission if( self.comboBoxEmission.count() == 1 ): # On lance l'ajout des fichiers self.listerFichiers( self.comboBoxEmission.currentText() ) else: # On ne selectionne pas d'emission self.comboBoxEmission.setCurrentIndex( -1 ) ############################################### # Methodes pour remplir la liste des fichiers # ############################################### ## Methode pour ajouter des fichiers a l'interface # @param listeFichiers Liste des fichiers a ajouter def ajouterFichiers( self, listeFichiers ): self.listeFichiers = listeFichiers # On efface la liste des fichiers self.tableWidgetFichier.toutSupprimer() # On commence au depart ligneCourante = 0 # On met en place chacun des fichiers for fichier in listeFichiers: # On ajoute une ligne self.tableWidgetFichier.insertRow( ligneCourante ) # On ajoute les informations au tableWidgetFichier liste = [] liste.append( self.tableWidgetFichier.creerItem( "" ) ) liste.append( self.tableWidgetFichier.creerItem( getattr( fichier, "date" ) ) ) liste.append( self.tableWidgetFichier.creerItem( getattr( fichier, "nom" ) ) ) self.tableWidgetFichier.setLigne( ligneCourante, liste ) # On met en place l'icone qui va bien self.gererIconeListeFichier( fichier ) ligneCourante += 1 # On adapte la taille des colonnes self.tableWidgetFichier.adapterColonnes() ## Methode qui rafraichit le plugin courant def rafraichirPlugin( self ): def threadRafraichirPlugin( self, nomPlugin ): self.signaux.signal( "debutActualisation", nomPlugin ) self.api.pluginRafraichir( nomPlugin ) self.signaux.signal( "finActualisation" ) threading.Thread( target = threadRafraichirPlugin, args = ( self, self.nomPluginCourant ) ).start() ## Methode qui met en place l'image de la description d'un fichier # @param image Image a mettre en place (binaire) def mettreEnPlaceImage( self, image ): logoFichier = QtGui.QPixmap() logoFichier.loadFromData( image ) self.labelLogo.setPixmap( logoFichier.scaled( QtCore.QSize( 150, 150 ), QtCore.Qt.KeepAspectRatio ) ) ## Methode qui affiche des informations sur le fichier selectionne def afficherInformationsFichier( self, ligne, colonne ): def threadRecupererImage( self, urlImage ): image = self.api.getPage( urlImage ) self.cacheImage[ urlImage ] = image self.signaux.signal( "nouvelleImage", image ) fichier = self.listeFichiers[ ligne ] # On recupere le lien de l'image et le texte descriptif urlImage = getattr( fichier, "urlImage" ) texteDescriptif = getattr( fichier, "descriptif" ) self.plainTextEdit.clear() # Si on a un texte descriptif, on l'affiche if( texteDescriptif != "" ): self.plainTextEdit.appendPlainText( stringToQstring( texteDescriptif ) ) else: self.plainTextEdit.appendPlainText( u"Aucune information disponible" ) # Si on n'a pas d'image if( urlImage == "" ): # On met en place celle par defaut self.logoFichier = self.logoFichierDefaut self.labelLogo.setPixmap( self.logoFichier.scaled( QtCore.QSize( 150, 150 ), QtCore.Qt.KeepAspectRatio ) ) else: # Si on en a une # Si elle est dans le cache des images if( self.cacheImage.has_key( urlImage ) ): self.mettreEnPlaceImage( self.cacheImage[ urlImage ] ) else: # Sinon # On lance le thread pour la recuperer threading.Thread( target = threadRecupererImage, args = ( self, urlImage ) ).start() ## Methode qui gere l'icone d'un fichier dans la liste des telechargements # Il y a 3 icones possible : # - C'est un fichier # - C'est un fichier present dans l'historique (donc deja telecharge) # - C'est un fichier present dans la liste des telechargements # @param fichier Fichier a gerer def gererIconeListeFichier( self, fichier ): if( fichier in self.listeFichiers ): ligneFichier = self.listeFichiers.index( fichier ) # On cherche quel icone mettre en place if( fichier in self.listeFichiersATelecharger ): icone = self.iconeAjoute elif( self.historique.comparerHistorique( fichier ) ): icone = self.iconeTelecharge else: icone = self.iconeFichier # On met en place l'icone self.tableWidgetFichier.item( ligneFichier, 0 ).setIcon( icone ) ###################################################### # Methodes pour remplir la liste des telechargements # ###################################################### ## Methode qui gere la liste des telechargements # @param ligne Numero de la ligne (dans la liste des fichiers) de l'element a ajouter # @param colonne Numero de colonne (inutile, juste pour le slot) def gererTelechargement( self, ligne, colonne = 0 ): fichier = self.listeFichiers[ ligne ] # Si le fichier est deja dans la liste des telechargements if( fichier in self.listeFichiersATelecharger ): ligneTelechargement = self.listeFichiersATelecharger.index( fichier ) self.supprimerTelechargement( ligneTelechargement ) else: # S'il n'y est pas, on l'ajoute self.listeFichiersATelecharger.append( fichier ) numLigne = self.tableWidgetTelechargement.rowCount() # On insere une nouvelle ligne dans la liste des telechargements self.tableWidgetTelechargement.insertRow( numLigne ) # On y insere les elements qui vont biens self.tableWidgetTelechargement.setLigne( numLigne, [ self.tableWidgetTelechargement.creerItem( getattr( fichier, "date" ) ), self.tableWidgetTelechargement.creerItem( getattr( fichier, "nom" ) ), self.tableWidgetTelechargement.creerItem( u"En attente de téléchargement" ) ] ) # On adapte la taille des colonnes self.tableWidgetTelechargement.adapterColonnes() # On modifie l'icone dans la liste des fichiers self.gererIconeListeFichier( fichier ) ## Methode qui ajoute tous les fichiers a la liste des telechargements def ajouterTousLesFichiers( self ): for i in range( self.tableWidgetFichier.rowCount() ): self.gererTelechargement( i ) ## Methode qui supprime un fichier de la liste des telechargements # @param ligne Numero de la ligne a supprimer # @param colonne Numero de colonne (inutile, juste pour le slot) def supprimerTelechargement( self, ligne, colonne = 0 ): fichier = self.listeFichiersATelecharger[ ligne ] # On supprime l'element du tableWidgetTelechargement self.tableWidgetTelechargement.removeRow( ligne ) # On supprime l'element de la liste des fichiers a telecharger self.listeFichiersATelecharger.remove( fichier ) # On modifie l'icone dans la liste des fichiers self.gererIconeListeFichier( fichier ) ## Methode qui supprime tous les telechargement de la liste des telechargements def supprimerTousLesTelechargements( self ): for i in range( self.tableWidgetTelechargement.rowCount() -1, -1, -1 ): self.supprimerTelechargement( i ) ## Methode qui lance le telechargement des fichiers def lancerTelechargement( self ): # On liste les emissions a telecharger avec leurs numeros de ligne listeFichiers = [] for i in range( self.tableWidgetTelechargement.rowCount() ): # Pour chaque ligne fichier = self.listeFichiersATelecharger[ i ] listeFichiers.append( [ i, getattr( fichier, "lien" ), getattr( fichier, "nomFichierSortie" ) ] ) nbATelecharger = len( listeFichiers ) # Si on a des elements a charger if( nbATelecharger > 0 ): # On met en place la valeur du progressBar self.progressBarTelechargement.setMaximum( nbATelecharger ) self.progressBarTelechargement.setValue( 0 ) # On lance le telechargement threading.Thread( target = self.downloader.lancerTelechargement, args = ( listeFichiers, ) ).start() # On active/desactive ce qui va bien sur l'interface self.activerDesactiverInterface( True ) ## Methode qui stoppe le telechargement def stopperTelechargement( self ): # On stoppe le telechargement self.downloader.stopperTelechargement() ############################################ # Methodes pour ouvrir les autres fenetres # ############################################ # # Fenetre About # ## Methode pour afficher la fenetre About def ouvrirFenetreAPropos( self ): if( self.aProposDialog == None ): self.aProposDialog = AProposDialog() self.aProposDialog.show() # # Fenetre de preference du logiciel # ## Methode pour ouvrir les preferences du logiciel def ouvrirPreferencesLogiciel( self ): if( self.preferencesDialog == None ): self.preferencesDialog = PreferencesDialog( self, self.signaux ) self.preferencesDialog.afficher() # # Fenetre de mise a jour des plugins # ## Methode pour ouvrir la fenetre de mise a jour des plugins def ouvrirFenetreMiseAJour( self ): if( self.updateManagerDialog == None ): self.updateManagerDialog = UpdateManagerDialog( self ) self.updateManagerDialog.afficher() # # Fenetre de preference des plugins # ## Methode pour ouvrir les preferences du plugin courant def ouvrirPreferencesPlugin( self ): listeOptions = self.api.getPluginListeOptions( self.nomPluginCourant ) self.preferencesPluginDialog.ouvrirDialogPreferences( self.nomPluginCourant, listeOptions ) ######### # Slots # ######### ## Methode qui ouvre le repertoire de telechargement def ouvrirRepertoireTelechargement( self ): QtGui.QDesktopServices.openUrl( QtCore.QUrl.fromLocalFile( self.preferences.getPreference( "repertoireTelechargement" ) ) ) ## Methode qui rafraichit le plugin courant def rafraichirPlugin( self ): def threadRafraichirPlugin( self, nomPlugin ): self.signaux.signal( "debutActualisation", nomPlugin ) self.api.pluginRafraichir( nomPlugin ) self.signaux.signal( "finActualisation" ) threading.Thread( target = threadRafraichirPlugin, args = ( self, self.nomPluginCourant ) ).start() ## Methode qui rafraichit tous les plugins # A utiliser au lancement du programme def rafraichirTousLesPlugins( self ): def threadRafraichirTousLesPlugins( self ): self.signaux.signal( "debutActualisation", "TVDownloader" ) self.api.pluginRafraichirAuto() self.signaux.signal( "finActualisation" ) self.signaux.signal( "actualiserListesDeroulantes" ) threading.Thread( target = threadRafraichirTousLesPlugins, args = ( self, ) ).start() ## Slot qui active/desactive des elements de l'interface pendant un telechargement # @param telechargementEnCours Indique si on telecharge ou pas def activerDesactiverInterface( self, telechargementEnCours = False ): # Les boutons self.pushButtonLancer.setEnabled( not telechargementEnCours ) self.pushButtonStop.setEnabled( telechargementEnCours ) self.pushButtonExtremiteMonter.setEnabled( not telechargementEnCours ) self.pushButtonMonter.setEnabled( not telechargementEnCours ) self.pushButtonDescendre.setEnabled( not telechargementEnCours ) self.pushButtonExtremiteDescendre.setEnabled( not telechargementEnCours ) self.pushButtonToutSupprimer.setEnabled( not telechargementEnCours ) self.pushButtonNettoyer.setEnabled( not telechargementEnCours ) # Le table widget self.tableWidgetTelechargement.setEnabled( not telechargementEnCours ) ## Slot appele lors ce qu'un le debut d'un telechargement commence # @param numero Position dans la liste des telechargement du telechargement qui commence def debutTelechargement( self, numero ): self.tableWidgetTelechargement.item( numero, 2 ).setText( stringToQstring( u"Téléchargement en cours..." ) ) self.tableWidgetTelechargement.adapterColonnes() self.progressBarTelechargementFichier.setValue( 0 ) ## Slot appele lorsqu'un telechargement se finit # @param numero Position dans la liste des telechargement du telechargement qui se finit def finTelechargement( self, numero ): fichier = self.listeFichiersATelecharger[ numero ] # On ajoute le fichier a l'historique self.historique.ajouterHistorique( fichier ) # On modifie l'icone dans la liste des fichiers self.gererIconeListeFichier( fichier ) # On modifie l'interface self.tableWidgetTelechargement.item( numero, 2 ).setText( stringToQstring( u"Fini !" ) ) self.progressBarTelechargement.setValue( self.progressBarTelechargement.value() + 1 ) self.tableWidgetTelechargement.adapterColonnes() self.progressBarTelechargementFichier.setValue( 100 ) ## Slot qui nettoie la liste des telechargements de tous les telechargements finis def nettoyer( self ): for i in range( self.tableWidgetTelechargement.rowCount() - 1, -1, -1 ): # [ nbLignes - 1, nbLignes - 2, ..., 1, 0 ] if( self.tableWidgetTelechargement.item( i, 2 ).text() == u"Fini !" ): # Si c'est telecharge self.supprimerTelechargement( i )
def download(cookies, url, folder): downloader = Downloader(cookies, url, folder) downloader.run()
class Watcher: def __init__(self, filename, period='1yr', stockPicks=10, avoidDowntrends=True, sellAllOnCrash=True, hyperparameters={}): self.filename = filename self.period = period self.stockPicks = stockPicks self.hyperparameters = hyperparameters self.sl = hyperparameters['sl'] self.tp = hyperparameters['tp'] self.ts = hyperparameters['ts'] self.avoidDowntrends = avoidDowntrends self.sellAllOnCrash = sellAllOnCrash def create(self, balance): # Create the portolio self.portfolio = Portfolio(cash=balance, sl=self.sl, tp=self.tp) self.portfolio.save(self.filename) def start(self): # Open the portolio self.portfolio = Portfolio(cash=0, sl=self.sl, tp=self.tp) self.portfolio.load(self.filename) # Download the stock data self.downloader = Downloader(cache=False) self.downloader.download(period=self.period) date = self.downloader.prices.index[-1] prices = self.downloader.prices print("#", date) # Refresh the portfolio with the latest prices self.portfolio.refresh(prices=prices, date=date) # Get the portfolio Summary portfolio_summary = self.portfolio.summary() # Get the opened positions opened = self.portfolio.holdings[self.portfolio.holdings['status'] == 'open'] # Get the current trend zt = TradingZones(prices=prices) zones = zt.get() output = pd.DataFrame() # Activate the trailing stops targets = self.portfolio.getPositionsBelowTS(self.ts) if len(targets) > 0: sell_orders = self.portfolio.getSellOrdersFromSubset( subset=targets) print("\n----- TS -----\n", sell_orders) #self.portfolio.sell(sell_orders, prices, date, label='TS') # Activate the stop loss if self.sl < 0: poor_perfs = opened[opened['profits_pct'] <= self.sl] if len(poor_perfs) > 0: sell_list = poor_perfs['symbol'].unique() sell_orders = self.portfolio.getSellAllOrders( symbols=sell_list) print("\n----- SL -----\n", sell_orders) #self.portfolio.sell(sell_orders, prices, date, label='SL') # Sell all on drop if self.sellAllOnCrash == True and zones.iloc[-1][ 'signal'] == 'DOWN' and len(opened) > 0: print("Sell all") print("\n----- Sell All -----\n") #self.portfolio.sellAll(prices, date, label='Market Drop') # Get the ones that reached take-profit if self.tp > 0: good_perfs = opened[opened['profits_pct'] >= self.tp] if len(good_perfs) > 0: sell_list = good_perfs['symbol'].unique() sell_orders = self.portfolio.getSellAllOrders( symbols=sell_list) print("\n----- TP -----\n", sell_orders) #self.portfolio.sell(sell_orders, prices, date, label='TP') # Avoid trading in a general market downtrend if self.avoidDowntrends == False or (self.avoidDowntrends == True and zones.iloc[-1]['signal'] == 'UP'): # Pick the stocks picker = Picker(prices=prices) picker.pick(count=self.stockPicks) #print(">", picker.selected) # Get the order size portfolio_summary = self.portfolio.summary() order = Order(prices=prices, picks=picker.selected) order.get(budget=portfolio_summary['cash']) #print("!", portfolio_summary['cash'], order.orders) print("\n----- BUY -----\n", order.orders) # Update our portfolio #self.portfolio.update(orders=order.orders, prices=prices, date=date) else: # No new positions in a downtrend! # Update our portfolio self.portfolio.update(orders=pd.DataFrame(), prices=prices, date=date) def stats(self): # Open the portolio self.portfolio = Portfolio(cash=0, sl=self.sl, tp=self.tp) self.portfolio.load(self.filename) symbols = list(self.portfolio.holdings['symbol'].unique()) if len(symbols) == 0: print("Nothing to update") return False elif len(symbols) == 1: symbols = symbols + [ 'MSFT', 'TSLA' ] # otherwise single symbol = no multi-index = breaking # Download the stock data self.downloader = Downloader(cache=False) self.downloader.download(period=self.period, symbols=symbols) date = self.downloader.prices.index[-1] prices = self.downloader.prices print("#", date) # Refresh the portfolio with the latest prices self.portfolio.refresh(prices=prices, date=date) # Save the portfolio with the latest prices self.portfolio.save(self.filename) print(self.portfolio.summary()) print('') g = self.portfolio.holdings[self.portfolio.holdings['status'] == 'open'].copy().groupby('symbol').sum() g['profits_pct'] = (g['current_price'] - g['purchase_price']) / g['purchase_price'] * 100 g['price'] = prices.loc[date] print(g) print(list(g.index)) print(prices.loc[date]) def check(self, symbol='TSLA', period='6mo'): # Download the stock data self.downloader = Downloader(cache=False) if symbol not in self.downloader.symbols: self.downloader.add(symbol) print('Symbol added to the watch list') self.downloader.download(period=period, symbols=[symbol]) print(self.downloader.prices[symbol].tail(50))
log.setLevel(level) log2.setLevel(level) fmt = logging.Formatter('%(levelname)s:%(name)s:%(message)s') h = logging.StreamHandler() h.setFormatter(fmt) log.addHandler(h) log2.addHandler(h) #mongodb app.config["MONGO_URI"] = "mongodb://"+Config.get('DB', 'mongoHost')+":"+Config.get('DB', 'mongoPort')+"/"+Config.get('DB', 'mongoDbName') mongo = PyMongo(app) # objects doing the work downloader = Downloader(mongo, Config.get('PARSER', 'videoFolder')) parser = Parser(mongo, Config.get('PARSER', 'videoFolder'), Config.get('PARSER', 'characterFolder'), Config.get('PARSER', 'templateFolder')) parserTest = ParserTest(mongo, Config.get('PARSER', 'videoFolder'), Config.get('PARSER', 'characterFolder'), Config.get('PARSER', 'templateFolder')) # fire the main job @app.route('/firemainjob') def index(): print("fire scheduler") scheduler.get_job("mainJob").modify(next_run_time=datetime.datetime.now()) return jsonify({"fire": "ok"}) # download only @app.route('/onlydl/<idvid>') def onlydl(idvid): print("download:"+idvid) downloader.downloadVideo(idvid)
from MangaSiteFactory import get_manga_site, MangaSiteEnum from MangaSite import MangaSite from Manga import Manga, Chapter, MangaIndexTypeEnum from Downloader import Downloader from PySide2 import QtWidgets, QtCore from typing import List, Dict from functools import partial def call_timeout(loop): loop.quit() raise TimeoutError get_mhg = partial(get_manga_site, MangaSiteEnum.ManHuaGui, Downloader(None, './downloads')) class TestManHuaGui(unittest.TestCase): def test_init(self): mhg = get_mhg() self.assertEqual(mhg.name, '漫畫鬼') self.assertEqual(mhg.url, 'https://www.manhuagui.com/') def test_seach_manga(self): def search_callback(loop, result: List[Manga]): self.assertIsInstance(result, list) self.assertEqual(len(result), 2) foundFlag = False for manga in result: if manga.name == '火影忍者':
def scrapeKupujemProdajem(): # UTF-8 support reload(sys) sys.setdefaultencoding('utf-8') now = datetime.now() ads = [] try: down = Downloader( 'http://www.kupujemprodajem.com/search.php?action=list&data[category_id]=&data[group_id]=&data[location_id]=&data[keywords]=&submit[search]=Tra%C5%BEi' ) content = down.get_content() html = unicode(content) link = "" title = "" imageUrl = "" description = "/" category = "/" subcategory = "/" value = "/" currency = "/" region = "/" date = str(now.year) + "-" + str(now.month) + "-" + str(now.day) linkovi = xpath.search(html, '//div[@class="item clearfix"]') highlighted = xpath.search( html, '//div[@class="item clearfix adHighlighted"]') for h in highlighted: linkovi.append(h) for l in linkovi: try: link = "http://www.kupujemprodajem.com/" + xpath.get( l, '//a[@class="adName"]/@href') title = xpath.get(l, '//a[@class="adName"]') region = xpath.get(l, '//section[@class="locationSec"]').strip() region = region.split(" | ") region = region[0] price = xpath.get(l, '//span[@class="adPrice"]') price = price.split(" ") if len(price) == 2: value = price[0] value = value.replace(".", "") value = value.split(",") value = value[0] currency = price[1] else: value = "/" currency = "/" if currency == "€": currency = "EUR" elif currency == "din": currency = "DIN" down = Downloader(link) content = down.get_content() category = xpath.get(content, '//a[@class="crumbs"]') description = xpath.get( l, '//section[@class="nameSec"]/p[@class="adDescription"]') category = category.split("|") category = category[0] category = category.strip() imageUrl = xpath.get( content, '//div[@class="adThumbnailHolder"]/a/img/@src') imageUrl = imageUrl.replace("//", "/") imageUrl = imageUrl[1::] if imageUrl == "": imageUrl = "/" description = description.replace("...<p>", "") description = description.strip() country = u"Србија" ad = Ad(link, title, imageUrl, description, category, subcategory, value, currency, region, date, country) ads.append(ad) except: pass except: pass return adsToJson(ads) #print scrapeKupujemProdajem()
def __init__(self, DLlist): # On instancie le gestionnaire de preferences et sa fenetre self.preferences = Preferences() # On instancie le gestionnaire de download self.downloader = Downloader() # On instancie le gestionnaire d'historique et sa fenetre self.historique = Historique() current=0 global choice choice='' while choice!='r' and choice!='R': if choice=='s' or choice=='S': os.system(['clear','cls'][os.name == 'nt']) header ('','','Liste de telechargement\t') print "\n\n\n\tSupprimer tous les fichiers de la liste de telchargement" #pour chaque fichier de la liste ***(len(DLlist))!=0 while len(DLlist)>0: print "Supprimer le fichier :",DLlist[0].nom #supprimer le fichier de la liste DLlist.remove(DLlist[0]) #ajouter le fichier au log time.sleep (1) if choice=='t' or choice=='T': os.system(['clear','cls'][os.name == 'nt']) header ('','','Liste de telechargement\t') print "\n\n\n\ttelecharger tous les fichiers" #pour chaque fichier de la liste ***(len(DLlist))!=0 while len(DLlist)>0: if not self.historique.comparerHistorique(DLlist[0]): os.system(['clear','cls'][os.name == 'nt']) header ('','','Liste de telechargement\t') print "\n\n\n\ttelecharger le fichier :",DLlist[0].nom if(DLlist[0].nomFichierSortie=="" ): DLlist[0].nomFichierSortie=os.path.basename(getattr(DLlist[0],"lien" )) #telecharger le fichier self.downloader.lancerTelechargement([[0,DLlist[0].lien,DLlist[0].nomFichierSortie]]) #ajouter le fichier a l'historique de telechargement self.historique.ajouterHistorique(DLlist[0]) else: os.system(['clear','cls'][os.name == 'nt']) header ('','','Liste de telechargement\t') print "\n\n\n\tFichier deja telecharge" #supprimer le fichier de la liste DLlist.remove(DLlist[0]) time.sleep (1) elif choice=='q' or choice=='Q': quitter() elif choice.isdigit() and len(DLlist)>int(choice)+10*current and int(choice)>=0: value='' while (value!='r' and value!='t' and value!='s'): os.system(['clear','cls'][os.name == 'nt']) header ('','','Liste de telechargement\t') print "\n\n\n\tFichier :",DLlist[int(choice)+10*current].nom,"\n\n\tQue voulez vous faire?\n\n\t\tt:telecharger le fichier\n\t\ts:supprimer le fichier de la liste de telechargement\n\t\tr:retour a liste de telechargement\n\n\n\n\n" value=getch() if value=='t': if not self.historique.comparerHistorique(DLlist[int(choice)+10*current]): os.system(['clear','cls'][os.name == 'nt']) header ('','','Liste de telechargement\t') print "\n\n\n\ttelecharger le fichier :",DLlist[int(choice)].nom # Si le nom du fichier de sortie n'existe pas, on l'extrait de l'URL if(DLlist[int(choice)+10*current].nomFichierSortie=="" ): DLlist[int(choice)+10*current].nomFichierSortie=os.path.basename(getattr(DLlist[int(choice)+10*current],"lien" )) #telecharger le fichier self.historique.ajouterHistorique(DLlist[int(choice)+10*current]) self.downloader.lancerTelechargement([[0,DLlist[int(choice)].lien,DLlist[int(choice)].nomFichierSortie]]) #ajouter le fichier a l'historique de telechargement else: os.system(['clear','cls'][os.name == 'nt']) header ('','','Liste de telechargement\t') print "\n\n\n\tFichier deja telecharge" time.sleep(1) #supprimer le fichier de la liste DLlist.remove(DLlist[int(choice)]) elif value=='s': os.system(['clear','cls'][os.name == 'nt']) header ('','','Liste de telechargement\t') print "\n\n\n\tSuppression de la liste de telechargement du fichier :\n\n\t\t",DLlist[int(choice)].nom #supprimer le fichier de la liste DLlist.remove(DLlist[int(choice)]) elif value=='r': os.system(['clear','cls'][os.name == 'nt']) header ('','','Liste de telechargement\t') print "\n\n\n\tRetour a la liste de telechargement" #ajouter le fichier au log time.sleep(1) elif choice=='*' : value='' os.system(['clear','cls'][os.name == 'nt']) header ('','','Liste de telechargement\t') print "\n\n\n\n\n\tQue voulez vous faire?\n\n\t\tt:telecharger les fichiers\n\t\ts:supprimer les fichiers de la liste de telechargement\n\n\n\n\n\n" value=getch() while len(DLlist)>0: if value=='t': if not self.historique.comparerHistorique(DLlist[0]): os.system(['clear','cls'][os.name == 'nt']) header ('','','Liste de telechargement\t') print "\n\n\n\ttelecharger le fichier :",DLlist[0].nom # Si le nom du fichier de sortie n'existe pas, on l'extrait de l'URL if(DLlist[0].nomFichierSortie=="" ): DLlist[0].nomFichierSortie=os.path.basename(getattr(DLlist[0],"lien" )) #telecharger le fichier self.downloader.lancerTelechargement([[0,DLlist[0].lien,DLlist[0].nomFichierSortie]]) #ajouter le fichier a l'historique de telechargement self.historique.ajouterHistorique(DLlist[0]) else: os.system(['clear','cls'][os.name == 'nt']) header ('','','Liste de telechargement\t') print "\n\n\n\tFichier deja telecharge" time.sleep(0.5) #supprimer le fichier de la liste DLlist.remove(DLlist[0]) #ajouter le fichier au log elif choice=='+': if len(DLlist)>current*10+10: current+=1 elif choice=='-': if current!=0: current-=1 choice='' #sauvegarder l'historique de telechargement self.historique.sauverHistorique() #affichage a l'ecran de la liste header ('','','Liste de telechargement\t') for i in range(10): if len(DLlist)>i+10*current: if len(DLlist[int(i+10*current)].nom)>74: print " ",i,":",DLlist[int(i+10*current)].nom[:71]+"..." elif len(DLlist[int(i+10*current)].nom)<=74: print " ",i,":",DLlist[int(i+10*current)].nom else: print "" if len(DLlist)>10:print "\n\t+:fichiers suivants\t-:fichiers precedents (page",current+1,"/",len(DLlist)/10+1,",",len(DLlist),"chaines)" else:print"\n" print "\n\tt:telecharger tous les fichiers s:supprimer tous les fichiers" footer() if len(DLlist)==0: os.system(['clear','cls'][os.name == 'nt']) header ('','','Liste de telechargement\t') print "\n\n\n\n\n\t\tAucun fichier dans la liste" footer() choice='r' time.sleep(1) if not choice:choice=getch()
def download(self, collects, keyword): self.print_with_color("Download Images...", "b") image_count = 0 keyword = 'images/' + keyword # Make directory and path if not os.path.isdir(keyword): print("Make a directory.") os.makedirs(keyword) dir = os.getcwd() + "/" + str(keyword) # os.chdir(dir) self.print_with_color("Download path is %s" % dir, "g") if not dir.endswith("/"): dir += '/' thread_count = self.num_of_thread queue = Queue() self.printProgressBar(0, len(collects), prefix='Progress:', suffix='Complete') for i in range(thread_count): downloader = Downloader(queue, dir, self.collectorName, self.error_list, len(collects)) downloader.daemon = True downloader.start() for i in range(len(collects)): url = collects[i] queue.put(url) queue.join() self.printProgressBar(len(collects), len(collects), prefix='Progress:', suffix='Complete') # Start download # self.printProgressBar(0, len(collects), prefix='Progress:', suffix='Complete') # for i in range(len(collects)): # self.printProgressBar(i+1, len(collects), prefix='Progress:', suffix='Complete') # # print("Download Image (%d/%d)" % (i+1, len(collects))) # col = collects[i] # if (col == '알림'): # pass # else: # full_name = self.collectorName + str(i+1) + ".jpg" # save_path = os.path.join(dir, full_name) # 저장폴더 # # try: # with eventlet.Timeout(self.DOWNLOAD_TIMEOUT): # urllib2.urlretrieve(col, save_path) # except: # self.error_list.append(col) self.print_with_color( "Save %d images" % (len(collects) - len(self.error_list)), "g")
gui = GUI(u'script-%s-main.xml' % addon_name, addon_path).doModal() homewindow.clearProperty("homewindowactive") del gui elif window == "colorconfig": from ColorConfigDialog import ColorConfigDialog gui = ColorConfigDialog(u'script-%s-colorconfig.xml' % addon_name, addon_path).doModal() del gui elif window == "trailers": from TrailerWindow import TrailerWindow gui = TrailerWindow(u'script-%s-trailers.xml' % addon_name, addon_path).doModal() del gui elif window == "favourites": from FavWindow import FavWindow gui = FavWindow(u'script-globalsearch-main.xml', addon_path).doModal() del gui elif window == "dialogalbuminfo": from DialogAlbumInfo import DialogAlbumInfo gui = DialogAlbumInfo(u'script-%s-dialogalbuminfo.xml' % addon_name, addon_path).doModal() del gui elif window == "fullscreeninfo": from FullscreenInfo import FullscreenInfo gui = FullscreenInfo(u'script-%s-fullscreeninfo.xml' % addon_name, addon_path).doModal() del gui elif window == "downloader": from Downloader import Downloader dialog = Downloader() dialog.show_download_dialog() del dialog else: MoveProperties(container, focuscontrol)
def download_from_urls(url_file, out_dir): print("Downloading data from URLS in %s to %s" % (url_file, out_dir)) dl = Downloader(json.loads(open(url_file).read())) dl.save_files(out_dir)
class SchedulerGrid(Scheduler): def __init__(self, name): Scheduler.__init__(self, name) self.states = [ "Acl", "cancelReason", "cancelling","ce_node","children", \ "children_hist","children_num","children_states","condorId","condor_jdl", \ "cpuTime","destination", "done_code","exit_code","expectFrom", \ "expectUpdate","globusId","jdl","jobId","jobtype", \ "lastUpdateTime","localId","location", "matched_jdl","network_server", \ "owner","parent_job", "reason","resubmitted","rsl","seed",\ "stateEnterTime","stateEnterTimes","subjob_failed", \ "user tags" , "status" , "status_code","hierarchy"] return def configure(self, cfg_params): self.cfg_params = cfg_params self.jobtypeName = cfg_params.get('CRAB.jobtype', '') self.schedulerName = cfg_params.get('CRAB.scheduler', '') Scheduler.configure(self, cfg_params) self.proxyValid = 0 self.dontCheckProxy = int(cfg_params.get("GRID.dont_check_proxy", 0)) self.space_token = cfg_params.get("USER.space_token", None) try: self.proxyServer = Downloader( "http://cmsdoc.cern.ch/cms/LCG/crab/config/").config( "myproxy_server.conf") self.proxyServer = self.proxyServer.strip() if self.proxyServer is None: raise CrabException("myproxy_server.conf retrieved but empty") except Exception, e: common.logger.info( "Problem setting myproxy server endpoint: using myproxy.cern.ch" ) common.logger.debug(e) self.proxyServer = 'myproxy.cern.ch' self.group = cfg_params.get("GRID.group", None) self.role = cfg_params.get("GRID.role", None) removeBList = cfg_params.get("GRID.remove_default_blacklist", 0) blackAnaOps = None if int(removeBList) == 0: blacklist = Downloader( "http://cmsdoc.cern.ch/cms/LCG/crab/config/") result = blacklist.config("site_black_list.conf") if result != None: blackAnaOps = result common.logger.debug("Enforced black list: %s " % blackAnaOps) else: common.logger.info("WARNING: Skipping default black list!") self.EDG_ce_black_list = None if cfg_params.has_key( 'GRID.ce_black_list') and cfg_params['GRID.ce_black_list']: self.EDG_ce_black_list = cfg_params.get('GRID.ce_black_list') if int(removeBList) == 0 and blackAnaOps: self.EDG_ce_black_list += ",%s" % blackAnaOps elif int(removeBList) == 0 and blackAnaOps: self.EDG_ce_black_list = blackAnaOps if self.EDG_ce_black_list: self.EDG_ce_black_list = str(self.EDG_ce_black_list).split(',') self.EDG_ce_white_list = cfg_params.get('GRID.ce_white_list', None) if (self.EDG_ce_white_list): self.EDG_ce_white_list = str(self.EDG_ce_white_list).split(',') self.VO = cfg_params.get('GRID.virtual_organization', 'cms') self.EDG_clock_time = cfg_params.get('GRID.max_wall_clock_time', None) # Default minimum CPU time to >= 130 minutes self.EDG_cpu_time = cfg_params.get('GRID.max_cpu_time', '130') ## Add EDG_WL_LOCATION to the python path #if not self.CRAB_useServer and not self.CRAB_serverName: # if not os.environ.has_key('EDG_WL_LOCATION'): # msg = "Error: the EDG_WL_LOCATION variable is not set." # raise CrabException(msg) # path = os.environ['EDG_WL_LOCATION'] # libPath=os.path.join(path, "lib") # sys.path.append(libPath) # libPath=os.path.join(path, "lib", "python") # sys.path.append(libPath) self.checkProxy() return
from Controlador import Controlador from Downloader import Downloader if __name__ == '__main__': control = Controlador() Vista = Downloader(control) control.setVista(Vista) control.iniciar()
class SchedulerGrid(Scheduler): def __init__(self, name): Scheduler.__init__(self,name) self.states = [ "Acl", "cancelReason", "cancelling","ce_node","children", \ "children_hist","children_num","children_states","condorId","condor_jdl", \ "cpuTime","destination", "done_code","exit_code","expectFrom", \ "expectUpdate","globusId","jdl","jobId","jobtype", \ "lastUpdateTime","localId","location", "matched_jdl","network_server", \ "owner","parent_job", "reason","resubmitted","rsl","seed",\ "stateEnterTime","stateEnterTimes","subjob_failed", \ "user tags" , "status" , "status_code","hierarchy"] return def configure(self, cfg_params): self.cfg_params = cfg_params self.jobtypeName = cfg_params.get('CRAB.jobtype','') self.schedulerName = cfg_params.get('CRAB.scheduler','') Scheduler.configure(self,cfg_params) self.proxyValid=0 self.dontCheckProxy=int(cfg_params.get("GRID.dont_check_proxy",0)) self.space_token = cfg_params.get("USER.space_token",None) try: self.proxyServer = Downloader("http://cmsdoc.cern.ch/cms/LCG/crab/config/").config("myproxy_server.conf") self.proxyServer = self.proxyServer.strip() if self.proxyServer is None: raise CrabException("myproxy_server.conf retrieved but empty") except Exception, e: common.logger.info("Problem setting myproxy server endpoint: using myproxy.cern.ch") common.logger.debug(e) self.proxyServer= 'myproxy.cern.ch' self.group = cfg_params.get("GRID.group", None) self.role = cfg_params.get("GRID.role", None) removeBList = cfg_params.get("GRID.remove_default_blacklist", 0 ) blackAnaOps = None if int(removeBList) == 0: blacklist = Downloader("http://cmsdoc.cern.ch/cms/LCG/crab/config/") result = blacklist.config("site_black_list.conf") if result != None: blackAnaOps = result common.logger.debug("Enforced black list: %s "%str(blacklist)) else: common.logger.info("WARNING: Skipping default black list!") self.EDG_ce_black_list = None if cfg_params.has_key('GRID.ce_black_list') and cfg_params['GRID.ce_black_list']: self.EDG_ce_black_list = cfg_params.get('GRID.ce_black_list') if int(removeBList) == 0 and blackAnaOps: self.EDG_ce_black_list += ",%s"%blackAnaOps elif int(removeBList) == 0 and blackAnaOps: self.EDG_ce_black_list = blackAnaOps if self.EDG_ce_black_list: self.EDG_ce_black_list = str(self.EDG_ce_black_list).split(',') self.EDG_ce_white_list = cfg_params.get('GRID.ce_white_list',None) if (self.EDG_ce_white_list): self.EDG_ce_white_list = str(self.EDG_ce_white_list).split(',') self.VO = cfg_params.get('GRID.virtual_organization','cms') self.EDG_clock_time = cfg_params.get('GRID.max_wall_clock_time',None) # Default minimum CPU time to >= 130 minutes self.EDG_cpu_time = cfg_params.get('GRID.max_cpu_time', '130') ## Add EDG_WL_LOCATION to the python path #if not self.CRAB_useServer and not self.CRAB_serverName: # if not os.environ.has_key('EDG_WL_LOCATION'): # msg = "Error: the EDG_WL_LOCATION variable is not set." # raise CrabException(msg) # path = os.environ['EDG_WL_LOCATION'] # libPath=os.path.join(path, "lib") # sys.path.append(libPath) # libPath=os.path.join(path, "lib", "python") # sys.path.append(libPath) self.checkProxy() return
addon_path).doModal() del gui elif window == "trailers": from TrailerWindow import TrailerWindow gui = TrailerWindow(u'script-%s-trailers.xml' % addon_name, addon_path).doModal() del gui elif window == "favourites": from FavWindow import FavWindow gui = FavWindow(u'script-globalsearch-main.xml', addon_path).doModal() del gui elif window == "dialogalbuminfo": from DialogAlbumInfo import DialogAlbumInfo gui = DialogAlbumInfo( u'script-%s-dialogalbuminfo.xml' % addon_name, addon_path).doModal() del gui elif window == "fullscreeninfo": from FullscreenInfo import FullscreenInfo gui = FullscreenInfo(u'script-%s-fullscreeninfo.xml' % addon_name, addon_path).doModal() del gui elif window == "downloader": from Downloader import Downloader dialog = Downloader() dialog.show_download_dialog() del dialog else: MoveProperties(container, focuscontrol)
def scrapeHaloOglasi(): # UTF-8 support reload(sys) sys.setdefaultencoding('utf-8') now = datetime.now() down = Downloader( 'http://www.halooglasi.com/naslovna.240.html?search_text=&sortColumn=VremeDodavanja' ) content = down.get_content() html = unicode(content) celo = xpath.get(html, '//div[@class="results_container"]') linkovi = xpath.search(celo, '//div[@class="result_brza"]') ads = [] for l in linkovi: link = xpath.get(l, '//div[@style="height:auto;"]/h2/a/@href') link = "http://www.halooglasi.com" + link download = Downloader(link) cont = download.get_content() cont = unicode(cont) title = xpath.get(cont, '//div[@class="detail_bar_nek"]/h2').strip() if title == "": title = xpath.get(cont, '//div[@class="detail_bar"]/h2').strip() imageUrl = xpath.get(l, '//a[@class="thumb"]/img/@src') imageUrl = "http://www.halooglasi.com" + imageUrl description = xpath.get(l, '//div[@class="text_ogl"]/p') kategorija = xpath.get(l, '//div[@class="brza_link"]').strip() kategorija = kategorija.split("\r\n\t\t\t\t\t\t\r\n\t\t\t\t\t\t") kategorija = kategorija[1] kategorija = kategorija.split(" > ") category = kategorija[0] if len(kategorija) > 2: subcategory = kategorija[1] else: kategorija = kategorija[1].split("'>") kategorija = kategorija[1] kategorija = kategorija.split("<") subcategory = kategorija[0] price = xpath.get(cont, '//div[@class="price"]').strip() #price if price == "": price = xpath.get( cont, '//div[@class="price deal"]').strip() #price deal price = price.replace(".", "") price = price.replace("din", " DIN") price = price.replace("€", " EUR") if price == "Dogovor": value = "/" currency = "/" else: price = price.split(" ") value = price[0] currency = price[1] date_loc = xpath.search(l, '//div[@class="datum_grad"]/h6/span') date_loc[0] = date_loc[0].strip() date = date_loc[0].split("\r\n") date = date[0] date = date.replace(".", "") date = date.split(" ") if date[1] == "Jan": date[1] = "1" elif date[1] == "Feb": date[1] = "2" elif date[1] == "Mar": date[1] = "3" elif date[1] == "Apr": date[1] = "4" elif date[1] == "Maj": date[1] = "5" elif date[1] == "Jun": date[1] = "6" elif date[1] == "Jul": date[1] = "7" elif date[1] == "Avg": date[1] = "8" elif date[1] == "Sep": date[1] = "9" elif date[1] == "Okt": date[1] = "10" elif date[1] == "Nov": date[1] = "11" elif date[1] == "Dec": date[1] = "12" date = date[2] + "-" + date[1] + "-" + date[0] l = date_loc[1].strip() l = l.split(" ") region = l[0] country = u"Србија" ad = Ad(link, title, imageUrl, description, category, subcategory, value, currency, region, date, country) ads.append(ad) return adsToJson(ads) #print scrapeHaloOglasi()
class ServerConfig: def __init__(self, serverName): import string self.serverName = string.lower(serverName) common.logger.debug('Calling ServerConfig '+self.serverName) url ='http://cmsdoc.cern.ch/cms/LCG/crab/config/' self.downloader = Downloader(url) def config(self): """ """ if 'default' in self.serverName: self.serverName = self.selectServer() if 'server_' in self.serverName: configFileName = '%s.conf'%self.serverName else: configFileName = 'server_%s.conf'%self.serverName serverConfig = eval(self.downloader.config(configFileName)) if not serverConfig: serverConfig = {} serverConfig['serverGenericName']=self.serverName return serverConfig def selectServer(self): """ """ common.logger.debug('getting serverlist from web') # get a list of available servers serverListFileName ='AvailableServerList' serverListFile = self.downloader.config(serverListFileName) if not serverListFile: msg = 'List of avalable Server '+serverListFileName+' from '+self.url+' is empty\n' msg += 'Please report to CRAB feedback hypernews [email protected]' raise CrabException(msg) # clean up empty lines and comments serverList=[] [serverList.append(string.split(string.strip(it))) for it in serverListFile.split('\n') if (it.strip() and not it.strip()[0]=="#")] common.logger.debug('All avaialble servers: '+str(serverList)) # select servers from client version compatibleServerList=[] for s in serverList: #vv=string.split(s[1],'-') if len(s)<2: continue vv=s[1].split('-') if len(vv[0])==0: vv[0]='0.0.0' if len(vv[1])==0: vv[1]='99.99.99' for i in 0,1: tmp=[] [tmp.append(int(t)) for t in vv[i].split('.')] vv[i]=tuple(tmp) if vv[0]<=common.prog_version and common.prog_version<=vv[1] and common.scheduler.name()==string.lower(s[2]): compatibleServerList.append(s[0]) common.logger.debug('All avaialble servers compatible with %s: '%common.prog_version_str +str(compatibleServerList)) if len(compatibleServerList)==0: msg = "No compatible server for client version %s and scheduler %s\n"%(common.prog_version_str,common.scheduler.name()) msg += "Exiting" common.logger.info(msg) raise CrabException(msg) # if more than one, pick up a random one, waiting for something smarter (SiteDB) import random serverName = random.choice(compatibleServerList) common.logger.debug('Avaialble servers: '+str(compatibleServerList)+' choosen: '+serverName) return serverName
def threader_crawler(seed_url, resource_regiex=None, link_regiex=".*", delay=5, cache=None, download_source_callback=None, user_agent="wswp", proxies=None, num_retries=1, max_threads=10, timeout=60, max_url=500): downloaded = [] crawl_queue = [seed_url] seen = set([seed_url]) D = Downloader(cache=cache, delay=delay, user_agent=user_agent, proxies=proxies, num_retries=num_retries, timeout=timeout) print(user_agent) def process_queue(): while True: links = [] try: url = crawl_queue.pop() except IndexError: break else: html = D(url) downloaded.append(url) if download_source_callback: if resource_regiex and re.match(resource_regiex, url): download_source_callback(url, html) links.extend([ link for link in get_links(html) if re.match(link_regiex, link) ]) for link in links: link = normalize(seed_url, link) if link not in seen: seen.add(link) if same_domain(seed_url, link): crawl_queue.append(link) print("已经发现的总网页数目为", len(seen)) print("已经下载过的网页数目为", len(downloaded)) print("还没有遍历过的网页数目为", len(crawl_queue)) threads = [] while threads or crawl_queue: if len(downloaded) == max_url: return for thread in threads: if not thread.is_alive(): threads.remove(thread) while len(threads) < max_threads and crawl_queue: print("线程数量为", len(threads)) thread = threading.Thread(target=process_queue) thread.setDaemon(True) thread.start() print("线程数量为", len(threads)) threads.append(thread)
def run(self): tempPath = path + self.video.name + ".mp4" self.downLoader = Downloader(self.video.videoUrl, tempPath) self.downLoader.downLoad() self.singal.emit()
def __init__(self): print 'init' self.urlManager = UrlManager() self.downloader = Downloader() self.praser = HtmlPraser() self.outputer = Output()
def download32bit(self): url32bit = 'https://go.microsoft.com/fwlink/?LinkID=623231' Downloader(self.directoryName,self.IDEName,url32bit)
class ServerConfig: def __init__(self, serverName): import string self.serverName = string.lower(serverName) common.logger.debug('Calling ServerConfig ' + self.serverName) url = 'http://cmsdoc.cern.ch/cms/LCG/crab/config/' self.downloader = Downloader(url) def config(self): """ """ if 'default' in self.serverName: self.serverName = self.selectServer() if 'server_' in self.serverName: configFileName = '%s.conf' % self.serverName else: configFileName = 'server_%s.conf' % self.serverName serverConfig = eval(self.downloader.config(configFileName)) if not serverConfig: serverConfig = {} serverConfig['serverGenericName'] = self.serverName return serverConfig def selectServer(self): """ """ common.logger.debug('getting serverlist from web') # get a list of available servers serverListFileName = 'AvailableServerList' serverListFile = self.downloader.config(serverListFileName) if not serverListFile: msg = 'List of avalable Server ' + serverListFileName + ' from ' + self.url + ' is empty\n' msg += 'Please report to CRAB feedback hypernews [email protected]' raise CrabException(msg) # clean up empty lines and comments serverList = [] [ serverList.append(string.split(string.strip(it))) for it in serverListFile.split('\n') if (it.strip() and not it.strip()[0] == "#") ] common.logger.debug('All avaialble servers: ' + str(serverList)) # select servers from client version compatibleServerList = [] for s in serverList: #vv=string.split(s[1],'-') if len(s) < 2: continue vv = s[1].split('-') if len(vv[0]) == 0: vv[0] = '0.0.0' if len(vv[1]) == 0: vv[1] = '99.99.99' for i in 0, 1: tmp = [] [tmp.append(int(t)) for t in vv[i].split('.')] vv[i] = tuple(tmp) if vv[0] <= common.prog_version and common.prog_version <= vv[ 1] and common.scheduler.name() == string.lower(s[2]): compatibleServerList.append(s[0]) common.logger.debug('All avaialble servers compatible with %s: ' % common.prog_version_str + str(compatibleServerList)) if len(compatibleServerList) == 0: msg = "No compatible server for client version %s and scheduler %s\n" % ( common.prog_version_str, common.scheduler.name()) msg += "Exiting" common.logger.info(msg) raise CrabException(msg) # if more than one, pick up a random one, waiting for something smarter (SiteDB) import random serverName = random.choice(compatibleServerList) common.logger.debug('Avaialble servers: ' + str(compatibleServerList) + ' choosen: ' + serverName) return serverName
def download64bit(self): url64bit = 'https://go.microsoft.com/fwlink/?Linkid=850641' Downloader(self.directoryName,self.IDEName,url64bit)
def link_crawler(seed_url, link_regex=None, delay=5, max_depth=-1, max_urls=-1, headers=None, proxies=None, cookies=None, num_retries=5, scrape_callback=None, cache=None, max_threads=10, timeout=60): """Crawl from the given seed URL following links matched by link_regex Crawl this website in mltiple threads seed_url:种子链接,爬虫的初始链接 delay:避免请求太频繁,所采取的延时 max_depth:从种子链接向下,所链接的深度 max_urls:连续max_urls次下载同一个网页,得到404后,停止下载,跳出循环 headers:请求头 proxy:传入ProxiesPool类 num_retries:当网页的返回识别码在400和500之间时,重复下载的数,如果还不能下载返回空字符串 cahce:是否存在缓存,用法例如: cache = MongoCache(expires=timedelta()) D = Downloader(cache=cache) html = D('http://esf.nb.fang.com/') """ # the queue of URL's that still need to be crawled crawl_queue = deque([seed_url]) # the URL's that have been seen and at what depth seen = {seed_url: 0} # track how many URL's have been downloaded num_urls = {seed_url: 0} # rp = get_robots(seed_url) headers = headers D = Downloader(delay=5, headers=headers, cookies=cookies, proxies=proxies, num_retries=num_retries, cache=cache) def process_queue(): while True: try: url = crawl_queue.pop() except IndexError: break else: download_time = 1 while download_time <= 5: html = D(url) download_time = download_time + 1 print('we download the url %s times' % download_time) links = [] if scrape_callback: links.extend(scrape_callback(url, html) or []) next_urls = get_links(html) if next_urls: seen[seed_url] = seen[seed_url] + 1 break if link_regex: links.extend(link for link in next_urls if re.match(link_regex, link)) else: links.extend(link for link in next_urls) for link in links: if seen[seed_url] != max_depth: link = normalize(seed_url, link) # check whether already crawled this link if link not in seen and same_domain(seed_url, link): # check link is within same domain # success! add this new link to queue crawl_queue.append(link) # check whether have reached downloaded maximum try: num_urls[url] += 1 except KeyError: num_urls[url] = 1 if num_urls[url] == max_urls: break threads = [] while threads or crawl_queue: # the crawl is still activate for thread in threads: if not thread.is_alive(): # remove the stopped threads threads.remove(thread) while len(threads) < max_threads and crawl_queue: # can start some more threads thread = threading.Thread(target=process_queue) # set daemon so main thread can exit when receives ctrl-c thread.setDaemon(True) thread.start() # thread.run() threads.append(thread)
def scrapeAvtooglasi(): # UTF-8 support reload(sys) sys.setdefaultencoding('utf-8') now = datetime.now() down = Downloader('http://www.avtooglasi.com.mk/rezultati/show/?vid=0&orderby=0') content = down.get_content() html = unicode(content) sliki = xpath.search(html, '//div[@class="resultLeft"]') ostanato = xpath.search(html, '//div[@class="oglasInfoTopContent"]') ceni = xpath.search(html, '//a[@class="btn btn-info btn-xs oglasInfoAdditionalPrice"]') link = {} title = {} imageUrl = {} description = {} category = {} subcategory = {} value = {} currency = {} region = {} date = {} i = 0 ads = [] for slika in sliki: imageUrl[i] = xpath.search(slika, '//a[@class="thumbnail resultImg"]/img/@src')[0] i = i + 1 i = 0 for cena in ceni: price = xpath.get(cena,'//span/span').strip() price=price.split(" ") if len(price)>1: if price[0]=="По": price[0]="/" if price[1]=="договор": price[1]="/" value[i]=price[0] currency[i]=price[1] if currency[i]=="€": currency[i]="EUR" i = i + 1 i = 0 for advert in ostanato: link[i] = xpath.get(advert, '//a[@class="resultMainLink"]/@href') title[i] = xpath.get(advert, '//a[@class="resultMainLink"]/span').strip().replace("\"", "") path = xpath.search(getDescription(link[i],'//div[@class="centerC"]'), '/div/div[@class="padded"]') description[i] = path[1] subcategory[i]="/" category[i] = u"Возила" dodatok = xpath.get(advert, '//span[@class="oglasInfoAdditionalInfo"]') dodatok = dodatok.split(" | ") region[i] = dodatok[0] country = u"Македеонија" description[i] = dodatok[1] + u" година, "+ dodatok[2] +", "+ dodatok[3] +", "+ dodatok[4] +", "+ dodatok[5] +", "+ description[i] description[i] = description[i].strip().replace("\"", "") date[i]="" #print description[i] datum = dodatok[6].strip() datum = datum.split(" ") if datum[0]=="Денес": datum [0]= str(now.year)+"-"+str(now.month)+"-"+str(now.day) date[i]=datum[0]+" "+datum[2] elif datum[0]=="Вчера": da=datetime.now()-timedelta(days=1) datum[0]=str(da.year)+"-"+str(da.month)+"-"+str(da.day) date[i]=datum[0]+" "+datum[2] elif datum[0]=="пред": if datum[2]=="дена": da=datetime.now()-timedelta(days=int(datum[1])) datum[0]=str(da.year)+"-"+str(da.month)+"-"+str(da.day) date[i]=datum[0] else: if datum[1]=="1": da=datetime.now()-timedelta(days=30) datum[0]=str(da.year)+"-"+str(da.month)+"-"+str(da.day) date[i]=datum[0] else: da=datetime.now()-timedelta(days=60) datum[0]=str(da.year)+"-"+str(da.month)+"-"+str(da.day) date[i]=datum[0] else: date[i]=datum[0]+" "+datum[1] #print date[i] i = i + 1 for i in link: ad = Ad(link[i], title[i], imageUrl[i], description[i], category[i], subcategory[i], value[i], currency[i], region[i], date[i], country) ads.append(ad) return adsToJson(ads) # print scrapeAvtooglasi()
def start(self): # Open the portolio self.portfolio = Portfolio(cash=0, sl=self.sl, tp=self.tp) self.portfolio.load(self.filename) # Download the stock data self.downloader = Downloader(cache=False) self.downloader.download(period=self.period) date = self.downloader.prices.index[-1] prices = self.downloader.prices print("#", date) # Refresh the portfolio with the latest prices self.portfolio.refresh(prices=prices, date=date) # Get the portfolio Summary portfolio_summary = self.portfolio.summary() # Get the opened positions opened = self.portfolio.holdings[self.portfolio.holdings['status'] == 'open'] # Get the current trend zt = TradingZones(prices=prices) zones = zt.get() output = pd.DataFrame() # Activate the trailing stops targets = self.portfolio.getPositionsBelowTS(self.ts) if len(targets) > 0: sell_orders = self.portfolio.getSellOrdersFromSubset( subset=targets) print("\n----- TS -----\n", sell_orders) #self.portfolio.sell(sell_orders, prices, date, label='TS') # Activate the stop loss if self.sl < 0: poor_perfs = opened[opened['profits_pct'] <= self.sl] if len(poor_perfs) > 0: sell_list = poor_perfs['symbol'].unique() sell_orders = self.portfolio.getSellAllOrders( symbols=sell_list) print("\n----- SL -----\n", sell_orders) #self.portfolio.sell(sell_orders, prices, date, label='SL') # Sell all on drop if self.sellAllOnCrash == True and zones.iloc[-1][ 'signal'] == 'DOWN' and len(opened) > 0: print("Sell all") print("\n----- Sell All -----\n") #self.portfolio.sellAll(prices, date, label='Market Drop') # Get the ones that reached take-profit if self.tp > 0: good_perfs = opened[opened['profits_pct'] >= self.tp] if len(good_perfs) > 0: sell_list = good_perfs['symbol'].unique() sell_orders = self.portfolio.getSellAllOrders( symbols=sell_list) print("\n----- TP -----\n", sell_orders) #self.portfolio.sell(sell_orders, prices, date, label='TP') # Avoid trading in a general market downtrend if self.avoidDowntrends == False or (self.avoidDowntrends == True and zones.iloc[-1]['signal'] == 'UP'): # Pick the stocks picker = Picker(prices=prices) picker.pick(count=self.stockPicks) #print(">", picker.selected) # Get the order size portfolio_summary = self.portfolio.summary() order = Order(prices=prices, picks=picker.selected) order.get(budget=portfolio_summary['cash']) #print("!", portfolio_summary['cash'], order.orders) print("\n----- BUY -----\n", order.orders) # Update our portfolio #self.portfolio.update(orders=order.orders, prices=prices, date=date) else: # No new positions in a downtrend! # Update our portfolio self.portfolio.update(orders=pd.DataFrame(), prices=prices, date=date)
def threaded_crawler(seed_url, delay=5, cache=None, scrape_callback=None, proxies=None, num_retries=1, max_threads=10, timeout=60): """Crawl this website in multiple threads seed_url must be list """ # the queue of URL's that still need to be crawled crawl_queue = deque(seed_url) # the URL's that have been seen seen = dict((url, 0) for url in seed_url) D = Downloader(cache=cache, delay=delay, proxies=proxies, num_retries=num_retries) def process_queue(): while True: try: url = crawl_queue.pop() except IndexError: # crawl queue is empty break else: html = D(url) if scrape_callback: try: links = scrape_callback(url, html) or [] except Exception as e: print('Error in callback for: {}: {}'.format(url, e)) else: for link in links: link = normalize(seed_url, link) # check whether already crawled this link if link not in seen: # add this new link to queue crawl_queue.appendleft(link) # wait for all download threads to finish threads = [] while threads or crawl_queue: # the crawl is still active for thread in threads: if not thread.is_alive(): # remove the stopped threads threads.remove(thread) while len(threads) < max_threads and crawl_queue: # can start some more threads thread = threading.Thread(target=process_queue) thread.setDaemon( True ) # set daemon so main thread can exit when receives ctrl-c thread.start() threads.append(thread) # all threads have been processed # sleep temporarily so CPU can focus execution on other threads time.sleep(SLEEP_TIME)
from Downloader import Downloader url_metadata = "https://os.unil.cloud.switch.ch/fma/fma_metadata.zip" url_small = "https://os.unil.cloud.switch.ch/fma/fma_small.zip" url_medium = "https://os.unil.cloud.switch.ch/fma/fma_medium.zip" url_large = "https://os.unil.cloud.switch.ch/fma/fma_large.zip" url_full = "https://os.unil.cloud.switch.ch/fma/fma_full.zip" sha1sum_metadata = 'f0df49ffe5f2a6008d7dc83c6915b31835dfe733' sha1sum_small = 'ade154f733639d52e35e32f5593efe5be76c6d70' sha1sum_medium = 'c67b69ea232021025fca9231fc1c7c1a063ab50b' sha1sum_large = '497109f4dd721066b5ce5e5f250ec604dc78939e' sha1sum_full = '0f0ace23fbe9ba30ecb7e95f763e435ea802b8ab' threads = 10 code_geass = 'https://ca-east.uploadfiles.io/get/6tnazj0v' obj = Downloader(url=code_geass, number_of_threads=threads, destination_directory='D:\\Downloads') obj.start_download() # obj.merge_chunks() print(obj.get_metadata()) print(obj.get_remote_crc32c()) print(obj.get_downloaded_crc32c_large()) print(obj.get_downloaded_sha1_large())
class Sim: def __init__(self, period='2y', timedelay=100, window=100, timestep=5, budget=5000, stockPicks=10, sl=-0.1, tp=0.25, ts=0.2, avoidDowntrends=True, sellAllOnCrash=True, neptune=None, pick_kwargs={}): self.neptune = neptune # neptune.ai logging self.period = period self.timedelay = timedelay self.timestep = timestep self.budget = budget self.stockPicks = stockPicks self.sl = sl self.tp = tp self.ts = ts self.avoidDowntrends = avoidDowntrends self.sellAllOnCrash = sellAllOnCrash self.pick_kwargs = pick_kwargs self.portfolio = Portfolio(cash=budget, sl=self.sl, tp=self.tp) self.downloader = Downloader() self.downloader.download(period=self.period) self.current_index = timedelay self.is_notebook = self.isNotebook() def isNotebook(self): try: cfg = get_ipython().config return True except NameError: return False def progress(self, count, total=100, status=''): if self.is_notebook == True: return HTML(""" <progress value='{value}' max='{max}', style='width: 50%' > {value} </progress> """.format(value=count, max=total)) else: bar_len = 60 filled_len = int(round(bar_len * count / float(total))) percents = round(100.0 * count / float(total), 1) bar = '=' * filled_len + '-' * (bar_len - filled_len) sys.stdout.write('[%s] %s%s\r' % (bar, percents, '%')) sys.stdout.flush() def run(self): #out = display(self.progress(0, 100), display_id=True) steps = (len(self.downloader.prices) - self.timedelay) stepN = 0 self.stats = pd.DataFrame(columns=[ 'invested', 'portfolio_value', 'Positions', 'closed', 'opened', 'cash', 'total_value', 'portfolio_resistances', 'portfolio_supports', 'drawdown', 'drawdown_pct' ]) while self.current_index + self.timestep < len(self.downloader.prices): output = self.tick() # Neptune logging if self.neptune is not None: for k in list(output.keys()): self.neptune.log_metric(k, output[k]) output['date'] = self.downloader.prices.index[self.current_index] self.stats = self.stats.append(output, ignore_index=True) stepN = stepN + 1 pct = math.ceil((stepN / steps) * 100) if self.is_notebook == True: out.update(self.progress(pct, 100)) else: self.progress(pct, 100) #print(str(round(pct))+"%") self.portfolio.sellAll( prices=self.downloader.prices[:self.current_index], date=self.downloader.prices.index[self.current_index], label='End Sim') self.stats = self.stats.set_index('date') return self.stats # Run every night after the market close def tick(self): #print('\n\n----------- tick -----------') # Get the latest prices & date as of today date = self.downloader.prices.index[self.current_index] prices = self.downloader.prices[:self.current_index + 1] # Include the current price #print("#", date) # Refresh the portfolio with the latest prices self.portfolio.refresh(prices=prices, date=date) # Get the portfolio Summary portfolio_summary = self.portfolio.summary() # Get the opened positions opened = self.portfolio.holdings[self.portfolio.holdings['status'] == 'open'] # Get the current trend zt = TradingZones(prices=prices) zones = zt.get() # Activate the trailing stops targets = self.portfolio.getPositionsBelowTS(self.ts) if len(targets) > 0: sell_orders = self.portfolio.getSellOrdersFromSubset( subset=targets) #print("\n-----------\n",targets,"\n\n",sell_orders) self.portfolio.sell(sell_orders, prices, date, label='TS') # Activate the stop loss if self.sl < 0: poor_perfs = opened[opened['profits_pct'] <= self.sl] if len(poor_perfs) > 0: sell_list = poor_perfs['symbol'].unique() sell_orders = self.portfolio.getSellAllOrders( symbols=sell_list) self.portfolio.sell(sell_orders, prices, date, label='SL') # Sell all on drop if self.sellAllOnCrash == True and zones.iloc[-1][ 'signal'] == 'DOWN' and len(opened) > 0: print("Sell all") self.portfolio.sellAll(prices, date, label='Market Drop') # Get the ones that reached take-profit if self.tp > 0: good_perfs = opened[opened['profits_pct'] >= self.tp] if len(good_perfs) > 0: sell_list = good_perfs['symbol'].unique() sell_orders = self.portfolio.getSellAllOrders( symbols=sell_list) self.portfolio.sell(sell_orders, prices, date, label='TP') # Avoid trading in a general market downtrend if self.avoidDowntrends == False or (self.avoidDowntrends == True and zones.iloc[-1]['signal'] == 'UP'): # Pick the stocks picker = Picker(prices=prices) picker.pick(count=self.stockPicks, kwargs=self.pick_kwargs) #print(">", picker.selected) # Get the order size portfolio_summary = self.portfolio.summary() order = Order(prices=prices, picks=picker.selected) order.get(budget=portfolio_summary['cash']) #print("!", portfolio_summary['cash'], order.orders) # Update our portfolio self.portfolio.update( orders=order.orders, prices=prices, date=self.downloader.prices.index[self.current_index]) else: # No new positions in a downtrend! # Update our portfolio self.portfolio.update( orders=pd.DataFrame(), prices=prices, date=self.downloader.prices.index[self.current_index]) # Update the index if self.current_index + self.timestep < len(self.downloader.prices): self.current_index = self.current_index + self.timestep portfolio_summary = self.portfolio.summary() return portfolio_summary
from Downloader import Downloader from Builder import Builder from Utils import loadconfig from Controller import Controller if __name__ == '__main__': # -----------------load config-------------------- configpath = "Config/config.json" configs = loadconfig(configpath) dir_config = configs["Dir"] file_config = configs["File"] # -----------------create classes-------------- builder = Builder(configpath) downloader = Downloader(dir_config["Save_Path"]) controller = Controller(file_config["MusicList"]) # ------------------------------------------ controller = controller.init(downloader, builder) controller.AutoDownload()
def link_crawler(seed_url, link_regex=".*", resource_regex=".*", delay=5, max_depth=-1, max_urls=-1, headers=None, user_agent="wswp", proxies=None, num_retries=1, download_source_callback=None, cache=None, data=None): ''' ''' #用于存储未被下载的页面 craw_queue = [seed_url] #用于存储已经下载过的页面 seen = [seed_url] #下载过的url数量 num_urls = 0 isdow = [] downloader = Downloader(delay=delay, user_agent=user_agent, proxies=proxies, num_retries=num_retries, cache=cache) while craw_queue: url = craw_queue.pop() #depth = seen[] html = downloader(url) print(html) links = [] isdow.append(url) seen.append(url) if download_source_callback and re.match(resource_regex, url): #如果是资源页面,下载页面上的所有资源 print("属于资源,开始下载") download_source_callback(url, html) #if depth != max_depth: links.extend( [link for link in get_links(html) if re.search(link_regex, link)]) for link in links: link = normalize(seed_url, link) if link not in seen: seen.append(link) if same_domain(seed_url, link): craw_queue.append(link) print(link + "符合要求加入队列,现在队列的长度是" + str(len(craw_queue))) #craw_queue.append(link) #print(link + "符合要求加入队列,现在队列的长度是" + str(len(craw_queue))) print("当前可怕去的url数量胃" + str(len(craw_queue))) print("已经发现的总url数量为", str(len(set(seen)))) print("已经下载过的url数量为", str(len(isdow))) num_urls += 1 if num_urls == max_urls: break print(seed_url + "上所有符合要求的网站下载完毕,一共遍历了" + str(len(seen)) + "个网页")
def __init__( self ): # Appel au constructeur de la classe mere QtGui.QMainWindow.__init__( self ) ########### # Fenetre # ########### ### # Reglages de la fenetre principale ### # Nom de la fenetre self.setWindowTitle( "TVDownloader" ) # Mise en place de son icone self.setWindowIcon( QtGui.QIcon( "ico/TVDownloader.png" ) ) ### # Mise en place des widgets dans la fenetre ### # Widget central qui contiendra tout self.centralWidget = QtGui.QWidget( self ) # # Barre du haut # # Layout horizontal qui contiendra les listes deroulantes self.horizontalLayoutBarreHaut = QtGui.QHBoxLayout() # Liste deroulante pour choisir le site (plugin) self.comboBoxSite = QtGui.QComboBox( self.centralWidget ) self.horizontalLayoutBarreHaut.addWidget( self.comboBoxSite ) # Liste deroulante pour choisir une chaine du site courant self.comboBoxChaine = QtGui.QComboBox( self.centralWidget) self.horizontalLayoutBarreHaut.addWidget( self.comboBoxChaine ) # Liste deroulante pour choisir une emission de la chaine courante self.comboBoxEmission = QtGui.QComboBox( self.centralWidget ) self.horizontalLayoutBarreHaut.addWidget( self.comboBoxEmission ) # # Onglets # # Gestionnaire onglets self.tabWidget = QtGui.QTabWidget( self.centralWidget ) # Onglet Fichiers self.tabFichiers = QtGui.QSplitter( self.centralWidget ) # L'onglet Fichier contient un splitter self.tabWidget.addTab( self.tabFichiers, u"Choix des fichiers" ) # Onglet Telechargements self.tabTelechargements = QtGui.QWidget( self.centralWidget ) self.tabWidget.addTab( self.tabTelechargements, u"Téléchargements" ) # # Liste des fichiers # # Layout de grille qui contient le tableau qui liste les fichiers + boutons self.gridLayoutFichiers = QtGui.QGridLayout( self.tabFichiers ) # Tableau qui contient la liste des fichiers disponibles pour l'emission courante self.tableWidgetFichier = MyQTableWidget( self.tabFichiers ) # Il a 4 colonnes et 0 ligne (pour l'instant) self.tableWidgetFichier.setColumnCount( 3 ) self.tableWidgetFichier.setRowCount( 0 ) # On ajoute les titres self.tableWidgetFichier.setHorizontalHeaderItem( 0, self.tableWidgetFichier.creerItem( "" ) ) self.tableWidgetFichier.setHorizontalHeaderItem( 1, self.tableWidgetFichier.creerItem( "Date" ) ) self.tableWidgetFichier.setHorizontalHeaderItem( 2, self.tableWidgetFichier.creerItem( "Emission" ) ) # On l'ajoute au layout self.gridLayoutFichiers.addWidget( self.tableWidgetFichier, 0, 1, 6, 1 ) # Icones du tableWidget self.iconeFichier = QtGui.QIcon( "ico/gtk-file.svg" ) self.iconeAjoute = QtGui.QIcon( "ico/gtk-add.svg" ) self.iconeTelecharge = QtGui.QIcon( "ico/gtk-apply.svg" ) # Bouton pour ajouter tous les fichiers a la liste des telechargements self.pushButtonToutAjouter = MyQPushButton( self.tabFichiers ) self.pushButtonToutAjouter.setIcon( QtGui.QIcon( "ico/gtk-add.svg" ) ) self.pushButtonToutAjouter.setToolTip( u"Ajouter tous les fichiers à la liste des téléchargements" ) self.gridLayoutFichiers.addWidget( self.pushButtonToutAjouter, 0, 0, 2, 1 ) # Bouton pour rafraichir le plugin courant self.pushButtonRafraichirPlugin = MyQPushButton( self.tabFichiers ) self.pushButtonRafraichirPlugin.setIcon( QtGui.QIcon( "ico/gtk-refresh.svg" ) ) self.pushButtonRafraichirPlugin.setToolTip( "Rafraichir le plugin" ) self.gridLayoutFichiers.addWidget( self.pushButtonRafraichirPlugin, 2, 0, 2, 1 ) # Bouton pour ouvrir la fenetre des preferences du plugin courant self.pushButtonPreferencesPlugin = MyQPushButton( self.tabFichiers ) self.pushButtonPreferencesPlugin.setIcon( QtGui.QIcon( "ico/gtk-preferences.svg" ) ) self.pushButtonPreferencesPlugin.setToolTip( u"Ouvrir les préférences du plugin" ) self.gridLayoutFichiers.addWidget( self.pushButtonPreferencesPlugin, 4, 0, 2, 1 ) # On met en place ce layout sur un widget (pour le splitter) self.widgetFichiers = QtGui.QWidget() self.widgetFichiers.setLayout( self.gridLayoutFichiers ) # # Descriptif des fichiers # # Layout de grille self.gridLayoutDescriptif = QtGui.QGridLayout() # Label pour afficher un logo self.logoFichierDefaut = QtGui.QPixmap() self.logoFichierDefaut.load( "img/gtk-dialog-question.svg" ) self.labelLogo = QtGui.QLabel( self.centralWidget ) self.labelLogo.setPixmap( self.logoFichierDefaut.scaled( QtCore.QSize( 150, 150 ), QtCore.Qt.KeepAspectRatio ) ) self.gridLayoutDescriptif.addWidget( self.labelLogo, 0, 0, 1, 1 ) # Zone de texte pour afficher un descriptif self.plainTextEdit = QtGui.QPlainTextEdit( self.centralWidget ) self.gridLayoutDescriptif.addWidget( self.plainTextEdit, 0, 1, 1, 2 ) # On met en place ce layout sur un widget (pour le splitter) self.widgetDescriptif = QtGui.QWidget() self.widgetDescriptif.setLayout( self.gridLayoutDescriptif ) # Onrientation verticale du splitter self.tabFichiers.setOrientation( QtCore.Qt.Vertical ) # On ajoute les 2 elements au splitter (qui est notre onglet) self.tabFichiers.addWidget( self.widgetFichiers ) self.tabFichiers.addWidget( self.widgetDescriptif ) # # Liste des telechargements # # Layout de grille qui contient le tableau qui liste les fichiers a telecharger + les boutons pour le controller self.gridLayoutTelechargement = QtGui.QGridLayout( self.tabTelechargements ) # Tableau qui contient la liste des fichiers a telecharger self.tableWidgetTelechargement = MyQTableWidget( self.tabTelechargements ) # Il a 5 colonnes et 0 ligne (pour l'instant) self.tableWidgetTelechargement.setColumnCount( 3 ) self.tableWidgetTelechargement.setRowCount( 0 ) # On ajoute le titre des 5 colonnes self.tableWidgetTelechargement.setHorizontalHeaderItem( 0, self.tableWidgetTelechargement.creerItem( "Date" ) ) self.tableWidgetTelechargement.setHorizontalHeaderItem( 1, self.tableWidgetTelechargement.creerItem( "Emission" ) ) self.tableWidgetTelechargement.setHorizontalHeaderItem( 2, self.tableWidgetTelechargement.creerItem( "Etat" ) ) # On l'ajoute au layout self.gridLayoutTelechargement.addWidget( self.tableWidgetTelechargement, 0, 1, 4, 1 ) # Bouton pour monter l'element selectionne tout en haut de la liste self.pushButtonExtremiteMonter = MyQPushButton( self.tabTelechargements ) self.pushButtonExtremiteMonter.setIcon( QtGui.QIcon( "ico/gtk-jump-to-rtl.svg" ) ) self.pushButtonExtremiteMonter.setToolTip( u"Placer l'élément sélectionné tout en haut" ) self.gridLayoutTelechargement.addWidget( self.pushButtonExtremiteMonter, 0, 0, 1, 1 ) # Bouton pour monter l'element selectionne d'un cran dans la liste self.pushButtonMonter = MyQPushButton( self.tabTelechargements ) self.pushButtonMonter.setIcon( QtGui.QIcon( "ico/gtk-go-up.svg" ) ) self.pushButtonMonter.setToolTip( u"Monter l'élément sélectionné" ) self.gridLayoutTelechargement.addWidget( self.pushButtonMonter, 1, 0, 1, 1 ) # Bouton pour descendre l'element selectionne d'un cran dans la liste self.pushButtonDescendre = MyQPushButton( self.tabTelechargements ) self.pushButtonDescendre.setIcon( QtGui.QIcon( "ico/gtk-go-down.svg" ) ) self.pushButtonDescendre.setToolTip( u"Descendre l'élément selectionné" ) self.gridLayoutTelechargement.addWidget( self.pushButtonDescendre, 2, 0, 1, 1 ) # Bouton pour descendre l'element selectionne tout en bas de la liste self.pushButtonExtremiteDescendre = MyQPushButton( self.tabTelechargements ) self.pushButtonExtremiteDescendre.setIcon( QtGui.QIcon( "ico/gtk-jump-to-ltr.svg" ) ) self.pushButtonExtremiteDescendre.setToolTip( u"Placer l'élément sélectionné tout en bas" ) self.gridLayoutTelechargement.addWidget( self.pushButtonExtremiteDescendre, 3, 0, 1, 1 ) # Bouton pour supprimer tous les elements de la liste self.pushButtonToutSupprimer = MyQPushButton( self.tabTelechargements ) self.pushButtonToutSupprimer.setIcon( QtGui.QIcon( "ico/gtk-cancel.svg" ) ) self.pushButtonToutSupprimer.setToolTip( u"Supprimer tous les téléchargements" ) self.gridLayoutTelechargement.addWidget( self.pushButtonToutSupprimer, 0, 2, 1, 1 ) # Bouton pour supprimer de la liste les telechargements termines self.pushButtonNettoyer = MyQPushButton( self.tabTelechargements ) self.pushButtonNettoyer.setIcon( QtGui.QIcon( "ico/gtk-delete-full.svg" ) ) self.pushButtonNettoyer.setToolTip( u"Supprimer les téléchargement terminés" ) self.gridLayoutTelechargement.addWidget( self.pushButtonNettoyer, 1, 2, 1, 1 ) # Bouton pour ouvrir le dossier des telechargements self.pushButtonOuvrirDossierTelechargement = MyQPushButton( self.tabTelechargements ) self.pushButtonOuvrirDossierTelechargement.setIcon( QtGui.QIcon( "ico/gtk-folder.svg" ) ) self.pushButtonOuvrirDossierTelechargement.setToolTip( u"Ouvrir le dossier des téléchargements" ) self.gridLayoutTelechargement.addWidget( self.pushButtonOuvrirDossierTelechargement, 2, 2, 1, 1 ) # # Barre progression de telechargement d'un fichier # self.progressBarTelechargementFichier = QtGui.QProgressBar( self.centralWidget ) self.progressBarTelechargementFichier.setProperty( "value", 0 ) # # Barre de progression de telechargement des fichiers # self.progressBarTelechargement = QtGui.QProgressBar( self.centralWidget ) self.progressBarTelechargement.setProperty( "value", 0 ) # # Boutons du bas pour gerer ajouter/supprimer/lancer telechargements # # Layout horizontal qui contiendra les boutons self.horizontalLayoutBarreBas = QtGui.QHBoxLayout() # Bouton pour lancer les telechargements self.pushButtonLancer = QtGui.QPushButton( QtGui.QIcon( "ico/gtk-media-play-ltr.svg" ), u"Lancer téléchargement", self.centralWidget ) self.horizontalLayoutBarreBas.addWidget( self.pushButtonLancer ) # Bouton pour stopper les telechargements self.pushButtonStop = QtGui.QPushButton( QtGui.QIcon( "ico/gtk-media-stop.svg" ), u"Stopper le téléchargement", self.centralWidget ) self.pushButtonStop.setEnabled( False ) self.horizontalLayoutBarreBas.addWidget( self.pushButtonStop ) ### # Positionnement des differents widgets/layouts sur le layout de grille ### # Layout de grille dans lequel on va placer nos widgets/layouts self.gridLayout = QtGui.QGridLayout( self.centralWidget ) # On ajoute la barre du haut self.gridLayout.addLayout( self.horizontalLayoutBarreHaut, 0, 0, 1, 3 ) # On ajoute le gestionnaire d'onglets self.gridLayout.addWidget( self.tabWidget, 1, 0, 1, 3 ) # On ajoute la barre de progression de telechargement d'un fichier self.gridLayout.addWidget( self.progressBarTelechargementFichier, 2, 0, 1, 3 ) # On ajoute la barre de progression de telechargement des fichiers self.gridLayout.addWidget( self.progressBarTelechargement, 3, 0, 1, 3 ) # On ajoute les boutons ajouter/supprimer/lancer self.gridLayout.addLayout( self.horizontalLayoutBarreBas, 4, 0, 1, 3 ) ### # Mise en place le central widget dans la fenetre ### self.setCentralWidget( self.centralWidget ) ### # Mise en place du menu ### # Menu barre self.menubar = QtGui.QMenuBar( self ) self.menubar.setGeometry( QtCore.QRect( 0, 0, 480, 25 ) ) # Menu Fichier self.menuFichier = QtGui.QMenu( "&Fichier", self.menubar ) self.menubar.addAction( self.menuFichier.menuAction() ) # Action Fichier -> Quitter self.actionQuitter = QtGui.QAction( QtGui.QIcon( "ico/gtk-quit.svg" ), "&Quitter", self ) self.actionQuitter.setIconVisibleInMenu( True ) self.menuFichier.addAction( self.actionQuitter ) # Menu Edition self.menuEdition = QtGui.QMenu( "&Edition", self.menubar ) self.menubar.addAction( self.menuEdition.menuAction() ) # Action Edition -> Mise a jour self.actionMAJ = QtGui.QAction( QtGui.QIcon( "ico/gtk-refresh.svg" ), u"&Mise à jour des plugins", self ) self.actionMAJ.setIconVisibleInMenu( True ) self.menuEdition.addAction( self.actionMAJ ) # Action Edition -> Preferences self.actionPreferences = QtGui.QAction( QtGui.QIcon( "ico/gtk-preferences.svg" ), u"&Préférences", self ) self.actionPreferences.setIconVisibleInMenu( True ) self.menuEdition.addAction( self.actionPreferences ) # Menu Aide self.menuAide = QtGui.QMenu( "&Aide", self.menubar ) self.menubar.addAction( self.menuAide.menuAction() ) # Action Aide -> A propos self.actionAPropos = QtGui.QAction( QtGui.QIcon( "ico/gtk-about.svg" ), u"À p&ropos", self ) self.actionAPropos.setIconVisibleInMenu( True ) self.menuAide.addAction( self.actionAPropos ) # Ajout du menu a l'interface self.setMenuBar( self.menubar ) ### # Signaux provenants de l'interface ### QtCore.QObject.connect( self.tableWidgetFichier, QtCore.SIGNAL( "cellClicked(int,int)" ), self.afficherInformationsFichier ) QtCore.QObject.connect( self.tableWidgetFichier, QtCore.SIGNAL( "cellDoubleClicked(int,int)" ), self.gererTelechargement ) QtCore.QObject.connect( self.pushButtonToutAjouter, QtCore.SIGNAL( "clicked()" ), self.ajouterTousLesFichiers ) QtCore.QObject.connect( self.pushButtonRafraichirPlugin, QtCore.SIGNAL( "clicked()" ), self.rafraichirPlugin ) QtCore.QObject.connect( self.tableWidgetTelechargement, QtCore.SIGNAL( "cellDoubleClicked(int,int)" ), self.supprimerTelechargement ) QtCore.QObject.connect( self.pushButtonExtremiteMonter, QtCore.SIGNAL( "clicked()" ), lambda versLeHaut = True, extremite = True : self.tableWidgetTelechargement.deplacerLigne( versLeHaut, extremite ) ) QtCore.QObject.connect( self.pushButtonMonter, QtCore.SIGNAL( "clicked()" ), lambda versLeHaut = True, extremite = False : self.tableWidgetTelechargement.deplacerLigne( versLeHaut, extremite ) ) QtCore.QObject.connect( self.pushButtonDescendre, QtCore.SIGNAL( "clicked()" ), lambda versLeHaut = False, extremite = False : self.tableWidgetTelechargement.deplacerLigne( versLeHaut, extremite ) ) QtCore.QObject.connect( self.pushButtonExtremiteDescendre, QtCore.SIGNAL( "clicked()" ), lambda versLeHaut = False, extremite = True : self.tableWidgetTelechargement.deplacerLigne( versLeHaut, extremite ) ) QtCore.QObject.connect( self.pushButtonToutSupprimer, QtCore.SIGNAL( "clicked()" ), self.supprimerTousLesTelechargements ) QtCore.QObject.connect( self.pushButtonNettoyer, QtCore.SIGNAL( "clicked()" ), self.nettoyer ) QtCore.QObject.connect( self.pushButtonLancer, QtCore.SIGNAL( "clicked()" ), self.lancerTelechargement ) QtCore.QObject.connect( self.pushButtonStop, QtCore.SIGNAL( "clicked()" ), self.stopperTelechargement ) QtCore.QObject.connect( self.actionQuitter, QtCore.SIGNAL( "triggered()" ), self.close ) ################################################ # Instanciations + initialisation de variables # ################################################ # Fenetre About self.aProposDialog = None # Fenetre des preferences du logiciel self.preferencesDialog = None # Fenetre de mise a jour des plugins self.updateManagerDialog = None # Nom plugin courant self.nomPluginCourant = "" # Liste des fichiers self.listeFichiers = [] # Liste des fichiers a telecharger self.listeFichiersATelecharger = [] # Cache des images descriptive # Clef : urlImage Valeur : image (binaire) self.cacheImage = {} # On intancie le lanceur de signaux self.signaux = Signaux() # On instancie le gestionnaire de preferences self.preferences = Preferences() # On instancie le gestionnaire de preferences des plugins self.preferencesPluginDialog = PreferencePluginDialog( self ) # On instancie le gestionnaire de download self.downloader = Downloader( self.signaux ) # On recupere l'instance de API self.api = API.getInstance() # On instancie le gestionnaire d'historique self.historique = Historique() # On instancie la fenetre d'attente self.fenetreAttenteProgressDialog = FenetreAttenteProgressDialog( self ) # On instancie le gest # # Fenetre de confirmation pour quitter le logiciel # self.quitterMessageBox = QtGui.QMessageBox( self ) self.quitterMessageBox.setWindowTitle( "Fermeture de TVDownloader" ) self.quitterMessageBox.setText( u"Voulez-vous réellement quitter TVDownloader ?" ) self.quitterMessageBox.setInformativeText( u"Votre liste de téléchargement sera perdue" ) self.quitterMessageBox.addButton( "Oui", QtGui.QMessageBox.AcceptRole ) self.quitterMessageBox.addButton( "Non", QtGui.QMessageBox.RejectRole ) ############################################################ # On connecte les signaux des instances precedements crees # ############################################################ QtCore.QObject.connect( self.pushButtonOuvrirDossierTelechargement, QtCore.SIGNAL( "clicked()" ), self.ouvrirRepertoireTelechargement ) QtCore.QObject.connect( self.comboBoxSite, QtCore.SIGNAL( "activated(QString)" ), self.listerChaines ) QtCore.QObject.connect( self.comboBoxChaine, QtCore.SIGNAL( "activated(QString)" ), self.listerEmissions ) QtCore.QObject.connect( self.comboBoxEmission, QtCore.SIGNAL( "activated(QString)" ), self.listerFichiers ) QtCore.QObject.connect( self.pushButtonPreferencesPlugin, QtCore.SIGNAL( "clicked()" ), self.ouvrirPreferencesPlugin ) QtCore.QObject.connect( self.actionPreferences, QtCore.SIGNAL( "triggered()" ), self.ouvrirPreferencesLogiciel ) QtCore.QObject.connect( self.actionMAJ, QtCore.SIGNAL( "triggered()" ), self.ouvrirFenetreMiseAJour ) QtCore.QObject.connect( self.actionAPropos, QtCore.SIGNAL( "triggered()" ), self.ouvrirFenetreAPropos ) QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "debutActualisation(PyQt_PyObject)" ) , self.fenetreAttenteProgressDialog.ouvrirFenetreAttente ) QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "finActualisation()" ) , self.fenetreAttenteProgressDialog.fermerFenetreAttente ) QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "actualiserListesDeroulantes()" ) , self.actualiserListesDeroulantes ) QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "listeChaines(PyQt_PyObject)" ) , self.ajouterChaines ) QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "listeEmissions(PyQt_PyObject)" ) , self.ajouterEmissions ) QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "listeFichiers(PyQt_PyObject)" ) , self.ajouterFichiers ) QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "nouvelleImage(PyQt_PyObject)" ) , self.mettreEnPlaceImage ) QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "debutTelechargement(int)" ) , self.debutTelechargement ) QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "finTelechargement(int)" ) , self.finTelechargement ) QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "finDesTelechargements()" ) , self.activerDesactiverInterface ) QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "pourcentageFichier(int)" ) , self.progressBarTelechargementFichier.setValue ) ######### # Début # ######### # La fenetre prend la dimension qu'elle avait a sa fermeture taille = self.preferences.getPreference( "tailleFenetre" ) self.resize( taille[ 0 ], taille[ 1 ] ) # Si aucun plugin n'est active, on ouvre la fenetre des preferences if( len( self.preferences.getPreference( "pluginsActifs" ) ) == 0 ): self.ouvrirPreferencesLogiciel() # On actualise tous les plugins self.rafraichirTousLesPlugins()
def main(filenameList): lock = _threading.Lock() # 获取线程锁,预防多个listener竞争lftp的使用 ftp = FTPFactory().GetFTP() downloader = Downloader(ftp, lock, *filenameList) downloader.Run()
YOUTUBE_VID_DIR = 'Youtube Videos' # Parent directory for all Yt videos ! current_user = os.getlogin() if system()=='Windows': DOWNLOADS = f'C:/Users/{current_user}/Downloads' # get the currently logged in user's download dir ! elif system()=='Linux': DOWNLOADS = f'/home/{current_user}/Downloads' else: raise Exception("Not Implemented For Current Platform !") FINAL_PATH = os.path.join(DOWNLOADS, YOUTUBE_VID_DIR) # joining the downloads and the new vids directory ! if not os.path.isdir(FINAL_PATH): os.mkdir(FINAL_PATH) print(f'New Download Directory Created At {FINAL_PATH}') download = Downloader() while True: option = int(input('1)Video\n2)Playlist\n3)Exit\n>>')) if option == 1: link = input('Please Paste A Video Link : ') download.download_single_video(FINAL_PATH, link) elif option == 2: playlist_link = input('Please Paste A Playlist Link :') download.download_playlist(FINAL_PATH, playlist_link=playlist_link) elif option == 3: print('Exiting........!') break
from sentinelsat import read_geojson, geojson_to_wkt from Downloader import Downloader import StringProcessor as process import zipfile import os username = '******' password = '******' path = '/Users/DavidLei/PycharmProjects/untitled/testfile.geojson' link: str = 'https://scihub.copernicus.eu/apihub/' from_satelite: str = 'Sentinel-2' package_info = list() keys = [] downloader = Downloader(username, password, link) wkt = geojson_to_wkt(read_geojson(path)) downloader.search_polygon(wkt, '20190501', '20190503', str_platform_name=from_satelite, percentage=(0, 100)) f = open('FileList.txt', 'w+') for key, value in downloader.products.items(): if "S2B_MSIL1C" in str(value): # print('downloading') # print(key) # print(value) package_info.append(str(value)) keys.append(str(key)) print(value['title']) downloader.download_one(
def download(self): self.downloader = Downloader(self, self.url, self.path) self.thread = threading.Thread(target = self.downloader.download) self.thread.start()
def __init__(self, uuid): # Movement_has_Category.delete().execute() # Movement.delete().execute() downloader = Downloader(uuid) account_statements = downloader.download_account_data() for statement in account_statements: try: movement = Movement.get(hash=statement['id']) if movement.id: continue except: date = datetime.strptime(statement['postDate'], '%Y-%m-%d') if statement['__typename'] == 'BarcodePaymentEvent': type = 'OUTCOME' original_name = statement['title'] + " " + statement[ 'detail'] elif statement['__typename'] == 'TransferOutEvent': type = 'OUTCOME' original_name = statement['title'] + " " + statement[ 'destinationAccount']['name'] elif statement['__typename'] == 'TransferInEvent': original_name = statement['title'] if statement['originAccount']: original_name = original_name + " " + statement[ 'originAccount']['name'] type = 'INCOME' if statement['__typename'] == 'TransferOutReversalEvent': value = float(statement['detail'].split( 'R$ ', 1)[1].replace('.', '').replace(',', '.')) type = 'REVERSAL' else: value = statement['amount'] Movement.create(original_name=original_name, date=date, value=value, status='SHOW', type=type, hash=statement['id']) card_statements = downloader.download_credit_card_data() for statement in card_statements: try: movement = Movement.get(hash=statement['id']) if movement.id: continue except: rest = statement['time'].split( 'T', 1)[0] # todo Haw ficou de corrigir para a data iso date = datetime.strptime(rest, '%Y-%m-%d') Movement.create(original_name=statement['description'], date=date, value=statement['amount'] / 100, status='SHOW', type='OUTCOME', hash=statement['id'])