Exemplo n.º 1
0
    def __crawljobs(self, keyword, city=u'全国'):

        page_num = 1

        url = u'http://www.lagou.com/jobs/positionAjax.json?px=default&city=%s&needAddtionalResult=false' % (city,)

        page_size = sys.maxint

        errors = []

        while page_num < page_size:
            data = u'first=false&pn=%d&kd=%s' % (page_num, keyword)
            cache, content = Downloader.get_html(url, data, city)
            if content:
                try:
                    result = json.loads(content)
                    page_no = result['content']['pageNo']
                    if page_no > 0:
                        print '{kv}: {page_num}'.format(kv=keyword, page_num=page_num)
                        page_num += 1
                        if not cache:
                            # 每个网络请求间隔一秒
                            sleep(0.1)
                    else:
                        # 为空,说明最后一页
                        break
                except Exception, e:
                    page_num += 1
                    # 发生异常, 删除缓存文件
                    Downloader.remove_file(url, data, city)
                    errors.append((1, url, data, city, e))
            else:
                # 请求结果为空
                errors.append((2, url, data, city, e))
Exemplo n.º 2
0
def applyGloablBlackList(cfg_params):
    
    removeBList = cfg_params.get("GRID.remove_default_blacklist", 0 )
    blackAnaOps = None
    if int(removeBList) == 0:
        try:
            blacklist = Downloader("http://cmsdoc.cern.ch/cms/LCG/crab/config/")
            result = blacklist.config("site_black_list.conf")
        except:            
            msg = "ERROR talking to cmsdoc\n%s"%str(sys.exc_info()[1])
            common.logger.info(msg)
            common.logger.info("WARNING: Could not download default site black list")
            result = []
        if result :
            blackAnaOps = result.strip()
            common.logger.debug("Enforced black list: %s "%blackAnaOps)
        else:
            common.logger.info("WARNING: Skipping default black list!")
    if blackAnaOps:
        blackUser = cfg_params.get("GRID.se_black_list", '' )
        if blackUser:
            cfg_params['GRID.se_black_list'] = blackUser + ',' + blackAnaOps 
        else:
            cfg_params['GRID.se_black_list'] = blackAnaOps
        msg = "PSN black list: %s" % cfg_params['GRID.se_black_list']
        common.logger.info(msg)
Exemplo n.º 3
0
    def configure(self, cfg_params):
        self.cfg_params = cfg_params
        self.jobtypeName   = cfg_params.get('CRAB.jobtype','')
        self.schedulerName = cfg_params.get('CRAB.scheduler','')
        Scheduler.configure(self,cfg_params)
        #self.proxyValid=0

        #self.dontCheckProxy=int(cfg_params.get("GRID.dont_check_proxy",0)) 	 
        self.space_token = cfg_params.get("USER.space_token",None) 	 
        self.proxyServer= 'myproxy.cern.ch'
        #self.group = cfg_params.get("GRID.group", None) 	 
        #self.role = cfg_params.get("GRID.role", None)

        removeBList = cfg_params.get("GRID.remove_default_blacklist", 0 )
        blackAnaOps = None
        if int(removeBList) == 0:
            blacklist = Downloader("http://cmsdoc.cern.ch/cms/LCG/crab/config/")
            result = blacklist.config("site_black_list.conf")
            if result != None:
                blackAnaOps = result
            common.logger.debug("Enforced black list: %s "%blackAnaOps)
        else:
            common.logger.info("WARNING: Skipping default black list!")

        self.EDG_ce_black_list = None
        if cfg_params.has_key('GRID.ce_black_list') and cfg_params['GRID.ce_black_list']:
            self.EDG_ce_black_list = cfg_params.get('GRID.ce_black_list')
            if int(removeBList) == 0 and blackAnaOps: 
                self.EDG_ce_black_list += ",%s"%blackAnaOps
        elif int(removeBList) == 0 and blackAnaOps:
            self.EDG_ce_black_list = blackAnaOps
        if self.EDG_ce_black_list:
            self.EDG_ce_black_list = str(self.EDG_ce_black_list).split(',')

        self.EDG_ce_white_list = cfg_params.get('GRID.ce_white_list',None)
        if (self.EDG_ce_white_list): self.EDG_ce_white_list = str(self.EDG_ce_white_list).split(',')

        self.VO = cfg_params.get('GRID.virtual_organization','cms')

        self.EDG_clock_time = cfg_params.get('GRID.max_wall_clock_time',None)

        # Default minimum CPU time to >= 130 minutes
        self.EDG_cpu_time = cfg_params.get('GRID.max_cpu_time', '130')

        ## Add EDG_WL_LOCATION to the python path
        #if not self.CRAB_useServer and not self.CRAB_serverName:
        #    if not os.environ.has_key('EDG_WL_LOCATION'):
        #        msg = "Error: the EDG_WL_LOCATION variable is not set."
        #        raise CrabException(msg)
        #    path = os.environ['EDG_WL_LOCATION']
        #    libPath=os.path.join(path, "lib")
        #    sys.path.append(libPath)
        #    libPath=os.path.join(path, "lib", "python")
        #    sys.path.append(libPath)

        self.checkProxy()
        return
Exemplo n.º 4
0
def runProgram(youtube_search_query, text_to_find):
    dl = Downloader()
    celeb_videos = Search.search(youtube_search_query)
    links = [Caption.YOUTUBE_LINK + x for x in celeb_videos]
    print links
    count = 0
    for videoid in celeb_videos:
        cap = Caption(videoid)
        results = cap.find(text_to_find)
        for result in results:
            dl.download(result, text_to_find + str(count))
            count += 1
Exemplo n.º 5
0
    def manageCredential(self):
        """
        Prepare configuration and Call credential API
        """
        common.logger.info("Registering credential to the server : %s"%self.server_name)

        try:
            myproxyserver = Downloader("http://cmsdoc.cern.ch/cms/LCG/crab/config/").config("myproxy_server.conf")
            myproxyserver = myproxyserver.strip()
            if myproxyserver is None:
                raise CrabException("myproxy_server.conf retrieved but empty")
        except Exception, e:
            common.logger.info("Problem setting myproxy server endpoint: using myproxy.cern.ch")
            common.logger.debug(e)
            myproxyserver = 'myproxy.cern.ch'
Exemplo n.º 6
0
    def insert_jobs(self):
        errors = []

        types = self.get_types()
        for keyword in types:
            page_num = 1
            city = u'全国'
            url = u'http://www.lagou.com/jobs/positionAjax.json?px=default&city=%s&needAddtionalResult=false' % (city,)
            page_size = sys.maxint

            while page_num < page_size:
                data = u'first=false&pn=%d&kd=%s' % (page_num, keyword)
                cache, content = Downloader.get_html(url, data, city)
                if content:
                    try:
                        result = json.loads(content)
                        pos_result = result['content']['positionResult']
                        jobs = pos_result['result']
                        if len(jobs) > 0:
                            self.db.insert_jobs(jobs)
                            print data
                        else:
                            # 为空说明是最后一页
                            errors.append('result empty {0} {1}'.format(url, data, content))
                            break

                        page_num += 1
                    except Exception, e:
                        page_num += 1
                        # 发生异常
                        errors.append((1, url, data, city, e))
                else:
                    # 请求结果为空
                    errors.append((2, url, data, city, e))
Exemplo n.º 7
0
 def craw(self, root_url):
     # todo: 层数设置?
     page_count = self.page_count
     UrlManager.add_new_url(root_url)
     try:
         while UrlManager.has_new_url():
             todo_url = UrlManager.get_new_url()
             try:
                 print u"\nNO: %d 正在检测链接:%s" % (page_count, todo_url)
             except:
                 print u"出错", todo_url
                 # todo: 换成 contiune 是不是更好?
             # page_status, bad_links = Downloader.test(todo_url)
             # print u"\n结果:%s" % page_status
             # todo: 加入状态码的说明
             state, content = Downloader.download(todo_url)
             if state:
                 prase_state, new_urls = Parser.parse(todo_url, content)
                 if prase_state:
                     UrlManager.add_new_urls(new_urls)
                 else:
                     Outputer.collect_data(new_urls)
             else:
                 Outputer.collect_data(content)
             page_count += 1
             # except:
             #     print u"页面爬取失败"
             #     UrlManager.add_wrong_url(todo_url)
             # todo:测试代码
             if page_count == 5000:
                 self.page_count = page_count
                 break
             print UrlManager.num_new_url()
     finally:
         Outputer.output_txt(self.page_count)
Exemplo n.º 8
0
    def infomation_crew(self):
        # 一条一条取
        total_count = 20000
        i = 0
        while i < total_count:
            i += 1
            # 去除诗词的详情页url
            info = self.db.select_unanalyzed_infomation()

            if info is not None:
                # 下载分析
                url = info['url']
                html = Downloader.get_html(url, 'infomation')
                if html:
                    content = Analyzer.get_info_detail(html)
                    if content:
                        self.db.update_infomationurl(url, content)
                        print '%d/%d %s %s' % (i, total_count, info['title'], url)
                    else:
                        self.db.insert_error('analyze_info_detail_error', 7, 'reason', url)
                else:
                    self.db.insert_error('download_info_detail_error', 6, 'reason', url)
            else:
                # 没有了
                return
Exemplo n.º 9
0
    def _createXMLcommand(
        self,
        taskUName,
        cmdSpec="status",
        rng="all",
        newTaskAddIns=False,
        flavour="analysis",
        type="fullySpecified",
        jobs="-1",
    ):

        xmlString = ""
        cfile = minidom.Document()

        ver = common.prog_version_str
        node = cfile.createElement("TaskCommand")
        node.setAttribute("Task", str(taskUName))
        node.setAttribute("Subject", str(self.userSubj))
        node.setAttribute("Command", str(cmdSpec))
        node.setAttribute("Range", str(rng))
        node.setAttribute("TotJob", str(jobs))
        node.setAttribute("Scheduler", str(self.cfg_params["CRAB.scheduler"]))
        node.setAttribute("Flavour", str(flavour))
        node.setAttribute("Type", str(type))
        node.setAttribute("ClientVersion", str(ver))

        ## Only Temporary. it should be at Server level
        removeBList = self.cfg_params.get("GRID.remove_default_blacklist", 0)
        blackAnaOps = "None.Nowhere"
        if int(removeBList) == 0:
            blacklist = Downloader("http://cmsdoc.cern.ch/cms/LCG/crab/config/")
            result = blacklist.config("site_black_list.conf")
            if result != None:
                blackAnaOps = result
            common.logger.debug("Enforced black list: %s " % str(blacklist))
        else:
            common.logger.info("WARNING: Skipping default black list!")

        try:
            myproxyserver = Downloader("http://cmsdoc.cern.ch/cms/LCG/crab/config/").config("myproxy_server.conf")
            myproxyserver = myproxyserver.strip()
            if myproxyserver is None:
                raise CrabException("myproxy_server.conf retrieved but empty")
        except Exception, e:
            common.logger.info("Problem setting myproxy server endpoint: using myproxy.cern.ch")
            common.logger.debug(e)
            myproxyserver = "myproxy.cern.ch"
Exemplo n.º 10
0
    def __init__(self, serverName):
        import string
        self.serverName = string.lower(serverName)
        common.logger.debug('Calling ServerConfig '+self.serverName)

        url ='http://cmsdoc.cern.ch/cms/LCG/crab/config/'

        self.downloader = Downloader(url)
Exemplo n.º 11
0
    def rb_configure(self, RB):
        url ='http://cmsdoc.cern.ch/cms/LCG/crab/config/'
        from Downloader import Downloader
        import httplib
        common.logger.debug('Downloading config files for WMS: '+url)
        ## 25-Jun-2009 SL: patch to use Cream enabled WMS
        if ( self.cfg_params.get('GRID.use_cream',None) ):
            RB='CREAM'
        if not RB: return None
        rb_param_file = None
        configFileName = 'glite_wms_'+str(RB)+'.conf'

        results = Downloader(url)
        try:
            gliteConfig  = results.filePath(configFileName)
        except httplib.HTTPException, ex: 
            raise CrabException( "Problem getting RB config file: %s, reason:"%(configFileName, ex) )
Exemplo n.º 12
0
    def poem_list_crew(self):
        for i in range(1, 6):
            url = 'http://www.haoshiwen.org/type.php?x=%d' % i

            content = Downloader.get_html(url, 'poemlist')
            if content:
                page_count = Analyzer.get_page_count(content)
                # 分析
                for j in range(1, page_count + 1):
                    page_url = 'http://www.haoshiwen.org/type.php?x=%d&page=%d' % (i, j)
                    # 入库
                    self.db.insert_url(page_url, 1)

                    # 判断是否分析过
                    if self.db.url_analyzed(page_url):
                        pass
                    else:
                        content = Downloader.get_html(page_url, 'poemlist')
                        if content:
                            # 分析诗的列表
                            poems = Analyzer.get_poems_from_list_page(content)

                            if poems:
                                # 入库
                                self.db.insert_urls(poems, 2)
                                self.db.update_url(page_url)
                                print '%d %d/%d: %s' % (i, j, page_count, page_url)
                            else:
                                if Analyzer.check_poem_list_last_page(content):
                                    # 最后一页
                                    break
                                else:
                                    print u'分析失败'
                                    self.db.insert_error('analyze_poem_list_error', 3, 'reason', page_url)
                                    # 错误入库:analyze_poem_list_error

                        else:
                            print u'获取页面诗词列表错误'
                            self.db.insert_error('get_poem_list_error', 2, 'reason', page_url)
                            # 错误入库:get_poem_list_error
            else:
                print u'分析首页失败'
                self.db.insert_error('analyze_poem_list_first_page_error', 1, 'reason', page_url)
Exemplo n.º 13
0
    def parseAnswerServer(self, updateInfo):
        updateDict = parse_qs(updateInfo, strict_parsing=True)
        # updateDict has: mustUpdate, urlUser, newVersionUser, urlAdmin,
        #                 newVersionAdmin
        updates = dict()  # updates has: mustUpdate, userFile, adminFile

        mustUpdate = int(updateDict['mustUpdate'][0])
        updates['mustUpdate'] = mustUpdate

        downloader = Downloader()
        location = "../../persistent/downloads"

        if mustUpdate == 0:
            logger.info('No update available')
        elif (mustUpdate & UPDATE_ADMIN_MODE):
            adminURL = updateDict['urlAdmin'][0]
            logger.info('Downloading Admin update: %s' % adminURL)
            adminFile = downloader.downloadUpdate(location, adminURL)
            updates['adminFile'] = adminFile
            logger.info('Administrator update is available called: %s' %
                        adminFile)

            if DISPLAY_GUI_MESSAGES and not(checkFiles.checkIfAdmin()):
                root = Tk()
                root.title('HiSPARC')
                Message(root, anchor='s', text="Update is available requiring "
                        "administrator rights!\nPlease ask your administrator "
                        "to reboot and install it!").pack(padx=150, pady=100)
                root.mainloop()

        elif (mustUpdate & UPDATE_USER_MODE):
            userURL = updateDict['urlUser'][0]
            logger.info('Downloading User update: %s' % userURL)
            userFile = downloader.downloadUpdate(location, userURL)
            updates['userFile'] = userFile
            logger.info('User update is available called: %s' % userFile)
            # Run the update to install it.
            # First call a batch file so that Python can be closed.
            os.system(".\\runUserUpdate.bat %s" % userFile)

        return updates
Exemplo n.º 14
0
class Dispatcher(object):
    """ Dispatcher class to control process of working. """

    def __init__(self):
        create_or_check_path("logs/")

        fileConfig('config/logger.conf')

        self.database = Database()
        self.parser = Parser()
        self.event_processor = EventProcessor()
        self.downloader = Downloader()

        self.new_events_file = None
        self.new_data = None
        self.new_event = None

    def processor(self):
        """ Process of downloading, parsing and saving information. """
        download_status = True
        while download_status:
            getting_event_status = self.get_event()
            if getting_event_status:
                processing_event_status = self.process_event()
                if processing_event_status:
                    self.process_data()
            else:
                download_status = self.download()

    def get_event(self):
        """ Asks Parser for a new event.
        :return: False if getting event failed else True
        """
        self.new_event = self.parser.get_event(self.new_events_file)
        return False if self.new_event is None else True

    def download(self):
        """ Sends a request to Downloader.
        :return: False if downloading failed else True
        """
        self.new_events_file = self.downloader.download_archive()
        return False if self.new_events_file is None else True

    def process_event(self):
        """ Asks EventProcessor to get data from event.
        :return: False if processing event failed else True
        """
        self.new_data = self.event_processor.process_event(self.new_event)
        return False if self.new_data is None else True

    def process_data(self):
        """ Sends data to Database. """
        self.database.process_data(self.new_data)
Exemplo n.º 15
0
 def poem_type_crew(self):
     url = 'http://www.haoshiwen.org/type.php'
     content = Downloader.get_html(url, 'poemlist')
     if content:
         result = Analyzer.get_poem_types(content)
         k = 1
         for i in result:
             # 只处理类型
             if i[0] == u'类型':
                 for j in i[1]:
                     self.db.insert_poem_types(k, i[0], j[0], j[1])
                     print '\t' + j[0]
                 k += 1
Exemplo n.º 16
0
    def __init__(self):
        create_or_check_path("logs/")

        fileConfig('config/logger.conf')

        self.database = Database()
        self.parser = Parser()
        self.event_processor = EventProcessor()
        self.downloader = Downloader()

        self.new_events_file = None
        self.new_data = None
        self.new_event = None
Exemplo n.º 17
0
def main():


    log = setup_logging("sentinal.log")

    parser = argparse.ArgumentParser(description='Grabs sentinal images from the ESA stite',
                                     epilog='-h for help',
                                     formatter_class=argparse.RawTextHelpFormatter)

    parser.add_argument('-c', action="store_true", help='create db table only', dest='create_db') 
    parser.add_argument('-C', type=str, help='path to the configuration file', dest='configuration_file') 
    parser.add_argument('-f', action="store_true", help='force', dest='force')
    parser.add_argument('-v', action="store_true", help='run verbosely', dest='verbose')
    parser.add_argument('-o', action="store_true", help='overwrite data .zip file even if it exists', dest='overwrite')

    args = parser.parse_args()

    db = Database(args)

    if args.create_db: 
        db.createDb(args.verbose)
        sys.exit(0)

    config = ConfigManager(args.configuration_file,log)

    auth = config.getAuth()
    if not len(auth):
        log.error('Missing ESA SCIHUB authentication information')
        sys.exit(7)

    types = config.getTypes()
    polygons = config.getPolygons()
    if len(types) != len(polygons):
        log.error( 'Incorrect number of polygons and types in configuration file' )
        sys.exit(6)

    downloader = Downloader(args, config, db, log)
    downloader.getProducts()
    downloader.downloadProducts()
Exemplo n.º 18
0
    def configure(self, cfg_params):
        self.cfg_params = cfg_params
        self.jobtypeName   = cfg_params.get('CRAB.jobtype','')
        self.schedulerName = cfg_params.get('CRAB.scheduler','')
        Scheduler.configure(self,cfg_params)
        self.proxyValid=0

        self.dontCheckProxy=int(cfg_params.get("GRID.dont_check_proxy",0)) 	 
        self.space_token = cfg_params.get("USER.space_token",None) 	 
 	try:
            self.proxyServer = Downloader("http://cmsdoc.cern.ch/cms/LCG/crab/config/").config("myproxy_server.conf")
            self.proxyServer = self.proxyServer.strip()
            if self.proxyServer is None:
                raise CrabException("myproxy_server.conf retrieved but empty")
        except Exception, e:
            common.logger.info("Problem setting myproxy server endpoint: using myproxy.cern.ch")
            common.logger.debug(e)
            self.proxyServer= 'myproxy.cern.ch'
Exemplo n.º 19
0
    def poem_detail_crew(self):
        # update urls set analyzed = 1 where url = 'http://www.haoshiwen.org/view.php?id=9510'
        # 一条一条取
        total_count = 10
        c = 1
        while c < total_count:
            c += 1
            continue

            # 去除诗词的详情页url
            url = self.db.select_unanalyzed_url(2)

            if url is not None:
                url = url['url']

                # 判断是否抓取过
                if not self.db.poem_exists(url):
                    content = Downloader.get_html(url, 'poem')
                    if content:
                        try:
                            poem = Analyzer.get_poem_detail(content, url)
                            if poem:
                                # 分析成功,入库
                                poem_content = poem[0]
                                poem_info = poem[1]

                                self.db.insert_poem(poem_content)
                                self.db.insert_infomations(url, 1, poem_info)

                                self.db.update_url(url)

                                print "%d/%d %s %s" % (c, total_count, poem_content['title'], poem_content['url'])
                            else:
                                self.db.insert_error('analyze_poem_detail_error', 5, 'reason', url)
                        except Exception, e:
                            continue

                    else:
                        self.db.insert_error('get_poem_detail_error', 4, 'reason', url)
                else:
                    self.db.update_url(url)
            else:
                count = total_count
Exemplo n.º 20
0
    def poem_author_crew(self):
        # 一条一条取
        total_count = 20000
        i = 0
        while i < total_count:
            i += 1
            # 去除诗词的详情页url
            authors = self.db.select_authors()
            cou = len(authors)
            i = 1
            for author in authors:
                url = author[0]

                if url is not None and url is not '':
                    # 判断是否存在
                    if not self.db.author_exists(url):
                        html = Downloader.get_html(url, 'author')
                        if html:
                            try:
                                author_info = Analyzer.get_author_detail(html, url)

                                if author_info:
                                    author_content = author_info[0]
                                    author_infos = author_info[1]

                                    self.db.insert_author(author_content)
                                    self.db.insert_infomations(url, 2, author_infos)
                                    print '%d/%d %s %s' % (i, cou, author_content['name'], url)
                                    i += 1
                                else:
                                    self.db.insert_error('analyze_author_detail_error', 9, 'reason', url)
                            except Exception, e:
                                print 'error %s' % (url,)

                        else:
                            self.db.insert_error('download_author_detail_error', 8, 'reason', url)
            else:
                # 没有了
                return
Exemplo n.º 21
0
    def poem_type_poem_list_craw(self):
        types = self.db.get_poem_types()
        for i in types:
            j = 1

            while True:
                page_url = i['url'] + "&page=" + str(j)

                # 入库
                self.db.insert_url(page_url, 3)

                # 判断是否分析过
                if self.db.url_analyzed(page_url):
                    j += 1
                    continue
                else:
                    content = Downloader.get_html(page_url, 'poemlist')
                    if content:
                        # 分析诗的列表
                        poems = Analyzer.get_poems_from_list_page(content)

                        if poems:
                            # 入库
                            self.db.insert_type_poems(i['id'], poems)
                            self.db.update_url(page_url)
                            print '%d: %s' % (j, page_url)
                            j += 1
                        else:
                            if Analyzer.check_poem_list_last_page(content):
                                # 最后一页
                                break
                            else:
                                print u'分析失败'
                                self.db.insert_error('analyze_poem_list_error', 3, 'reason', page_url)
                                # 错误入库:analyze_poem_list_error

                    else:
                        print u'获取页面诗词列表错误'
                        self.db.insert_error('get_poem_list_error', 2, 'reason', page_url)
Exemplo n.º 22
0
def MainProcess(logger, uperList, saveRootPath, concurrency=3):
    pp = None
    try:
        # --------------------------------------------------------------
        # 进行每个 UP 主视频页数的获取
        pp = PreProcess(logger=logger, uperList=uperList)
        pp.ScanLoclInfo(saveRootPath)
        pp.Process()
        # --------------------------------------------------------------
        # 爬取要下载视频的 url
        for uper in pp.uperList:
            logger.info(uper.UserName + " Spider Start···")
            OneSpiderRetryTimes = 0
            # 打算下载的数量,要去网络动态获取的数量进行对比
            while ((uper.NeedDownloadFilmCount > len(
                    uper.VideoInfoDic_NetFileName)
                    or len(uper.ErrorUrl_Dic) > 0)
                   and OneSpiderRetryTimes <= 10):
                # dd = BiliSpider()
                # GithubDeveloperSpider
                BiliSpider.start(logger=logger,
                                 uper=uper,
                                 saveRootPath=saveRootPath,
                                 concurrency=concurrency,
                                 middleware=middleware)

                OneSpiderRetryTimes = OneSpiderRetryTimes + 1
                logger.info("Try Spider " + uper.UserName + " " +
                            str(OneSpiderRetryTimes) + " times.")
                RandomSleep()

            logger.info(uper.UserName + " Spider Done.")

            if OneSpiderRetryTimes > 10:
                logger.error(uper.UserName + " Spider Retry " +
                             str(OneSpiderRetryTimes) + "times.")
                logger.error("Error Url:")
                for eUrl in uper.ErrorUrl_Dic:
                    logger.error(eUrl)
            else:
                # 本地应该原有+准备要下载的 != 网络总数,需要提示
                if len(uper.VideoInfoDic_NetFileName) != len(
                        uper.VideoInfoDic_loaclFileName):
                    logger.warn("VideoInfoDic_NetFileName Count: " +
                                str(len(uper.VideoInfoDic_NetFileName)) +
                                " != VideoInfoDic_loaclFileName Count: " +
                                str(len(uper.VideoInfoDic_loaclFileName)))
            uper.ErrorUrl_Dic.clear()

        logger.info("Spider All Done.")
        # --------------------------------------------------------------
        logger.info("Start Download" + "----" * 20)
        # 开始下载
        # 先对 local 与 net 的字典进行同步
        logger.info("Start Sync Dic")
        for uper in pp.uperList:
            iNeedDl = 0

            for fileName, oneVideo in zip(
                    uper.VideoInfoDic_loaclFileName.keys(),
                    uper.VideoInfoDic_loaclFileName.values()):
                # 匹配到对应的项目才进行处理
                findList = list(
                    filter(lambda d: d.split('_')[1] == fileName,
                           uper.VideoInfoDic_NetFileName.keys()))
                if any(findList):
                    uper.VideoInfoDic_NetFileName[
                        findList[0]].isDownloaded = oneVideo.isDownloaded
                    if oneVideo.isDownloaded == False:
                        iNeedDl = iNeedDl + 1
            logger.info(uper.UserName + "NetFile / LocalFile -- NeedDl: " +
                        str(len(uper.VideoInfoDic_NetFileName)) + " / " +
                        str(len(uper.VideoInfoDic_loaclFileName)) + " -- " +
                        str(iNeedDl))
        logger.info("End Sync Dic")
        for uper in pp.uperList:
            directory = os.path.join(saveRootPath, uper.UserName)
            for fileName, oneVideo in zip(
                    uper.VideoInfoDic_NetFileName.keys(),
                    uper.VideoInfoDic_NetFileName.values()):
                if oneVideo.isDownloaded == True:
                    continue
                DownloadRetryTimes = 0
                oneRe = False
                while oneRe is False and DownloadRetryTimes <= 10:
                    oneRe = Downloader(logger, directory, oneVideo.time,
                                       oneVideo.title,
                                       oneVideo.url).ProcessOne()
                    DownloadRetryTimes = DownloadRetryTimes + 1
                    logger.info("Try Download " + str(DownloadRetryTimes) +
                                " times.")
                    RandomSleep()

                if OneSpiderRetryTimes > 10:
                    logger.error("Retry Download " + str(DownloadRetryTimes) +
                                 " times.")
                    logger.error("Error Url: " + oneVideo.url)
                # 标记下载完成
                if oneRe:
                    oneVideo.isDownloaded = True
                    uper.ThisTimeDownloadCount = uper.ThisTimeDownloadCount + 1

    except Exception as ex:
        errInfo = "Catch Exception: " + str(ex)
        logger.error(errInfo)
    finally:
        logger.info("finally" + "----" * 20)
        for uper in pp.uperList:
            logger.info("This Time Download: " + uper.UserName + " -- " +
                        str(uper.ThisTimeDownloadCount))
        for uper in pp.uperList:
            for fileName, oneVideo in zip(
                    uper.VideoInfoDic_NetFileName.keys(),
                    uper.VideoInfoDic_NetFileName.values()):
                if oneVideo.isDownloaded == False:
                    logger.error('Download Fail:' + uper.UserName)
                    logger.error(oneVideo.url)
        logger.info("This Time Done.")
Exemplo n.º 23
0
def download_callback(params):
    downloader = Downloader(params['url'])
    downloader.start()
Exemplo n.º 24
0
class MainWindow( QtGui.QMainWindow ):
	
	## Constructeur
	# Le constructeur va creer la fenetre principale en y ajoutant tous les widgets necessaires au programme
	def __init__( self ):
		# Appel au constructeur de la classe mere
		QtGui.QMainWindow.__init__( self )
		
		###########
		# Fenetre #
		###########
		
		###
		# Reglages de la fenetre principale
		###
		
		# Nom de la fenetre
		self.setWindowTitle( "TVDownloader" )
		# Mise en place de son icone
		self.setWindowIcon( QtGui.QIcon( "ico/TVDownloader.png" ) )
		
		###
		# Mise en place des widgets dans la fenetre
		###
		
		# Widget central qui contiendra tout
		self.centralWidget = QtGui.QWidget( self )
		
		#
		# Barre du haut
		#
		
		# Layout horizontal qui contiendra les listes deroulantes
		self.horizontalLayoutBarreHaut = QtGui.QHBoxLayout()
		
		# Liste deroulante pour choisir le site (plugin)
		self.comboBoxSite = QtGui.QComboBox( self.centralWidget )
		self.horizontalLayoutBarreHaut.addWidget( self.comboBoxSite )
		
		# Liste deroulante pour choisir une chaine du site courant
		self.comboBoxChaine = QtGui.QComboBox( self.centralWidget)
		self.horizontalLayoutBarreHaut.addWidget( self.comboBoxChaine )
		
		# Liste deroulante pour choisir une emission de la chaine courante
		self.comboBoxEmission = QtGui.QComboBox( self.centralWidget )
		self.horizontalLayoutBarreHaut.addWidget( self.comboBoxEmission )
		
		#
		# Onglets
		#
		
		# Gestionnaire onglets
		self.tabWidget = QtGui.QTabWidget( self.centralWidget )
		
		# Onglet Fichiers
		self.tabFichiers = QtGui.QSplitter( self.centralWidget ) # L'onglet Fichier contient un splitter
		self.tabWidget.addTab( self.tabFichiers, u"Choix des fichiers" )
		
		# Onglet Telechargements
		self.tabTelechargements = QtGui.QWidget( self.centralWidget )
		self.tabWidget.addTab( self.tabTelechargements, u"Téléchargements" )
		
		#
		# Liste des fichiers
		#
		
		# Layout de grille qui contient le tableau qui liste les fichiers + boutons
		self.gridLayoutFichiers = QtGui.QGridLayout( self.tabFichiers )
		
		# Tableau qui contient la liste des fichiers disponibles pour l'emission courante
		self.tableWidgetFichier = MyQTableWidget( self.tabFichiers )
		# Il a 4 colonnes et 0 ligne (pour l'instant)
		self.tableWidgetFichier.setColumnCount( 3 )
		self.tableWidgetFichier.setRowCount( 0 )
		# On ajoute les titres
		self.tableWidgetFichier.setHorizontalHeaderItem( 0,
														 self.tableWidgetFichier.creerItem( "" ) )
		self.tableWidgetFichier.setHorizontalHeaderItem( 1,
														 self.tableWidgetFichier.creerItem( "Date" ) )
		self.tableWidgetFichier.setHorizontalHeaderItem( 2,
														 self.tableWidgetFichier.creerItem( "Emission" ) )
		# On l'ajoute au layout
		self.gridLayoutFichiers.addWidget( self.tableWidgetFichier, 0, 1, 6, 1 )
		
		# Icones du tableWidget
		self.iconeFichier     = QtGui.QIcon( "ico/gtk-file.svg" )
		self.iconeAjoute      = QtGui.QIcon( "ico/gtk-add.svg" )
		self.iconeTelecharge  = QtGui.QIcon( "ico/gtk-apply.svg" )
		
		# Bouton pour ajouter tous les fichiers a la liste des telechargements
		self.pushButtonToutAjouter = MyQPushButton( self.tabFichiers )
		self.pushButtonToutAjouter.setIcon( QtGui.QIcon( "ico/gtk-add.svg" ) )
		self.pushButtonToutAjouter.setToolTip( u"Ajouter tous les fichiers à la liste des téléchargements" )
		self.gridLayoutFichiers.addWidget( self.pushButtonToutAjouter, 0, 0, 2, 1 )
		
		# Bouton pour rafraichir le plugin courant
		self.pushButtonRafraichirPlugin = MyQPushButton( self.tabFichiers )
		self.pushButtonRafraichirPlugin.setIcon( QtGui.QIcon( "ico/gtk-refresh.svg" ) )
		self.pushButtonRafraichirPlugin.setToolTip( "Rafraichir le plugin" )
		self.gridLayoutFichiers.addWidget( self.pushButtonRafraichirPlugin, 2, 0, 2, 1 )

		# Bouton pour ouvrir la fenetre des preferences du plugin courant
		self.pushButtonPreferencesPlugin = MyQPushButton( self.tabFichiers )
		self.pushButtonPreferencesPlugin.setIcon( QtGui.QIcon( "ico/gtk-preferences.svg" ) )
		self.pushButtonPreferencesPlugin.setToolTip( u"Ouvrir les préférences du plugin" )
		self.gridLayoutFichiers.addWidget( self.pushButtonPreferencesPlugin, 4, 0, 2, 1 )
		
		# On met en place ce layout sur un widget (pour le splitter)
		self.widgetFichiers = QtGui.QWidget()
		self.widgetFichiers.setLayout( self.gridLayoutFichiers )
		
		#
		# Descriptif des fichiers
		#
		
		# Layout de grille
		self.gridLayoutDescriptif = QtGui.QGridLayout()
		
		# Label pour afficher un logo
		self.logoFichierDefaut = QtGui.QPixmap()
		self.logoFichierDefaut.load( "img/gtk-dialog-question.svg" )
		
		self.labelLogo = QtGui.QLabel( self.centralWidget )
		self.labelLogo.setPixmap( self.logoFichierDefaut.scaled( QtCore.QSize( 150, 150 ), QtCore.Qt.KeepAspectRatio ) )
		
		self.gridLayoutDescriptif.addWidget( self.labelLogo, 0, 0, 1, 1 )
		
		# Zone de texte pour afficher un descriptif
		self.plainTextEdit = QtGui.QPlainTextEdit( self.centralWidget )
		self.gridLayoutDescriptif.addWidget( self.plainTextEdit, 0, 1, 1, 2 )
		
		# On met en place ce layout sur un widget (pour le splitter)
		self.widgetDescriptif = QtGui.QWidget()
		self.widgetDescriptif.setLayout( self.gridLayoutDescriptif )
		

		# Onrientation verticale du splitter
		self.tabFichiers.setOrientation( QtCore.Qt.Vertical )
		
		# On ajoute les 2 elements au splitter (qui est notre onglet)
		self.tabFichiers.addWidget( self.widgetFichiers )
		self.tabFichiers.addWidget( self.widgetDescriptif )
		
		#
		# Liste des telechargements
		#
		
		# Layout de grille qui contient le tableau qui liste les fichiers a telecharger + les boutons pour le controller
		self.gridLayoutTelechargement = QtGui.QGridLayout( self.tabTelechargements )
		
		# Tableau qui contient la liste des fichiers a telecharger
		self.tableWidgetTelechargement = MyQTableWidget( self.tabTelechargements )
		# Il a 5 colonnes et 0 ligne (pour l'instant)
		self.tableWidgetTelechargement.setColumnCount( 3 )
		self.tableWidgetTelechargement.setRowCount( 0 )
		# On ajoute le titre des 5 colonnes
		self.tableWidgetTelechargement.setHorizontalHeaderItem( 0,
																self.tableWidgetTelechargement.creerItem( "Date" ) )
		self.tableWidgetTelechargement.setHorizontalHeaderItem( 1,
																self.tableWidgetTelechargement.creerItem( "Emission" ) )
		self.tableWidgetTelechargement.setHorizontalHeaderItem( 2,
																self.tableWidgetTelechargement.creerItem( "Etat" ) )
		# On l'ajoute au layout
		self.gridLayoutTelechargement.addWidget( self.tableWidgetTelechargement, 0, 1, 4, 1 )
		
		# Bouton pour monter l'element selectionne tout en haut de la liste
		self.pushButtonExtremiteMonter = MyQPushButton( self.tabTelechargements )
		self.pushButtonExtremiteMonter.setIcon( QtGui.QIcon( "ico/gtk-jump-to-rtl.svg" ) )
		self.pushButtonExtremiteMonter.setToolTip( u"Placer l'élément sélectionné tout en haut" )
		self.gridLayoutTelechargement.addWidget( self.pushButtonExtremiteMonter, 0, 0, 1, 1 )
		
		# Bouton pour monter l'element selectionne d'un cran dans la liste
		self.pushButtonMonter = MyQPushButton( self.tabTelechargements )
		self.pushButtonMonter.setIcon( QtGui.QIcon( "ico/gtk-go-up.svg" ) )
		self.pushButtonMonter.setToolTip( u"Monter l'élément sélectionné" )
		self.gridLayoutTelechargement.addWidget( self.pushButtonMonter, 1, 0, 1, 1 )

		# Bouton pour descendre l'element selectionne d'un cran dans la liste
		self.pushButtonDescendre = MyQPushButton( self.tabTelechargements )
		self.pushButtonDescendre.setIcon( QtGui.QIcon( "ico/gtk-go-down.svg" ) )
		self.pushButtonDescendre.setToolTip( u"Descendre l'élément selectionné" )
		self.gridLayoutTelechargement.addWidget( self.pushButtonDescendre, 2, 0, 1, 1 )
		
		# Bouton pour descendre l'element selectionne tout en bas de la liste
		self.pushButtonExtremiteDescendre = MyQPushButton( self.tabTelechargements )
		self.pushButtonExtremiteDescendre.setIcon( QtGui.QIcon( "ico/gtk-jump-to-ltr.svg" ) )
		self.pushButtonExtremiteDescendre.setToolTip( u"Placer l'élément sélectionné tout en bas" )
		self.gridLayoutTelechargement.addWidget( self.pushButtonExtremiteDescendre, 3, 0, 1, 1 )
		
		# Bouton pour supprimer tous les elements de la liste
		self.pushButtonToutSupprimer = MyQPushButton( self.tabTelechargements )
		self.pushButtonToutSupprimer.setIcon( QtGui.QIcon( "ico/gtk-cancel.svg" ) )
		self.pushButtonToutSupprimer.setToolTip( u"Supprimer tous les téléchargements" )
		self.gridLayoutTelechargement.addWidget( self.pushButtonToutSupprimer, 0, 2, 1, 1 )
		
		# Bouton pour supprimer de la liste les telechargements termines
		self.pushButtonNettoyer = MyQPushButton( self.tabTelechargements )
		self.pushButtonNettoyer.setIcon( QtGui.QIcon( "ico/gtk-delete-full.svg" ) )
		self.pushButtonNettoyer.setToolTip( u"Supprimer les téléchargement terminés" )
		self.gridLayoutTelechargement.addWidget( self.pushButtonNettoyer, 1, 2, 1, 1 )
		
		# Bouton pour ouvrir le dossier des telechargements
		self.pushButtonOuvrirDossierTelechargement = MyQPushButton( self.tabTelechargements )
		self.pushButtonOuvrirDossierTelechargement.setIcon( QtGui.QIcon( "ico/gtk-folder.svg" ) )
		self.pushButtonOuvrirDossierTelechargement.setToolTip( u"Ouvrir le dossier des téléchargements" )
		self.gridLayoutTelechargement.addWidget( self.pushButtonOuvrirDossierTelechargement, 2, 2, 1, 1 )
		
		#
		# Barre progression de telechargement d'un fichier
		#
		self.progressBarTelechargementFichier = QtGui.QProgressBar( self.centralWidget )
		self.progressBarTelechargementFichier.setProperty( "value", 0 )
		
		#
		# Barre de progression de telechargement des fichiers
		#
		self.progressBarTelechargement = QtGui.QProgressBar( self.centralWidget )
		self.progressBarTelechargement.setProperty( "value", 0 )
		
		#
		# Boutons du bas pour gerer ajouter/supprimer/lancer telechargements
		#
		
		# Layout horizontal qui contiendra les boutons
		self.horizontalLayoutBarreBas = QtGui.QHBoxLayout()
		
		# Bouton pour lancer les telechargements
		self.pushButtonLancer = QtGui.QPushButton( QtGui.QIcon( "ico/gtk-media-play-ltr.svg" ), u"Lancer téléchargement", self.centralWidget )
		self.horizontalLayoutBarreBas.addWidget( self.pushButtonLancer )

		# Bouton pour stopper les telechargements
		self.pushButtonStop = QtGui.QPushButton( QtGui.QIcon( "ico/gtk-media-stop.svg" ), u"Stopper le téléchargement", self.centralWidget )
		self.pushButtonStop.setEnabled( False )
		self.horizontalLayoutBarreBas.addWidget( self.pushButtonStop )	
	
		###
		# Positionnement des differents widgets/layouts sur le layout de grille
		###
		
		# Layout de grille dans lequel on va placer nos widgets/layouts
		self.gridLayout = QtGui.QGridLayout( self.centralWidget )
		# On ajoute la barre du haut
		self.gridLayout.addLayout( self.horizontalLayoutBarreHaut, 0, 0, 1, 3 )
		# On ajoute le gestionnaire d'onglets
		self.gridLayout.addWidget( self.tabWidget, 1, 0, 1, 3 )
		# On ajoute la barre de progression de telechargement d'un fichier
		self.gridLayout.addWidget( self.progressBarTelechargementFichier, 2, 0, 1, 3 )
		# On ajoute la barre de progression de telechargement des fichiers
		self.gridLayout.addWidget( self.progressBarTelechargement, 3, 0, 1, 3 )
		# On ajoute les boutons ajouter/supprimer/lancer
		self.gridLayout.addLayout( self.horizontalLayoutBarreBas, 4, 0, 1, 3 )
		
		###
		# Mise en place le central widget dans la fenetre
		###
		self.setCentralWidget( self.centralWidget )

		###
		# Mise en place du menu
		###
		
		# Menu barre
		self.menubar = QtGui.QMenuBar( self )
		self.menubar.setGeometry( QtCore.QRect( 0, 0, 480, 25 ) )
		
		# Menu Fichier
		self.menuFichier = QtGui.QMenu( "&Fichier", self.menubar )
		self.menubar.addAction( self.menuFichier.menuAction() )
		
		# Action Fichier -> Quitter
		self.actionQuitter = QtGui.QAction( QtGui.QIcon( "ico/gtk-quit.svg" ), "&Quitter", self )
		self.actionQuitter.setIconVisibleInMenu( True )
		self.menuFichier.addAction( self.actionQuitter )
		
		# Menu Edition
		self.menuEdition = QtGui.QMenu( "&Edition", self.menubar )
		self.menubar.addAction( self.menuEdition.menuAction() )
		
		# Action Edition -> Mise a jour
		self.actionMAJ = QtGui.QAction( QtGui.QIcon( "ico/gtk-refresh.svg" ), u"&Mise à jour des plugins", self )
		self.actionMAJ.setIconVisibleInMenu( True )
		self.menuEdition.addAction( self.actionMAJ )
		
		# Action Edition -> Preferences
		self.actionPreferences = QtGui.QAction( QtGui.QIcon( "ico/gtk-preferences.svg" ), u"&Préférences", self )
		self.actionPreferences.setIconVisibleInMenu( True )
		self.menuEdition.addAction( self.actionPreferences )
		
		# Menu Aide
		self.menuAide = QtGui.QMenu( "&Aide", self.menubar )
		self.menubar.addAction( self.menuAide.menuAction() )
		
		# Action Aide -> A propos
		self.actionAPropos = QtGui.QAction( QtGui.QIcon( "ico/gtk-about.svg" ), u"À p&ropos", self )
		self.actionAPropos.setIconVisibleInMenu( True )
		self.menuAide.addAction( self.actionAPropos )
		
		# Ajout du menu a l'interface
		self.setMenuBar( self.menubar )

		###
		# Signaux provenants de l'interface
		###

		QtCore.QObject.connect( self.tableWidgetFichier,
								QtCore.SIGNAL( "cellClicked(int,int)" ),
								self.afficherInformationsFichier )	
		
		QtCore.QObject.connect( self.tableWidgetFichier,
								QtCore.SIGNAL( "cellDoubleClicked(int,int)" ),
								self.gererTelechargement )	
								
		QtCore.QObject.connect( self.pushButtonToutAjouter,
								QtCore.SIGNAL( "clicked()" ),
								self.ajouterTousLesFichiers )								

		QtCore.QObject.connect( self.pushButtonRafraichirPlugin,
								QtCore.SIGNAL( "clicked()" ),
								self.rafraichirPlugin )

		QtCore.QObject.connect( self.tableWidgetTelechargement,
								QtCore.SIGNAL( "cellDoubleClicked(int,int)" ),
								self.supprimerTelechargement )	
		
		QtCore.QObject.connect( self.pushButtonExtremiteMonter,
								QtCore.SIGNAL( "clicked()" ),
								lambda versLeHaut = True, extremite = True : self.tableWidgetTelechargement.deplacerLigne( versLeHaut, extremite ) )
		
		QtCore.QObject.connect( self.pushButtonMonter,
								QtCore.SIGNAL( "clicked()" ),
								lambda versLeHaut = True, extremite = False : self.tableWidgetTelechargement.deplacerLigne( versLeHaut, extremite ) )
		
		QtCore.QObject.connect( self.pushButtonDescendre,
								QtCore.SIGNAL( "clicked()" ),
								lambda versLeHaut = False, extremite = False : self.tableWidgetTelechargement.deplacerLigne( versLeHaut, extremite ) )
		
		QtCore.QObject.connect( self.pushButtonExtremiteDescendre,
								QtCore.SIGNAL( "clicked()" ),
								lambda versLeHaut = False, extremite = True : self.tableWidgetTelechargement.deplacerLigne( versLeHaut, extremite ) )
		
		QtCore.QObject.connect( self.pushButtonToutSupprimer,
								QtCore.SIGNAL( "clicked()" ),
								self.supprimerTousLesTelechargements )
		
		QtCore.QObject.connect( self.pushButtonNettoyer,
								QtCore.SIGNAL( "clicked()" ),
								self.nettoyer )
		
		QtCore.QObject.connect( self.pushButtonLancer,
								QtCore.SIGNAL( "clicked()" ),
								self.lancerTelechargement )

		QtCore.QObject.connect( self.pushButtonStop,
								QtCore.SIGNAL( "clicked()" ),
								self.stopperTelechargement )
		
		QtCore.QObject.connect( self.actionQuitter,
								QtCore.SIGNAL( "triggered()" ),
								self.close )						
	
		################################################
		# Instanciations + initialisation de variables #
		################################################
		
		# Fenetre About
		self.aProposDialog = None
		# Fenetre des preferences du logiciel
		self.preferencesDialog = None
		# Fenetre de mise a jour des plugins
		self.updateManagerDialog = None
		# Nom plugin courant
		self.nomPluginCourant = ""
		# Liste des fichiers
		self.listeFichiers = []
		# Liste des fichiers a telecharger
		self.listeFichiersATelecharger = []
		# Cache des images descriptive
		# Clef : urlImage Valeur : image (binaire)
		self.cacheImage = {}
		
		# On intancie le lanceur de signaux
		self.signaux = Signaux()
		# On instancie le gestionnaire de preferences
		self.preferences = Preferences()
		# On instancie le gestionnaire de preferences des plugins
		self.preferencesPluginDialog = PreferencePluginDialog( self )
		# On instancie le gestionnaire de download
		self.downloader = Downloader( self.signaux )	
		# On recupere l'instance de API
		self.api = API.getInstance()
		# On instancie le gestionnaire d'historique
		self.historique = Historique()
		# On instancie la fenetre d'attente
		self.fenetreAttenteProgressDialog = FenetreAttenteProgressDialog( self )
		# On instancie le gest								 
		
		#
		# Fenetre de confirmation pour quitter le logiciel
		#
		self.quitterMessageBox = QtGui.QMessageBox( self )
		self.quitterMessageBox.setWindowTitle( "Fermeture de TVDownloader" )
		self.quitterMessageBox.setText( u"Voulez-vous réellement quitter TVDownloader ?" )
		self.quitterMessageBox.setInformativeText( u"Votre liste de téléchargement sera perdue" )
		self.quitterMessageBox.addButton( "Oui", QtGui.QMessageBox.AcceptRole )
		self.quitterMessageBox.addButton( "Non", QtGui.QMessageBox.RejectRole )
		
		############################################################
		# On connecte les signaux des instances precedements crees #
		############################################################	

		QtCore.QObject.connect( self.pushButtonOuvrirDossierTelechargement,
								QtCore.SIGNAL( "clicked()" ),
								self.ouvrirRepertoireTelechargement )
		
		QtCore.QObject.connect( self.comboBoxSite,
								QtCore.SIGNAL( "activated(QString)" ),
								self.listerChaines )
		
		QtCore.QObject.connect( self.comboBoxChaine,
								QtCore.SIGNAL( "activated(QString)" ),
								self.listerEmissions )
		
		QtCore.QObject.connect( self.comboBoxEmission,
								QtCore.SIGNAL( "activated(QString)" ),
								self.listerFichiers )

		QtCore.QObject.connect( self.pushButtonPreferencesPlugin,
								QtCore.SIGNAL( "clicked()" ),
								self.ouvrirPreferencesPlugin )

		QtCore.QObject.connect( self.actionPreferences,
								QtCore.SIGNAL( "triggered()" ),
								self.ouvrirPreferencesLogiciel )	
								
		QtCore.QObject.connect( self.actionMAJ,
								QtCore.SIGNAL( "triggered()" ),
								self.ouvrirFenetreMiseAJour )	

		QtCore.QObject.connect( self.actionAPropos,
								QtCore.SIGNAL( "triggered()" ),
								self.ouvrirFenetreAPropos )
		
		QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "debutActualisation(PyQt_PyObject)" ) , self.fenetreAttenteProgressDialog.ouvrirFenetreAttente )
		QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "finActualisation()" ) , self.fenetreAttenteProgressDialog.fermerFenetreAttente )		
		QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "actualiserListesDeroulantes()" ) , self.actualiserListesDeroulantes )
		QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "listeChaines(PyQt_PyObject)" ) , self.ajouterChaines )
		QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "listeEmissions(PyQt_PyObject)" ) , self.ajouterEmissions )
		QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "listeFichiers(PyQt_PyObject)" ) , self.ajouterFichiers )
		QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "nouvelleImage(PyQt_PyObject)" ) , self.mettreEnPlaceImage )
		QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "debutTelechargement(int)" ) , self.debutTelechargement )
		QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "finTelechargement(int)" ) , self.finTelechargement )
		QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "finDesTelechargements()" ) , self.activerDesactiverInterface )
		QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "pourcentageFichier(int)" ) , self.progressBarTelechargementFichier.setValue )		

		#########
		# Début #
		#########			
		
		# La fenetre prend la dimension qu'elle avait a sa fermeture
		taille = self.preferences.getPreference( "tailleFenetre" )
		self.resize( taille[ 0 ], taille[ 1 ] )
		
		# Si aucun plugin n'est active, on ouvre la fenetre des preferences
		if( len( self.preferences.getPreference( "pluginsActifs" ) ) == 0 ):
			self.ouvrirPreferencesLogiciel()
		
		# On actualise tous les plugins
		self.rafraichirTousLesPlugins()

	## Methode qui execute les actions necessaires avant de quitter le programme
	def actionsAvantQuitter( self ):
		# On sauvegarde les options des plugins
		self.api.fermeture()
		# On sauvegarde la taille de la fenetre
		taille = self.size()
		self.preferences.setPreference( "tailleFenetre", [ taille.width(), taille.height() ] )
		# On sauvegarde les options du logiciel
		self.preferences.sauvegarderConfiguration()
		# On sauvegarde l'historique
		self.historique.sauverHistorique()
		# On stoppe les telechargements
		self.stopperTelechargement()

	#########################################
	# Surcharge des methodes de QMainWindow #
	#########################################
	
	## Surcharge de la methode appelee lors de la fermeture de la fenetre
	# Ne doit pas etre appele explicitement
	# @param evenement Evenement qui a provoque la fermeture
	def closeEvent( self, evenement ):
		# On affiche une fenetre pour demander la fermeture si des fichiers sont dans la liste de telechargement
		if( self.tableWidgetTelechargement.rowCount() > 0 ):
			# On affiche une fenetre qui demande si on veut quitter
			retour = self.quitterMessageBox.exec_()
			# Si on veut quitter
			if( retour == 0 ):
				# On execute les actions necessaires
				self.actionsAvantQuitter()
				# On accept la fermeture
				evenement.accept()
			else:
				# On refuse la fermeture
				evenement.ignore()
		else: # S'il n'y a pas de fichier
			# On execute les actions necessaires
			self.actionsAvantQuitter()
			# On accept la fermeture
			evenement.accept()		

	##############################################
	# Methodes pour remplir les menus deroulants #
	##############################################

	## Methode qui actualise les listes deroulantes
	def actualiserListesDeroulantes( self ):
		# On lance juste l'ajout des sites en se basant sur les plugins actifs
		self.ajouterSites( self.preferences.getPreference( "pluginsActifs" ) )

	## Methode qui lance le listage des chaines
	# @param site Nom du plugin/site pour lequel on va lister les chaines
	def listerChaines( self, site ):
		def threadListerChaines( self, nomPlugin ):
			self.signaux.signal( "debutActualisation", nomPlugin )
			listeChaines = self.api.getPluginListeChaines( nomPlugin )
			self.signaux.signal( "listeChaines", listeChaines )
			self.signaux.signal( "finActualisation" )
			
		if( site != "" ):
			self.nomPluginCourant = qstringToString( site )
			threading.Thread( target = threadListerChaines, args = ( self, self.nomPluginCourant ) ).start()
			# On active (ou pas) le bouton de preference du plugin
			self.pushButtonPreferencesPlugin.setEnabled( self.api.getPluginListeOptions( self.nomPluginCourant ) != [] )
		
	## Methode qui lance le listage des emissions
	# @param chaine Nom de la chaine pour laquelle on va lister les emissions
	def listerEmissions( self, chaine ):
		def threadListerEmissions( self, nomPlugin, chaine ):
			self.signaux.signal( "debutActualisation", nomPlugin )
			listeEmissions = self.api.getPluginListeEmissions( nomPlugin, chaine )
			self.signaux.signal( "listeEmissions", listeEmissions )
			self.signaux.signal( "finActualisation" )
			
		if( chaine != "" ):
			threading.Thread( target = threadListerEmissions, args = ( self, self.nomPluginCourant, qstringToString( chaine ) ) ).start()
		
	## Methode qui lance le listage des fichiers
	# @param emission Nom de l'emission pour laquelle on va lister les fichiers
	def listerFichiers( self, emission ):
		def threadListerFichiers( self, nomPlugin, emission ):
			self.signaux.signal( "debutActualisation", nomPlugin )
			listeFichiers = self.api.getPluginListeFichiers( nomPlugin, emission )
			self.signaux.signal( "listeFichiers", listeFichiers )
			self.signaux.signal( "finActualisation" )
			
		if( emission != "" ):
			threading.Thread( target = threadListerFichiers, args = ( self, self.nomPluginCourant, qstringToString( emission ) ) ).start()
	
	## Methode qui met en place une liste de sites sur l'interface
	# @param listeSites Liste des sites a mettre en place
	def ajouterSites( self, listeSites ):
		# On efface la liste des sites
		self.comboBoxSite.clear()
		# On met en place les sites
		for site in listeSites:
			self.comboBoxSite.addItem( stringToQstring( site ) )
		# On selectionne par defaut celui choisis dans les preference
		index = self.comboBoxSite.findText( stringToQstring( self.preferences.getPreference( "pluginParDefaut" ) ) )
		if( index != -1 ):
			self.comboBoxSite.setCurrentIndex( index )
		# On lance l'ajout des chaines
		self.listerChaines( self.comboBoxSite.currentText() )
	
	## Methode qui met en place une liste de chaines sur l'interface
	# @param listeChaines Liste des chaines a mettre en place
	def ajouterChaines( self, listeChaines ):
		# On trie la liste des chaines
		listeChaines.sort()
		# On efface la liste des chaines
		self.comboBoxChaine.clear()
		# On efface la liste des emissions
		self.comboBoxEmission.clear()
		# On efface la liste des fichiers
		self.tableWidgetFichier.toutSupprimer()		
		# On met en place les chaines
		for chaine in listeChaines:
			self.comboBoxChaine.addItem( stringToQstring( chaine ) )
		# Si on a juste une seule chaine
		if( self.comboBoxChaine.count() == 1 ):
			# On lance l'ajout des emissions
			self.listerEmissions( self.comboBoxChaine.currentText() )
		else:
			# On ne selectionne pas de chaine
			self.comboBoxChaine.setCurrentIndex( -1 )			

	## Methode qui met en place une liste d'emissions sur l'interface
	# @param listeEmissions Liste des emissions a mettre en place	
	def ajouterEmissions( self, listeEmissions ):
		# On trie la liste des emissions
		listeEmissions.sort()
		# On efface la liste des emissions
		self.comboBoxEmission.clear()
		# On efface la liste des fichiers
		self.tableWidgetFichier.toutSupprimer()
		# On met en place la liste des emissions
		for emission in listeEmissions:
			self.comboBoxEmission.addItem( stringToQstring( emission ) )
		# Si on a juste une seule emission
		if( self.comboBoxEmission.count() == 1 ):
			# On lance l'ajout des fichiers
			self.listerFichiers( self.comboBoxEmission.currentText() )
		else:
			# On ne selectionne pas d'emission
			self.comboBoxEmission.setCurrentIndex( -1 )
	
	###############################################
	# Methodes pour remplir la liste des fichiers #
	###############################################
	
	## Methode pour ajouter des fichiers a l'interface
	# @param listeFichiers Liste des fichiers a ajouter
	def ajouterFichiers( self, listeFichiers ):
		self.listeFichiers = listeFichiers
		# On efface la liste des fichiers
		self.tableWidgetFichier.toutSupprimer()
		# On commence au depart
		ligneCourante = 0
		# On met en place chacun des fichiers
		for fichier in listeFichiers:
			# On ajoute une ligne
			self.tableWidgetFichier.insertRow( ligneCourante )
			# On ajoute les informations au tableWidgetFichier
			liste = []
			liste.append( self.tableWidgetFichier.creerItem( "" ) )
			liste.append( self.tableWidgetFichier.creerItem( getattr( fichier, "date" ) ) )
			liste.append( self.tableWidgetFichier.creerItem( getattr( fichier, "nom" ) ) )
			self.tableWidgetFichier.setLigne( ligneCourante, liste )
			# On met en place l'icone qui va bien
			self.gererIconeListeFichier( fichier )
			ligneCourante += 1
		# On adapte la taille des colonnes
		self.tableWidgetFichier.adapterColonnes()

	## Methode qui rafraichit le plugin courant
	def rafraichirPlugin( self ):
		def threadRafraichirPlugin( self, nomPlugin ):
			self.signaux.signal( "debutActualisation", nomPlugin )
			self.api.pluginRafraichir( nomPlugin )
			self.signaux.signal( "finActualisation" )
			
		threading.Thread( target = threadRafraichirPlugin, args = ( self, self.nomPluginCourant ) ).start()	


	## Methode qui met en place l'image de la description d'un fichier
	# @param image Image a mettre en place (binaire)
	def mettreEnPlaceImage( self, image ):
		logoFichier = QtGui.QPixmap()
		logoFichier.loadFromData( image )
		self.labelLogo.setPixmap( logoFichier.scaled( QtCore.QSize( 150, 150 ), QtCore.Qt.KeepAspectRatio ) )
	
	## Methode qui affiche des informations sur le fichier selectionne	
	def afficherInformationsFichier( self, ligne, colonne ):
		def threadRecupererImage( self, urlImage ):
			image = self.api.getPage( urlImage )
			self.cacheImage[ urlImage ] = image
			self.signaux.signal( "nouvelleImage", image )
		
		fichier = self.listeFichiers[ ligne ]
		# On recupere le lien de l'image et le texte descriptif
		urlImage        = getattr( fichier, "urlImage" )
		texteDescriptif = getattr( fichier, "descriptif" )
		
		self.plainTextEdit.clear()
		# Si on a un texte descriptif, on l'affiche
		if( texteDescriptif != "" ):
			self.plainTextEdit.appendPlainText( stringToQstring( texteDescriptif ) )
		else:
			self.plainTextEdit.appendPlainText( u"Aucune information disponible" )
		
		# Si on n'a pas d'image
		if( urlImage == "" ):
			# On met en place celle par defaut
			self.logoFichier = self.logoFichierDefaut
			self.labelLogo.setPixmap( self.logoFichier.scaled( QtCore.QSize( 150, 150 ), QtCore.Qt.KeepAspectRatio ) )
		else: # Si on en a une
			# Si elle est dans le cache des images
			if( self.cacheImage.has_key( urlImage ) ):
				self.mettreEnPlaceImage( self.cacheImage[ urlImage ] )
			else: # Sinon
				# On lance le thread pour la recuperer
				threading.Thread( target = threadRecupererImage, args = ( self, urlImage ) ).start()

	## Methode qui gere l'icone d'un fichier dans la liste des telechargements
	# Il y a 3 icones possible :
	#  - C'est un fichier
	#  - C'est un fichier present dans l'historique (donc deja telecharge)
	#  - C'est un fichier present dans la liste des telechargements
	# @param fichier Fichier a gerer
	def gererIconeListeFichier( self, fichier ):
		if( fichier in self.listeFichiers ):
			ligneFichier = self.listeFichiers.index( fichier )
			# On cherche quel icone mettre en place
			if( fichier in self.listeFichiersATelecharger ):
				icone = self.iconeAjoute
			elif( self.historique.comparerHistorique( fichier ) ):
				icone = self.iconeTelecharge
			else:
				icone = self.iconeFichier
			# On met en place l'icone
			self.tableWidgetFichier.item( ligneFichier, 0 ).setIcon( icone )	

	######################################################
	# Methodes pour remplir la liste des telechargements #
	######################################################
	
	## Methode qui gere la liste des telechargements
	# @param ligne   Numero de la ligne (dans la liste des fichiers) de l'element a ajouter
	# @param colonne Numero de colonne (inutile, juste pour le slot)
	def gererTelechargement( self, ligne, colonne = 0 ):
		fichier = self.listeFichiers[ ligne ]
		# Si le fichier est deja dans la liste des telechargements
		if( fichier in self.listeFichiersATelecharger ):
			ligneTelechargement = self.listeFichiersATelecharger.index( fichier )
			self.supprimerTelechargement( ligneTelechargement )
		else: # S'il n'y est pas, on l'ajoute
			self.listeFichiersATelecharger.append( fichier )
			numLigne = self.tableWidgetTelechargement.rowCount()
			# On insere une nouvelle ligne dans la liste des telechargements
			self.tableWidgetTelechargement.insertRow( numLigne )
			# On y insere les elements qui vont biens
			self.tableWidgetTelechargement.setLigne( numLigne, 
													 [ self.tableWidgetTelechargement.creerItem( getattr( fichier, "date" ) ),
													   self.tableWidgetTelechargement.creerItem( getattr( fichier, "nom" ) ),
													   self.tableWidgetTelechargement.creerItem( u"En attente de téléchargement" )
													 ]
												   )
			# On adapte la taille des colonnes
			self.tableWidgetTelechargement.adapterColonnes()
			# On modifie l'icone dans la liste des fichiers
			self.gererIconeListeFichier( fichier )
	
	## Methode qui ajoute tous les fichiers a la liste des telechargements
	def ajouterTousLesFichiers( self ):
		for i in range( self.tableWidgetFichier.rowCount() ):
			self.gererTelechargement( i )
	
	## Methode qui supprime un fichier de la liste des telechargements
	# @param ligne   Numero de la ligne a supprimer
	# @param colonne Numero de colonne (inutile, juste pour le slot)	
	def supprimerTelechargement(  self, ligne, colonne = 0 ):
		fichier = self.listeFichiersATelecharger[ ligne ]
		# On supprime l'element du tableWidgetTelechargement
		self.tableWidgetTelechargement.removeRow( ligne )
		# On supprime l'element de la liste des fichiers a telecharger
		self.listeFichiersATelecharger.remove( fichier )
		# On modifie l'icone dans la liste des fichiers
		self.gererIconeListeFichier( fichier )
	
	## Methode qui supprime tous les telechargement de la liste des telechargements
	def supprimerTousLesTelechargements( self ):
		for i in range( self.tableWidgetTelechargement.rowCount() -1, -1, -1 ):
			self.supprimerTelechargement( i )
		
	## Methode qui lance le telechargement des fichiers	
	def lancerTelechargement( self ):	
		# On liste les emissions a telecharger avec leurs numeros de ligne
		listeFichiers = []
		for i in range( self.tableWidgetTelechargement.rowCount() ): # Pour chaque ligne
			fichier = self.listeFichiersATelecharger[ i ]
			listeFichiers.append( [ i,
									getattr( fichier, "lien" ),
									getattr( fichier, "nomFichierSortie" ) ] )
				
		nbATelecharger = len( listeFichiers )
		# Si on a des elements a charger
		if( nbATelecharger > 0 ):
			# On met en place la valeur du progressBar
			self.progressBarTelechargement.setMaximum( nbATelecharger )
			self.progressBarTelechargement.setValue( 0 )
			# On lance le telechargement
			threading.Thread( target = self.downloader.lancerTelechargement, args = ( listeFichiers, ) ).start()
			# On active/desactive ce qui va bien sur l'interface
			self.activerDesactiverInterface( True )
			
	## Methode qui stoppe le telechargement
	def stopperTelechargement( self ):
		# On stoppe le telechargement
		self.downloader.stopperTelechargement()

	############################################
	# Methodes pour ouvrir les autres fenetres #
	############################################
	
	#
	# Fenetre About
	#
	
	## Methode pour afficher la fenetre About
	def ouvrirFenetreAPropos( self ):
		if( self.aProposDialog == None ):
			self.aProposDialog = AProposDialog()
		self.aProposDialog.show()
		
	#
	# Fenetre de preference du logiciel
	#
	
	## Methode pour ouvrir les preferences du logiciel
	def ouvrirPreferencesLogiciel( self ):
		if( self.preferencesDialog == None ):
			self.preferencesDialog = PreferencesDialog( self, self.signaux )
		self.preferencesDialog.afficher()
	
	#
	# Fenetre de mise a jour des plugins
	#
	
	## Methode pour ouvrir la fenetre de mise a jour des plugins
	def ouvrirFenetreMiseAJour( self ):
		if( self.updateManagerDialog == None ):
			self.updateManagerDialog = UpdateManagerDialog( self )
		self.updateManagerDialog.afficher()	
	
	#
	# Fenetre de preference des plugins
	# 
	
	## Methode pour ouvrir les preferences du plugin courant	
	def ouvrirPreferencesPlugin( self ):
		listeOptions = self.api.getPluginListeOptions( self.nomPluginCourant )
		self.preferencesPluginDialog.ouvrirDialogPreferences( self.nomPluginCourant, listeOptions )
			
	#########
	# Slots #
	#########
	
	## Methode qui ouvre le repertoire de telechargement
	def ouvrirRepertoireTelechargement( self ):
		QtGui.QDesktopServices.openUrl( QtCore.QUrl.fromLocalFile( self.preferences.getPreference( "repertoireTelechargement" ) ) )
	
	## Methode qui rafraichit le plugin courant
	def rafraichirPlugin( self ):
		def threadRafraichirPlugin( self, nomPlugin ):
			self.signaux.signal( "debutActualisation", nomPlugin )
			self.api.pluginRafraichir( nomPlugin )
			self.signaux.signal( "finActualisation" )
			
		threading.Thread( target = threadRafraichirPlugin, args = ( self, self.nomPluginCourant ) ).start()		
	
	## Methode qui rafraichit tous les plugins
	# A utiliser au lancement du programme
	def rafraichirTousLesPlugins( self ):
		def threadRafraichirTousLesPlugins( self ):
			self.signaux.signal( "debutActualisation", "TVDownloader" )
			self.api.pluginRafraichirAuto()
			self.signaux.signal( "finActualisation" )
			self.signaux.signal( "actualiserListesDeroulantes" )
			
		threading.Thread( target = threadRafraichirTousLesPlugins, args = ( self, ) ).start()
	
	## Slot qui active/desactive des elements de l'interface pendant un telechargement
	# @param telechargementEnCours Indique si on telecharge ou pas
	def activerDesactiverInterface( self, telechargementEnCours = False ):
		# Les boutons
		self.pushButtonLancer.setEnabled( not telechargementEnCours )
		self.pushButtonStop.setEnabled( telechargementEnCours )
		self.pushButtonExtremiteMonter.setEnabled( not telechargementEnCours )
		self.pushButtonMonter.setEnabled( not telechargementEnCours )
		self.pushButtonDescendre.setEnabled( not telechargementEnCours )
		self.pushButtonExtremiteDescendre.setEnabled( not telechargementEnCours )
		self.pushButtonToutSupprimer.setEnabled( not telechargementEnCours )
		self.pushButtonNettoyer.setEnabled( not telechargementEnCours )
		
		# Le table widget
		self.tableWidgetTelechargement.setEnabled( not telechargementEnCours )

	## Slot appele lors ce qu'un le debut d'un telechargement commence
	# @param numero Position dans la liste des telechargement du telechargement qui commence
	def debutTelechargement( self, numero ):
		self.tableWidgetTelechargement.item( numero, 2 ).setText( stringToQstring( u"Téléchargement en cours..." ) )
		self.tableWidgetTelechargement.adapterColonnes()
		self.progressBarTelechargementFichier.setValue( 0 )

	## Slot appele lorsqu'un telechargement se finit
	# @param numero Position dans la liste des telechargement du telechargement qui se finit	
	def finTelechargement( self, numero ):
		fichier = self.listeFichiersATelecharger[ numero ]
		# On ajoute le fichier a l'historique
		self.historique.ajouterHistorique( fichier )
		# On modifie l'icone dans la liste des fichiers
		self.gererIconeListeFichier( fichier )	
		# On modifie l'interface
		self.tableWidgetTelechargement.item( numero, 2 ).setText( stringToQstring( u"Fini !" ) )
		self.progressBarTelechargement.setValue( self.progressBarTelechargement.value() + 1 )
		self.tableWidgetTelechargement.adapterColonnes()
		self.progressBarTelechargementFichier.setValue( 100 )
	
	## Slot qui nettoie la liste des telechargements de tous les telechargements finis
	def nettoyer( self ):
		for i in range( self.tableWidgetTelechargement.rowCount() - 1, -1, -1 ): # [ nbLignes - 1, nbLignes - 2, ..., 1, 0 ]
			if( self.tableWidgetTelechargement.item( i, 2 ).text() == u"Fini !" ): # Si c'est telecharge
				self.supprimerTelechargement( i )
Exemplo n.º 25
0
def download(cookies, url, folder):
    downloader = Downloader(cookies, url, folder)
    downloader.run()
Exemplo n.º 26
0
class Watcher:
    def __init__(self,
                 filename,
                 period='1yr',
                 stockPicks=10,
                 avoidDowntrends=True,
                 sellAllOnCrash=True,
                 hyperparameters={}):
        self.filename = filename
        self.period = period
        self.stockPicks = stockPicks
        self.hyperparameters = hyperparameters
        self.sl = hyperparameters['sl']
        self.tp = hyperparameters['tp']
        self.ts = hyperparameters['ts']
        self.avoidDowntrends = avoidDowntrends
        self.sellAllOnCrash = sellAllOnCrash

    def create(self, balance):
        # Create the portolio
        self.portfolio = Portfolio(cash=balance, sl=self.sl, tp=self.tp)
        self.portfolio.save(self.filename)

    def start(self):
        # Open the portolio
        self.portfolio = Portfolio(cash=0, sl=self.sl, tp=self.tp)
        self.portfolio.load(self.filename)
        # Download the stock data
        self.downloader = Downloader(cache=False)
        self.downloader.download(period=self.period)

        date = self.downloader.prices.index[-1]
        prices = self.downloader.prices

        print("#", date)

        # Refresh the portfolio with the latest prices
        self.portfolio.refresh(prices=prices, date=date)

        # Get the portfolio Summary
        portfolio_summary = self.portfolio.summary()

        # Get the opened positions
        opened = self.portfolio.holdings[self.portfolio.holdings['status'] ==
                                         'open']

        # Get the current trend
        zt = TradingZones(prices=prices)
        zones = zt.get()

        output = pd.DataFrame()

        # Activate the trailing stops
        targets = self.portfolio.getPositionsBelowTS(self.ts)
        if len(targets) > 0:
            sell_orders = self.portfolio.getSellOrdersFromSubset(
                subset=targets)
            print("\n----- TS -----\n", sell_orders)
            #self.portfolio.sell(sell_orders, prices, date, label='TS')

        # Activate the stop loss
        if self.sl < 0:
            poor_perfs = opened[opened['profits_pct'] <= self.sl]
            if len(poor_perfs) > 0:
                sell_list = poor_perfs['symbol'].unique()
                sell_orders = self.portfolio.getSellAllOrders(
                    symbols=sell_list)
                print("\n----- SL -----\n", sell_orders)
                #self.portfolio.sell(sell_orders, prices, date, label='SL')

        # Sell all on drop
        if self.sellAllOnCrash == True and zones.iloc[-1][
                'signal'] == 'DOWN' and len(opened) > 0:
            print("Sell all")
            print("\n----- Sell All -----\n")
            #self.portfolio.sellAll(prices, date, label='Market Drop')

        # Get the ones that reached take-profit
        if self.tp > 0:
            good_perfs = opened[opened['profits_pct'] >= self.tp]
            if len(good_perfs) > 0:
                sell_list = good_perfs['symbol'].unique()
                sell_orders = self.portfolio.getSellAllOrders(
                    symbols=sell_list)
                print("\n----- TP -----\n", sell_orders)
                #self.portfolio.sell(sell_orders, prices, date, label='TP')

        # Avoid trading in a general market downtrend
        if self.avoidDowntrends == False or (self.avoidDowntrends == True and
                                             zones.iloc[-1]['signal'] == 'UP'):
            # Pick the stocks
            picker = Picker(prices=prices)
            picker.pick(count=self.stockPicks)
            #print(">", picker.selected)
            # Get the order size
            portfolio_summary = self.portfolio.summary()
            order = Order(prices=prices, picks=picker.selected)
            order.get(budget=portfolio_summary['cash'])
            #print("!", portfolio_summary['cash'], order.orders)
            print("\n----- BUY -----\n", order.orders)
            # Update our portfolio
            #self.portfolio.update(orders=order.orders, prices=prices, date=date)
        else:
            # No new positions in a downtrend!
            # Update our portfolio
            self.portfolio.update(orders=pd.DataFrame(),
                                  prices=prices,
                                  date=date)

    def stats(self):
        # Open the portolio
        self.portfolio = Portfolio(cash=0, sl=self.sl, tp=self.tp)
        self.portfolio.load(self.filename)

        symbols = list(self.portfolio.holdings['symbol'].unique())

        if len(symbols) == 0:
            print("Nothing to update")
            return False
        elif len(symbols) == 1:
            symbols = symbols + [
                'MSFT', 'TSLA'
            ]  # otherwise single symbol = no multi-index = breaking

        # Download the stock data
        self.downloader = Downloader(cache=False)
        self.downloader.download(period=self.period, symbols=symbols)

        date = self.downloader.prices.index[-1]
        prices = self.downloader.prices

        print("#", date)

        # Refresh the portfolio with the latest prices
        self.portfolio.refresh(prices=prices, date=date)

        # Save the portfolio with the latest prices
        self.portfolio.save(self.filename)

        print(self.portfolio.summary())
        print('')

        g = self.portfolio.holdings[self.portfolio.holdings['status'] ==
                                    'open'].copy().groupby('symbol').sum()
        g['profits_pct'] = (g['current_price'] -
                            g['purchase_price']) / g['purchase_price'] * 100
        g['price'] = prices.loc[date]
        print(g)
        print(list(g.index))
        print(prices.loc[date])

    def check(self, symbol='TSLA', period='6mo'):
        # Download the stock data
        self.downloader = Downloader(cache=False)
        if symbol not in self.downloader.symbols:
            self.downloader.add(symbol)
            print('Symbol added to the watch list')
        self.downloader.download(period=period, symbols=[symbol])
        print(self.downloader.prices[symbol].tail(50))
Exemplo n.º 27
0
log.setLevel(level)
log2.setLevel(level)
fmt = logging.Formatter('%(levelname)s:%(name)s:%(message)s')
h = logging.StreamHandler()
h.setFormatter(fmt)
log.addHandler(h)
log2.addHandler(h)

#mongodb
app.config["MONGO_URI"] = "mongodb://"+Config.get('DB', 'mongoHost')+":"+Config.get('DB', 'mongoPort')+"/"+Config.get('DB', 'mongoDbName')
mongo = PyMongo(app)



# objects doing the work
downloader = Downloader(mongo, Config.get('PARSER', 'videoFolder'))
parser = Parser(mongo, Config.get('PARSER', 'videoFolder'), Config.get('PARSER', 'characterFolder'), Config.get('PARSER', 'templateFolder'))
parserTest = ParserTest(mongo, Config.get('PARSER', 'videoFolder'), Config.get('PARSER', 'characterFolder'), Config.get('PARSER', 'templateFolder'))

# fire the main job
@app.route('/firemainjob')
def index():
    print("fire scheduler")
    scheduler.get_job("mainJob").modify(next_run_time=datetime.datetime.now())
    return jsonify({"fire": "ok"})

# download only
@app.route('/onlydl/<idvid>')
def onlydl(idvid):
    print("download:"+idvid)
    downloader.downloadVideo(idvid)
Exemplo n.º 28
0
from MangaSiteFactory import get_manga_site, MangaSiteEnum
from MangaSite import MangaSite
from Manga import Manga, Chapter, MangaIndexTypeEnum
from Downloader import Downloader
from PySide2 import QtWidgets, QtCore
from typing import List, Dict
from functools import partial


def call_timeout(loop):
    loop.quit()
    raise TimeoutError


get_mhg = partial(get_manga_site, MangaSiteEnum.ManHuaGui,
                  Downloader(None, './downloads'))


class TestManHuaGui(unittest.TestCase):
    def test_init(self):
        mhg = get_mhg()
        self.assertEqual(mhg.name, '漫畫鬼')
        self.assertEqual(mhg.url, 'https://www.manhuagui.com/')

    def test_seach_manga(self):
        def search_callback(loop, result: List[Manga]):
            self.assertIsInstance(result, list)
            self.assertEqual(len(result), 2)
            foundFlag = False
            for manga in result:
                if manga.name == '火影忍者':
Exemplo n.º 29
0
def scrapeKupujemProdajem():
    # UTF-8 support
    reload(sys)
    sys.setdefaultencoding('utf-8')
    now = datetime.now()
    ads = []
    try:
        down = Downloader(
            'http://www.kupujemprodajem.com/search.php?action=list&data[category_id]=&data[group_id]=&data[location_id]=&data[keywords]=&submit[search]=Tra%C5%BEi'
        )
        content = down.get_content()
        html = unicode(content)
        link = ""
        title = ""
        imageUrl = ""
        description = "/"
        category = "/"
        subcategory = "/"
        value = "/"
        currency = "/"
        region = "/"
        date = str(now.year) + "-" + str(now.month) + "-" + str(now.day)
        linkovi = xpath.search(html, '//div[@class="item clearfix"]')
        highlighted = xpath.search(
            html, '//div[@class="item clearfix adHighlighted"]')
        for h in highlighted:
            linkovi.append(h)
        for l in linkovi:
            try:
                link = "http://www.kupujemprodajem.com/" + xpath.get(
                    l, '//a[@class="adName"]/@href')
                title = xpath.get(l, '//a[@class="adName"]')
                region = xpath.get(l,
                                   '//section[@class="locationSec"]').strip()
                region = region.split(" | ")
                region = region[0]
                price = xpath.get(l, '//span[@class="adPrice"]')
                price = price.split("&nbsp;")
                if len(price) == 2:
                    value = price[0]
                    value = value.replace(".", "")
                    value = value.split(",")
                    value = value[0]
                    currency = price[1]
                else:
                    value = "/"
                    currency = "/"

                if currency == "&euro;":
                    currency = "EUR"
                elif currency == "din":
                    currency = "DIN"

                down = Downloader(link)
                content = down.get_content()

                category = xpath.get(content, '//a[@class="crumbs"]')
                description = xpath.get(
                    l, '//section[@class="nameSec"]/p[@class="adDescription"]')
                category = category.split("|")
                category = category[0]
                category = category.strip()
                imageUrl = xpath.get(
                    content, '//div[@class="adThumbnailHolder"]/a/img/@src')
                imageUrl = imageUrl.replace("//", "/")
                imageUrl = imageUrl[1::]
                if imageUrl == "":
                    imageUrl = "/"
                description = description.replace("...<p>", "")
                description = description.strip()
                country = u"Србија"
                ad = Ad(link, title, imageUrl, description, category,
                        subcategory, value, currency, region, date, country)
                ads.append(ad)
            except:
                pass
    except:
        pass

    return adsToJson(ads)


#print scrapeKupujemProdajem()
Exemplo n.º 30
0
	def __init__(self, DLlist):
		# On instancie le gestionnaire de preferences et sa fenetre
		self.preferences = Preferences()
		# On instancie le gestionnaire de download
		self.downloader = Downloader()
		# On instancie le gestionnaire d'historique et sa fenetre
		self.historique = Historique()
		current=0
		global choice
		choice=''
		while choice!='r' and choice!='R':	
			if choice=='s' or choice=='S':
				os.system(['clear','cls'][os.name == 'nt'])
				header ('','','Liste de telechargement\t')
				print "\n\n\n\tSupprimer tous les fichiers de la liste de telchargement"
				#pour chaque fichier de la liste  ***(len(DLlist))!=0
				while len(DLlist)>0:
					print "Supprimer le fichier :",DLlist[0].nom
					#supprimer le fichier de la liste
					DLlist.remove(DLlist[0])
					#ajouter le fichier au log
					time.sleep (1)	
			if choice=='t' or choice=='T':
				os.system(['clear','cls'][os.name == 'nt'])
				header ('','','Liste de telechargement\t')
				print "\n\n\n\ttelecharger tous les fichiers"
				#pour chaque fichier de la liste  ***(len(DLlist))!=0
				while len(DLlist)>0:
					if not self.historique.comparerHistorique(DLlist[0]):
						os.system(['clear','cls'][os.name == 'nt'])
						header ('','','Liste de telechargement\t')
						print "\n\n\n\ttelecharger le fichier :",DLlist[0].nom
						if(DLlist[0].nomFichierSortie=="" ):
							DLlist[0].nomFichierSortie=os.path.basename(getattr(DLlist[0],"lien" ))
						#telecharger le fichier
						self.downloader.lancerTelechargement([[0,DLlist[0].lien,DLlist[0].nomFichierSortie]])
						#ajouter le fichier a l'historique de telechargement
						self.historique.ajouterHistorique(DLlist[0])		
					else:
						os.system(['clear','cls'][os.name == 'nt'])
						header ('','','Liste de telechargement\t')
						print "\n\n\n\tFichier deja telecharge"
					#supprimer le fichier de la liste
					DLlist.remove(DLlist[0])
					time.sleep (1)
			elif choice=='q' or choice=='Q':
				quitter()
			elif choice.isdigit() and len(DLlist)>int(choice)+10*current and int(choice)>=0:
				value=''
				while (value!='r' and value!='t' and value!='s'):
					os.system(['clear','cls'][os.name == 'nt'])
					header ('','','Liste de telechargement\t')
					print "\n\n\n\tFichier :",DLlist[int(choice)+10*current].nom,"\n\n\tQue voulez vous faire?\n\n\t\tt:telecharger le fichier\n\t\ts:supprimer le fichier de la liste de telechargement\n\t\tr:retour a liste de telechargement\n\n\n\n\n"
					value=getch()
				if value=='t':
					if not self.historique.comparerHistorique(DLlist[int(choice)+10*current]):
						os.system(['clear','cls'][os.name == 'nt'])
						header ('','','Liste de telechargement\t')
						print "\n\n\n\ttelecharger le fichier :",DLlist[int(choice)].nom
						# Si le nom du fichier de sortie n'existe pas, on l'extrait de l'URL
						if(DLlist[int(choice)+10*current].nomFichierSortie=="" ):
							DLlist[int(choice)+10*current].nomFichierSortie=os.path.basename(getattr(DLlist[int(choice)+10*current],"lien" ))
						#telecharger le fichier
						self.historique.ajouterHistorique(DLlist[int(choice)+10*current])
						self.downloader.lancerTelechargement([[0,DLlist[int(choice)].lien,DLlist[int(choice)].nomFichierSortie]])
						#ajouter le fichier a l'historique de telechargement
					else:
						os.system(['clear','cls'][os.name == 'nt'])
						header ('','','Liste de telechargement\t')
						print "\n\n\n\tFichier deja telecharge"
					time.sleep(1)
					#supprimer le fichier de la liste
					DLlist.remove(DLlist[int(choice)])
				elif value=='s':
					os.system(['clear','cls'][os.name == 'nt'])
					header ('','','Liste de telechargement\t')
					print "\n\n\n\tSuppression de la liste de telechargement du fichier :\n\n\t\t",DLlist[int(choice)].nom
					#supprimer le fichier de la liste
					DLlist.remove(DLlist[int(choice)])
				elif value=='r':
					os.system(['clear','cls'][os.name == 'nt'])
					header ('','','Liste de telechargement\t')
					print "\n\n\n\tRetour a la liste de telechargement"
				#ajouter le fichier au log
				time.sleep(1)
			elif choice=='*' :
				value=''
				os.system(['clear','cls'][os.name == 'nt'])
				header ('','','Liste de telechargement\t')
				print "\n\n\n\n\n\tQue voulez vous faire?\n\n\t\tt:telecharger les fichiers\n\t\ts:supprimer les fichiers de la liste de telechargement\n\n\n\n\n\n"
				value=getch()
				while len(DLlist)>0:
					if value=='t':
						if not self.historique.comparerHistorique(DLlist[0]):
							os.system(['clear','cls'][os.name == 'nt'])
							header ('','','Liste de telechargement\t')
							print "\n\n\n\ttelecharger le fichier :",DLlist[0].nom
							# Si le nom du fichier de sortie n'existe pas, on l'extrait de l'URL
							if(DLlist[0].nomFichierSortie=="" ):
								DLlist[0].nomFichierSortie=os.path.basename(getattr(DLlist[0],"lien" ))
							#telecharger le fichier
							self.downloader.lancerTelechargement([[0,DLlist[0].lien,DLlist[0].nomFichierSortie]])
							#ajouter le fichier a l'historique de telechargement
							self.historique.ajouterHistorique(DLlist[0])
						else:
							os.system(['clear','cls'][os.name == 'nt'])
							header ('','','Liste de telechargement\t')
							print "\n\n\n\tFichier deja telecharge"	
							time.sleep(0.5)
					#supprimer le fichier de la liste
					DLlist.remove(DLlist[0])
					#ajouter le fichier au log
			elif choice=='+':
				if len(DLlist)>current*10+10: current+=1
			elif choice=='-':
				if current!=0: current-=1
			choice=''
			#sauvegarder l'historique de telechargement
			self.historique.sauverHistorique()
			#affichage a l'ecran de la liste
			header ('','','Liste de telechargement\t')
			for i in range(10):
				if len(DLlist)>i+10*current:
					if len(DLlist[int(i+10*current)].nom)>74: print " ",i,":",DLlist[int(i+10*current)].nom[:71]+"..."
					elif len(DLlist[int(i+10*current)].nom)<=74: print " ",i,":",DLlist[int(i+10*current)].nom
				else: print ""
			if len(DLlist)>10:print "\n\t+:fichiers suivants\t-:fichiers precedents  (page",current+1,"/",len(DLlist)/10+1,",",len(DLlist),"chaines)"
			else:print"\n"
			print "\n\tt:telecharger tous les fichiers  s:supprimer tous les fichiers"
			footer()
			if len(DLlist)==0:
				os.system(['clear','cls'][os.name == 'nt'])
				header ('','','Liste de telechargement\t')
				print "\n\n\n\n\n\t\tAucun fichier dans la liste"
				footer()
				choice='r'
				time.sleep(1)
			if not choice:choice=getch()
Exemplo n.º 31
0
    def download(self, collects, keyword):
        self.print_with_color("Download Images...", "b")

        image_count = 0

        keyword = 'images/' + keyword

        # Make directory and path
        if not os.path.isdir(keyword):
            print("Make a directory.")
            os.makedirs(keyword)

        dir = os.getcwd() + "/" + str(keyword)
        # os.chdir(dir)
        self.print_with_color("Download path is %s" % dir, "g")

        if not dir.endswith("/"):
            dir += '/'

        thread_count = self.num_of_thread

        queue = Queue()

        self.printProgressBar(0,
                              len(collects),
                              prefix='Progress:',
                              suffix='Complete')
        for i in range(thread_count):
            downloader = Downloader(queue, dir, self.collectorName,
                                    self.error_list, len(collects))
            downloader.daemon = True
            downloader.start()

        for i in range(len(collects)):
            url = collects[i]
            queue.put(url)

        queue.join()

        self.printProgressBar(len(collects),
                              len(collects),
                              prefix='Progress:',
                              suffix='Complete')

        # Start download
        # self.printProgressBar(0, len(collects), prefix='Progress:', suffix='Complete')
        # for i in range(len(collects)):
        #     self.printProgressBar(i+1, len(collects), prefix='Progress:', suffix='Complete')
        #     # print("Download Image (%d/%d)" % (i+1, len(collects)))
        #     col = collects[i]
        #     if (col == '알림'):
        #         pass
        #     else:
        #         full_name = self.collectorName + str(i+1) + ".jpg"
        #         save_path = os.path.join(dir, full_name)  # 저장폴더
        #
        #         try:
        #             with eventlet.Timeout(self.DOWNLOAD_TIMEOUT):
        #                 urllib2.urlretrieve(col, save_path)
        #         except:
        #             self.error_list.append(col)

        self.print_with_color(
            "Save %d images" % (len(collects) - len(self.error_list)), "g")
Exemplo n.º 32
0
            gui = GUI(u'script-%s-main.xml' % addon_name, addon_path).doModal()
            homewindow.clearProperty("homewindowactive")
            del gui
        elif window == "colorconfig":
            from ColorConfigDialog import ColorConfigDialog
            gui = ColorConfigDialog(u'script-%s-colorconfig.xml' % addon_name, addon_path).doModal()
            del gui
        elif window == "trailers":
            from TrailerWindow import TrailerWindow
            gui = TrailerWindow(u'script-%s-trailers.xml' % addon_name, addon_path).doModal()
            del gui
        elif window == "favourites":
            from FavWindow import FavWindow
            gui = FavWindow(u'script-globalsearch-main.xml', addon_path).doModal()
            del gui
        elif window == "dialogalbuminfo":
            from DialogAlbumInfo import DialogAlbumInfo
            gui = DialogAlbumInfo(u'script-%s-dialogalbuminfo.xml' % addon_name, addon_path).doModal()
            del gui
        elif window == "fullscreeninfo":
            from FullscreenInfo import FullscreenInfo
            gui = FullscreenInfo(u'script-%s-fullscreeninfo.xml' % addon_name, addon_path).doModal()
            del gui
        elif window == "downloader":
            from Downloader import Downloader
            dialog = Downloader()
            dialog.show_download_dialog()
            del dialog
    else:
        MoveProperties(container, focuscontrol)
def download_from_urls(url_file, out_dir):
    print("Downloading data from URLS in %s to %s" % (url_file, out_dir))
    dl = Downloader(json.loads(open(url_file).read()))
    dl.save_files(out_dir)
Exemplo n.º 34
0
class SchedulerGrid(Scheduler):
    def __init__(self, name):
        Scheduler.__init__(self, name)
        self.states = [ "Acl", "cancelReason", "cancelling","ce_node","children", \
                      "children_hist","children_num","children_states","condorId","condor_jdl", \
                      "cpuTime","destination", "done_code","exit_code","expectFrom", \
                      "expectUpdate","globusId","jdl","jobId","jobtype", \
                      "lastUpdateTime","localId","location", "matched_jdl","network_server", \
                      "owner","parent_job", "reason","resubmitted","rsl","seed",\
                      "stateEnterTime","stateEnterTimes","subjob_failed", \
                      "user tags" , "status" , "status_code","hierarchy"]
        return

    def configure(self, cfg_params):
        self.cfg_params = cfg_params
        self.jobtypeName = cfg_params.get('CRAB.jobtype', '')
        self.schedulerName = cfg_params.get('CRAB.scheduler', '')
        Scheduler.configure(self, cfg_params)
        self.proxyValid = 0

        self.dontCheckProxy = int(cfg_params.get("GRID.dont_check_proxy", 0))
        self.space_token = cfg_params.get("USER.space_token", None)
        try:
            self.proxyServer = Downloader(
                "http://cmsdoc.cern.ch/cms/LCG/crab/config/").config(
                    "myproxy_server.conf")
            self.proxyServer = self.proxyServer.strip()
            if self.proxyServer is None:
                raise CrabException("myproxy_server.conf retrieved but empty")
        except Exception, e:
            common.logger.info(
                "Problem setting myproxy server endpoint: using myproxy.cern.ch"
            )
            common.logger.debug(e)
            self.proxyServer = 'myproxy.cern.ch'
        self.group = cfg_params.get("GRID.group", None)
        self.role = cfg_params.get("GRID.role", None)

        removeBList = cfg_params.get("GRID.remove_default_blacklist", 0)
        blackAnaOps = None
        if int(removeBList) == 0:
            blacklist = Downloader(
                "http://cmsdoc.cern.ch/cms/LCG/crab/config/")
            result = blacklist.config("site_black_list.conf")
            if result != None:
                blackAnaOps = result
            common.logger.debug("Enforced black list: %s " % blackAnaOps)
        else:
            common.logger.info("WARNING: Skipping default black list!")

        self.EDG_ce_black_list = None
        if cfg_params.has_key(
                'GRID.ce_black_list') and cfg_params['GRID.ce_black_list']:
            self.EDG_ce_black_list = cfg_params.get('GRID.ce_black_list')
            if int(removeBList) == 0 and blackAnaOps:
                self.EDG_ce_black_list += ",%s" % blackAnaOps
        elif int(removeBList) == 0 and blackAnaOps:
            self.EDG_ce_black_list = blackAnaOps
        if self.EDG_ce_black_list:
            self.EDG_ce_black_list = str(self.EDG_ce_black_list).split(',')

        self.EDG_ce_white_list = cfg_params.get('GRID.ce_white_list', None)
        if (self.EDG_ce_white_list):
            self.EDG_ce_white_list = str(self.EDG_ce_white_list).split(',')

        self.VO = cfg_params.get('GRID.virtual_organization', 'cms')

        self.EDG_clock_time = cfg_params.get('GRID.max_wall_clock_time', None)

        # Default minimum CPU time to >= 130 minutes
        self.EDG_cpu_time = cfg_params.get('GRID.max_cpu_time', '130')

        ## Add EDG_WL_LOCATION to the python path
        #if not self.CRAB_useServer and not self.CRAB_serverName:
        #    if not os.environ.has_key('EDG_WL_LOCATION'):
        #        msg = "Error: the EDG_WL_LOCATION variable is not set."
        #        raise CrabException(msg)
        #    path = os.environ['EDG_WL_LOCATION']
        #    libPath=os.path.join(path, "lib")
        #    sys.path.append(libPath)
        #    libPath=os.path.join(path, "lib", "python")
        #    sys.path.append(libPath)

        self.checkProxy()
        return
Exemplo n.º 35
0
from Controlador import Controlador
from Downloader import Downloader

if __name__ == '__main__':
    control = Controlador()
    Vista = Downloader(control)
    control.setVista(Vista)
    control.iniciar()
Exemplo n.º 36
0
class SchedulerGrid(Scheduler):

    def __init__(self, name):
        Scheduler.__init__(self,name)
        self.states = [ "Acl", "cancelReason", "cancelling","ce_node","children", \
                      "children_hist","children_num","children_states","condorId","condor_jdl", \
                      "cpuTime","destination", "done_code","exit_code","expectFrom", \
                      "expectUpdate","globusId","jdl","jobId","jobtype", \
                      "lastUpdateTime","localId","location", "matched_jdl","network_server", \
                      "owner","parent_job", "reason","resubmitted","rsl","seed",\
                      "stateEnterTime","stateEnterTimes","subjob_failed", \
                      "user tags" , "status" , "status_code","hierarchy"]
        return

    def configure(self, cfg_params):
        self.cfg_params = cfg_params
        self.jobtypeName   = cfg_params.get('CRAB.jobtype','')
        self.schedulerName = cfg_params.get('CRAB.scheduler','')
        Scheduler.configure(self,cfg_params)
        self.proxyValid=0

        self.dontCheckProxy=int(cfg_params.get("GRID.dont_check_proxy",0)) 	 
        self.space_token = cfg_params.get("USER.space_token",None) 	 
 	try:
            self.proxyServer = Downloader("http://cmsdoc.cern.ch/cms/LCG/crab/config/").config("myproxy_server.conf")
            self.proxyServer = self.proxyServer.strip()
            if self.proxyServer is None:
                raise CrabException("myproxy_server.conf retrieved but empty")
        except Exception, e:
            common.logger.info("Problem setting myproxy server endpoint: using myproxy.cern.ch")
            common.logger.debug(e)
            self.proxyServer= 'myproxy.cern.ch'
        self.group = cfg_params.get("GRID.group", None) 	 
        self.role = cfg_params.get("GRID.role", None)

        removeBList = cfg_params.get("GRID.remove_default_blacklist", 0 )
        blackAnaOps = None
        if int(removeBList) == 0:
            blacklist = Downloader("http://cmsdoc.cern.ch/cms/LCG/crab/config/")
            result = blacklist.config("site_black_list.conf")
            if result != None:
                blackAnaOps = result
            common.logger.debug("Enforced black list: %s "%str(blacklist))
        else:
            common.logger.info("WARNING: Skipping default black list!")

        self.EDG_ce_black_list = None
        if cfg_params.has_key('GRID.ce_black_list') and cfg_params['GRID.ce_black_list']:
            self.EDG_ce_black_list = cfg_params.get('GRID.ce_black_list')
            if int(removeBList) == 0 and blackAnaOps: 
                self.EDG_ce_black_list += ",%s"%blackAnaOps
        elif int(removeBList) == 0 and blackAnaOps:
            self.EDG_ce_black_list = blackAnaOps
        if self.EDG_ce_black_list:
            self.EDG_ce_black_list = str(self.EDG_ce_black_list).split(',')

        self.EDG_ce_white_list = cfg_params.get('GRID.ce_white_list',None)
        if (self.EDG_ce_white_list): self.EDG_ce_white_list = str(self.EDG_ce_white_list).split(',')

        self.VO = cfg_params.get('GRID.virtual_organization','cms')

        self.EDG_clock_time = cfg_params.get('GRID.max_wall_clock_time',None)

        # Default minimum CPU time to >= 130 minutes
        self.EDG_cpu_time = cfg_params.get('GRID.max_cpu_time', '130')

        ## Add EDG_WL_LOCATION to the python path
        #if not self.CRAB_useServer and not self.CRAB_serverName:
        #    if not os.environ.has_key('EDG_WL_LOCATION'):
        #        msg = "Error: the EDG_WL_LOCATION variable is not set."
        #        raise CrabException(msg)
        #    path = os.environ['EDG_WL_LOCATION']
        #    libPath=os.path.join(path, "lib")
        #    sys.path.append(libPath)
        #    libPath=os.path.join(path, "lib", "python")
        #    sys.path.append(libPath)

        self.checkProxy()
        return
Exemplo n.º 37
0
                                    addon_path).doModal()
            del gui
        elif window == "trailers":
            from TrailerWindow import TrailerWindow
            gui = TrailerWindow(u'script-%s-trailers.xml' % addon_name,
                                addon_path).doModal()
            del gui
        elif window == "favourites":
            from FavWindow import FavWindow
            gui = FavWindow(u'script-globalsearch-main.xml',
                            addon_path).doModal()
            del gui
        elif window == "dialogalbuminfo":
            from DialogAlbumInfo import DialogAlbumInfo
            gui = DialogAlbumInfo(
                u'script-%s-dialogalbuminfo.xml' % addon_name,
                addon_path).doModal()
            del gui
        elif window == "fullscreeninfo":
            from FullscreenInfo import FullscreenInfo
            gui = FullscreenInfo(u'script-%s-fullscreeninfo.xml' % addon_name,
                                 addon_path).doModal()
            del gui
        elif window == "downloader":
            from Downloader import Downloader
            dialog = Downloader()
            dialog.show_download_dialog()
            del dialog
    else:
        MoveProperties(container, focuscontrol)
Exemplo n.º 38
0
def scrapeHaloOglasi():
    # UTF-8 support
    reload(sys)
    sys.setdefaultencoding('utf-8')
    now = datetime.now()
    down = Downloader(
        'http://www.halooglasi.com/naslovna.240.html?search_text=&sortColumn=VremeDodavanja'
    )
    content = down.get_content()
    html = unicode(content)
    celo = xpath.get(html, '//div[@class="results_container"]')
    linkovi = xpath.search(celo, '//div[@class="result_brza"]')
    ads = []
    for l in linkovi:
        link = xpath.get(l, '//div[@style="height:auto;"]/h2/a/@href')
        link = "http://www.halooglasi.com" + link
        download = Downloader(link)
        cont = download.get_content()
        cont = unicode(cont)
        title = xpath.get(cont, '//div[@class="detail_bar_nek"]/h2').strip()
        if title == "":
            title = xpath.get(cont, '//div[@class="detail_bar"]/h2').strip()
        imageUrl = xpath.get(l, '//a[@class="thumb"]/img/@src')
        imageUrl = "http://www.halooglasi.com" + imageUrl

        description = xpath.get(l, '//div[@class="text_ogl"]/p')

        kategorija = xpath.get(l, '//div[@class="brza_link"]').strip()
        kategorija = kategorija.split("\r\n\t\t\t\t\t\t\r\n\t\t\t\t\t\t")
        kategorija = kategorija[1]
        kategorija = kategorija.split(" > ")
        category = kategorija[0]
        if len(kategorija) > 2:
            subcategory = kategorija[1]
        else:
            kategorija = kategorija[1].split("'>")
            kategorija = kategorija[1]
            kategorija = kategorija.split("<")
            subcategory = kategorija[0]
        price = xpath.get(cont, '//div[@class="price"]').strip()  #price
        if price == "":
            price = xpath.get(
                cont, '//div[@class="price deal"]').strip()  #price deal
        price = price.replace(".", "")
        price = price.replace("din", " DIN")
        price = price.replace("&euro;", " EUR")
        if price == "Dogovor":
            value = "/"
            currency = "/"
        else:
            price = price.split(" ")
            value = price[0]
            currency = price[1]
        date_loc = xpath.search(l, '//div[@class="datum_grad"]/h6/span')
        date_loc[0] = date_loc[0].strip()
        date = date_loc[0].split("\r\n")
        date = date[0]
        date = date.replace(".", "")
        date = date.split(" ")
        if date[1] == "Jan":
            date[1] = "1"
        elif date[1] == "Feb":
            date[1] = "2"
        elif date[1] == "Mar":
            date[1] = "3"
        elif date[1] == "Apr":
            date[1] = "4"
        elif date[1] == "Maj":
            date[1] = "5"
        elif date[1] == "Jun":
            date[1] = "6"
        elif date[1] == "Jul":
            date[1] = "7"
        elif date[1] == "Avg":
            date[1] = "8"
        elif date[1] == "Sep":
            date[1] = "9"
        elif date[1] == "Okt":
            date[1] = "10"
        elif date[1] == "Nov":
            date[1] = "11"
        elif date[1] == "Dec":
            date[1] = "12"
        date = date[2] + "-" + date[1] + "-" + date[0]
        l = date_loc[1].strip()
        l = l.split("&nbsp;")
        region = l[0]
        country = u"Србија"

        ad = Ad(link, title, imageUrl, description, category, subcategory,
                value, currency, region, date, country)
        ads.append(ad)

    return adsToJson(ads)


#print scrapeHaloOglasi()
Exemplo n.º 39
0
class ServerConfig:
    def __init__(self, serverName):
        import string
        self.serverName = string.lower(serverName)
        common.logger.debug('Calling ServerConfig '+self.serverName)

        url ='http://cmsdoc.cern.ch/cms/LCG/crab/config/'

        self.downloader = Downloader(url)
          

    def config(self):
        """
        """
        if 'default' in  self.serverName:
            self.serverName = self.selectServer() 
        if 'server_' in self.serverName:
            configFileName = '%s.conf'%self.serverName
        else: 
            configFileName = 'server_%s.conf'%self.serverName
 
        serverConfig = eval(self.downloader.config(configFileName))
 
        if not serverConfig:
            serverConfig = {} 
        serverConfig['serverGenericName']=self.serverName

        return serverConfig
 
    def selectServer(self):
        """
        """
        common.logger.debug('getting serverlist from web')
        # get a list of available servers 
        serverListFileName ='AvailableServerList'
 
        serverListFile = self.downloader.config(serverListFileName)
 
        if not serverListFile:
            msg = 'List of avalable Server '+serverListFileName+' from '+self.url+' is empty\n'
            msg += 'Please report to CRAB feedback hypernews [email protected]'
            raise CrabException(msg)
        # clean up empty lines and comments
        serverList=[]
        [serverList.append(string.split(string.strip(it))) for it in serverListFile.split('\n') if (it.strip() and not it.strip()[0]=="#")]
        common.logger.debug('All avaialble servers: '+str(serverList))
 
        # select servers from client version
        compatibleServerList=[]
        for s in serverList:
            #vv=string.split(s[1],'-')
            if len(s)<2:
                continue
            vv=s[1].split('-')
            if len(vv[0])==0: vv[0]='0.0.0'
            if len(vv[1])==0: vv[1]='99.99.99'
            for i in 0,1:
                tmp=[]
                [tmp.append(int(t)) for t in vv[i].split('.')]
                vv[i]=tuple(tmp)
            
            if vv[0]<=common.prog_version and common.prog_version<=vv[1] and common.scheduler.name()==string.lower(s[2]):
                compatibleServerList.append(s[0])
 
        common.logger.debug('All avaialble servers compatible with %s: '%common.prog_version_str +str(compatibleServerList))
        if len(compatibleServerList)==0: 
            msg = "No compatible server for client version %s and scheduler %s\n"%(common.prog_version_str,common.scheduler.name())
            msg += "Exiting"
            common.logger.info(msg)
            raise CrabException(msg)
 
        # if more than one, pick up a random one, waiting for something smarter (SiteDB)
        import random
        serverName = random.choice(compatibleServerList)
        common.logger.debug('Avaialble servers: '+str(compatibleServerList)+' choosen: '+serverName)
 
        return serverName
Exemplo n.º 40
0
def threader_crawler(seed_url,
                     resource_regiex=None,
                     link_regiex=".*",
                     delay=5,
                     cache=None,
                     download_source_callback=None,
                     user_agent="wswp",
                     proxies=None,
                     num_retries=1,
                     max_threads=10,
                     timeout=60,
                     max_url=500):

    downloaded = []

    crawl_queue = [seed_url]

    seen = set([seed_url])

    D = Downloader(cache=cache,
                   delay=delay,
                   user_agent=user_agent,
                   proxies=proxies,
                   num_retries=num_retries,
                   timeout=timeout)
    print(user_agent)

    def process_queue():
        while True:

            links = []
            try:
                url = crawl_queue.pop()
            except IndexError:
                break
            else:
                html = D(url)
                downloaded.append(url)

                if download_source_callback:
                    if resource_regiex and re.match(resource_regiex, url):
                        download_source_callback(url, html)
                links.extend([
                    link for link in get_links(html)
                    if re.match(link_regiex, link)
                ])
                for link in links:
                    link = normalize(seed_url, link)
                    if link not in seen:
                        seen.add(link)

                        if same_domain(seed_url, link):
                            crawl_queue.append(link)
                print("已经发现的总网页数目为", len(seen))
                print("已经下载过的网页数目为", len(downloaded))
                print("还没有遍历过的网页数目为", len(crawl_queue))

    threads = []
    while threads or crawl_queue:
        if len(downloaded) == max_url:
            return
        for thread in threads:
            if not thread.is_alive():
                threads.remove(thread)
        while len(threads) < max_threads and crawl_queue:
            print("线程数量为", len(threads))
            thread = threading.Thread(target=process_queue)
            thread.setDaemon(True)
            thread.start()
            print("线程数量为", len(threads))
            threads.append(thread)
Exemplo n.º 41
0
 def run(self):
     tempPath = path + self.video.name + ".mp4"
     self.downLoader = Downloader(self.video.videoUrl, tempPath)
     self.downLoader.downLoad()
     self.singal.emit()
Exemplo n.º 42
0
 def __init__(self):
     print 'init'
     self.urlManager = UrlManager()
     self.downloader = Downloader()
     self.praser = HtmlPraser()
     self.outputer = Output()
Exemplo n.º 43
0
 def download32bit(self):
     url32bit = 'https://go.microsoft.com/fwlink/?LinkID=623231'
     Downloader(self.directoryName,self.IDEName,url32bit)
Exemplo n.º 44
0
class ServerConfig:
    def __init__(self, serverName):
        import string
        self.serverName = string.lower(serverName)
        common.logger.debug('Calling ServerConfig ' + self.serverName)

        url = 'http://cmsdoc.cern.ch/cms/LCG/crab/config/'

        self.downloader = Downloader(url)

    def config(self):
        """
        """
        if 'default' in self.serverName:
            self.serverName = self.selectServer()
        if 'server_' in self.serverName:
            configFileName = '%s.conf' % self.serverName
        else:
            configFileName = 'server_%s.conf' % self.serverName

        serverConfig = eval(self.downloader.config(configFileName))

        if not serverConfig:
            serverConfig = {}
        serverConfig['serverGenericName'] = self.serverName

        return serverConfig

    def selectServer(self):
        """
        """
        common.logger.debug('getting serverlist from web')
        # get a list of available servers
        serverListFileName = 'AvailableServerList'

        serverListFile = self.downloader.config(serverListFileName)

        if not serverListFile:
            msg = 'List of avalable Server ' + serverListFileName + ' from ' + self.url + ' is empty\n'
            msg += 'Please report to CRAB feedback hypernews [email protected]'
            raise CrabException(msg)
        # clean up empty lines and comments
        serverList = []
        [
            serverList.append(string.split(string.strip(it)))
            for it in serverListFile.split('\n')
            if (it.strip() and not it.strip()[0] == "#")
        ]
        common.logger.debug('All avaialble servers: ' + str(serverList))

        # select servers from client version
        compatibleServerList = []
        for s in serverList:
            #vv=string.split(s[1],'-')
            if len(s) < 2:
                continue
            vv = s[1].split('-')
            if len(vv[0]) == 0: vv[0] = '0.0.0'
            if len(vv[1]) == 0: vv[1] = '99.99.99'
            for i in 0, 1:
                tmp = []
                [tmp.append(int(t)) for t in vv[i].split('.')]
                vv[i] = tuple(tmp)

            if vv[0] <= common.prog_version and common.prog_version <= vv[
                    1] and common.scheduler.name() == string.lower(s[2]):
                compatibleServerList.append(s[0])

        common.logger.debug('All avaialble servers compatible with %s: ' %
                            common.prog_version_str +
                            str(compatibleServerList))
        if len(compatibleServerList) == 0:
            msg = "No compatible server for client version %s and scheduler %s\n" % (
                common.prog_version_str, common.scheduler.name())
            msg += "Exiting"
            common.logger.info(msg)
            raise CrabException(msg)

        # if more than one, pick up a random one, waiting for something smarter (SiteDB)
        import random
        serverName = random.choice(compatibleServerList)
        common.logger.debug('Avaialble servers: ' + str(compatibleServerList) +
                            ' choosen: ' + serverName)

        return serverName
Exemplo n.º 45
0
 def download64bit(self):
     url64bit = 'https://go.microsoft.com/fwlink/?Linkid=850641'
     Downloader(self.directoryName,self.IDEName,url64bit)
def link_crawler(seed_url,
                 link_regex=None,
                 delay=5,
                 max_depth=-1,
                 max_urls=-1,
                 headers=None,
                 proxies=None,
                 cookies=None,
                 num_retries=5,
                 scrape_callback=None,
                 cache=None,
                 max_threads=10,
                 timeout=60):
    """Crawl from the given seed URL following links matched by link_regex
        Crawl this website in mltiple threads
    seed_url:种子链接,爬虫的初始链接
    delay:避免请求太频繁,所采取的延时
    max_depth:从种子链接向下,所链接的深度
    max_urls:连续max_urls次下载同一个网页,得到404后,停止下载,跳出循环
    headers:请求头
    proxy:传入ProxiesPool类
    num_retries:当网页的返回识别码在400和500之间时,重复下载的数,如果还不能下载返回空字符串
    cahce:是否存在缓存,用法例如:
        cache = MongoCache(expires=timedelta())
        D = Downloader(cache=cache)
        html = D('http://esf.nb.fang.com/')
    """
    # the queue of URL's that still need to be crawled
    crawl_queue = deque([seed_url])
    # the URL's that have been seen and at what depth
    seen = {seed_url: 0}
    # track how many URL's have been downloaded
    num_urls = {seed_url: 0}
    # rp = get_robots(seed_url)
    headers = headers
    D = Downloader(delay=5,
                   headers=headers,
                   cookies=cookies,
                   proxies=proxies,
                   num_retries=num_retries,
                   cache=cache)

    def process_queue():
        while True:
            try:
                url = crawl_queue.pop()
            except IndexError:
                break
            else:
                download_time = 1
                while download_time <= 5:
                    html = D(url)
                    download_time = download_time + 1
                    print('we download the url %s times' % download_time)
                    links = []
                    if scrape_callback:
                        links.extend(scrape_callback(url, html) or [])
                    next_urls = get_links(html)
                    if next_urls:
                        seen[seed_url] = seen[seed_url] + 1
                        break
                if link_regex:
                    links.extend(link for link in next_urls
                                 if re.match(link_regex, link))
                else:
                    links.extend(link for link in next_urls)
                for link in links:
                    if seen[seed_url] != max_depth:
                        link = normalize(seed_url, link)
                        # check whether already crawled this link
                        if link not in seen and same_domain(seed_url, link):
                            # check link is within same domain
                            # success! add this new link to queue
                            crawl_queue.append(link)

                # check whether have reached downloaded maximum
                try:
                    num_urls[url] += 1
                except KeyError:
                    num_urls[url] = 1
                if num_urls[url] == max_urls:
                    break

    threads = []
    while threads or crawl_queue:
        # the crawl is still activate
        for thread in threads:
            if not thread.is_alive():
                # remove the stopped threads
                threads.remove(thread)
        while len(threads) < max_threads and crawl_queue:
            # can start some more threads
            thread = threading.Thread(target=process_queue)
            # set daemon so main thread can exit when receives ctrl-c
            thread.setDaemon(True)
            thread.start()
            # thread.run()
            threads.append(thread)
Exemplo n.º 47
0
def scrapeAvtooglasi():
    # UTF-8 support
    reload(sys)
    sys.setdefaultencoding('utf-8')
    now = datetime.now()
    down = Downloader('http://www.avtooglasi.com.mk/rezultati/show/?vid=0&orderby=0')
    content = down.get_content()
    html = unicode(content)
    sliki = xpath.search(html, '//div[@class="resultLeft"]')
    ostanato = xpath.search(html, '//div[@class="oglasInfoTopContent"]')
    ceni = xpath.search(html, '//a[@class="btn btn-info btn-xs oglasInfoAdditionalPrice"]')
    
    link = {}
    title = {}
    imageUrl = {}
    description = {}
    category = {}
    subcategory = {}
    value = {}
    currency = {}
    region =  {}
    date = {}
    
    i = 0
    
    ads = []
    for slika in sliki:
        imageUrl[i] = xpath.search(slika, '//a[@class="thumbnail resultImg"]/img/@src')[0]
        i = i + 1
    
    i = 0
    
    for cena in ceni:
        price = xpath.get(cena,'//span/span').strip()
        price=price.split(" ")
        if len(price)>1:
            if price[0]=="По":
                price[0]="/"
            if price[1]=="договор":
                price[1]="/"
        
            value[i]=price[0]
            currency[i]=price[1]
            if currency[i]=="&euro;":
                currency[i]="EUR"  
        i = i + 1
        
    i = 0
    
    for advert in ostanato:
        link[i] = xpath.get(advert, '//a[@class="resultMainLink"]/@href')       
        title[i] = xpath.get(advert, '//a[@class="resultMainLink"]/span').strip().replace("\"", "")
        
        path = xpath.search(getDescription(link[i],'//div[@class="centerC"]'), '/div/div[@class="padded"]')
        description[i] = path[1]
        subcategory[i]="/"    
        category[i] = u"Возила"
        
        dodatok = xpath.get(advert, '//span[@class="oglasInfoAdditionalInfo"]')
        dodatok = dodatok.split(" | ")
        region[i] = dodatok[0]
        country = u"Македеонија"
        description[i] = dodatok[1] + u" година, "+ dodatok[2] +", "+ dodatok[3] +", "+ dodatok[4] +", "+ dodatok[5] +", "+ description[i]
        description[i] = description[i].strip().replace("\"", "")

        date[i]=""
        #print description[i]
        datum = dodatok[6].strip()
        datum = datum.split(" ")
        if datum[0]=="Денес":
            datum [0]= str(now.year)+"-"+str(now.month)+"-"+str(now.day)
            date[i]=datum[0]+" "+datum[2]
        elif datum[0]=="Вчера":
            da=datetime.now()-timedelta(days=1)
            datum[0]=str(da.year)+"-"+str(da.month)+"-"+str(da.day)
            date[i]=datum[0]+" "+datum[2]
        elif datum[0]=="пред":
            if datum[2]=="дена":
                da=datetime.now()-timedelta(days=int(datum[1]))
                datum[0]=str(da.year)+"-"+str(da.month)+"-"+str(da.day)
                date[i]=datum[0]
            else:
                if datum[1]=="1":
                    da=datetime.now()-timedelta(days=30)
                    datum[0]=str(da.year)+"-"+str(da.month)+"-"+str(da.day)
                    date[i]=datum[0]
                else:
                    da=datetime.now()-timedelta(days=60)
                    datum[0]=str(da.year)+"-"+str(da.month)+"-"+str(da.day)
                    date[i]=datum[0]
        else:
            date[i]=datum[0]+" "+datum[1]

        #print date[i]
        i = i + 1
        
    for i in link:
        ad = Ad(link[i], title[i], imageUrl[i], description[i], category[i], subcategory[i], value[i], currency[i], region[i], date[i], country)    
        ads.append(ad)
        
    return adsToJson(ads)

# print scrapeAvtooglasi()
Exemplo n.º 48
0
    def start(self):
        # Open the portolio
        self.portfolio = Portfolio(cash=0, sl=self.sl, tp=self.tp)
        self.portfolio.load(self.filename)
        # Download the stock data
        self.downloader = Downloader(cache=False)
        self.downloader.download(period=self.period)

        date = self.downloader.prices.index[-1]
        prices = self.downloader.prices

        print("#", date)

        # Refresh the portfolio with the latest prices
        self.portfolio.refresh(prices=prices, date=date)

        # Get the portfolio Summary
        portfolio_summary = self.portfolio.summary()

        # Get the opened positions
        opened = self.portfolio.holdings[self.portfolio.holdings['status'] ==
                                         'open']

        # Get the current trend
        zt = TradingZones(prices=prices)
        zones = zt.get()

        output = pd.DataFrame()

        # Activate the trailing stops
        targets = self.portfolio.getPositionsBelowTS(self.ts)
        if len(targets) > 0:
            sell_orders = self.portfolio.getSellOrdersFromSubset(
                subset=targets)
            print("\n----- TS -----\n", sell_orders)
            #self.portfolio.sell(sell_orders, prices, date, label='TS')

        # Activate the stop loss
        if self.sl < 0:
            poor_perfs = opened[opened['profits_pct'] <= self.sl]
            if len(poor_perfs) > 0:
                sell_list = poor_perfs['symbol'].unique()
                sell_orders = self.portfolio.getSellAllOrders(
                    symbols=sell_list)
                print("\n----- SL -----\n", sell_orders)
                #self.portfolio.sell(sell_orders, prices, date, label='SL')

        # Sell all on drop
        if self.sellAllOnCrash == True and zones.iloc[-1][
                'signal'] == 'DOWN' and len(opened) > 0:
            print("Sell all")
            print("\n----- Sell All -----\n")
            #self.portfolio.sellAll(prices, date, label='Market Drop')

        # Get the ones that reached take-profit
        if self.tp > 0:
            good_perfs = opened[opened['profits_pct'] >= self.tp]
            if len(good_perfs) > 0:
                sell_list = good_perfs['symbol'].unique()
                sell_orders = self.portfolio.getSellAllOrders(
                    symbols=sell_list)
                print("\n----- TP -----\n", sell_orders)
                #self.portfolio.sell(sell_orders, prices, date, label='TP')

        # Avoid trading in a general market downtrend
        if self.avoidDowntrends == False or (self.avoidDowntrends == True and
                                             zones.iloc[-1]['signal'] == 'UP'):
            # Pick the stocks
            picker = Picker(prices=prices)
            picker.pick(count=self.stockPicks)
            #print(">", picker.selected)
            # Get the order size
            portfolio_summary = self.portfolio.summary()
            order = Order(prices=prices, picks=picker.selected)
            order.get(budget=portfolio_summary['cash'])
            #print("!", portfolio_summary['cash'], order.orders)
            print("\n----- BUY -----\n", order.orders)
            # Update our portfolio
            #self.portfolio.update(orders=order.orders, prices=prices, date=date)
        else:
            # No new positions in a downtrend!
            # Update our portfolio
            self.portfolio.update(orders=pd.DataFrame(),
                                  prices=prices,
                                  date=date)
def threaded_crawler(seed_url,
                     delay=5,
                     cache=None,
                     scrape_callback=None,
                     proxies=None,
                     num_retries=1,
                     max_threads=10,
                     timeout=60):
    """Crawl this website in multiple threads
        seed_url must be list
    """
    # the queue of URL's that still need to be crawled
    crawl_queue = deque(seed_url)
    # the URL's that have been seen
    seen = dict((url, 0) for url in seed_url)
    D = Downloader(cache=cache,
                   delay=delay,
                   proxies=proxies,
                   num_retries=num_retries)

    def process_queue():
        while True:
            try:
                url = crawl_queue.pop()
            except IndexError:
                # crawl queue is empty
                break
            else:
                html = D(url)
                if scrape_callback:
                    try:
                        links = scrape_callback(url, html) or []
                    except Exception as e:
                        print('Error in callback for: {}: {}'.format(url, e))
                    else:
                        for link in links:
                            link = normalize(seed_url, link)
                            # check whether already crawled this link
                            if link not in seen:
                                # add this new link to queue
                                crawl_queue.appendleft(link)

    # wait for all download threads to finish
    threads = []
    while threads or crawl_queue:
        # the crawl is still active
        for thread in threads:
            if not thread.is_alive():
                # remove the stopped threads
                threads.remove(thread)
        while len(threads) < max_threads and crawl_queue:
            # can start some more threads
            thread = threading.Thread(target=process_queue)
            thread.setDaemon(
                True
            )  # set daemon so main thread can exit when receives ctrl-c
            thread.start()
            threads.append(thread)
        # all threads have been processed
        # sleep temporarily so CPU can focus execution on other threads
        time.sleep(SLEEP_TIME)
Exemplo n.º 50
0
from Downloader import Downloader

url_metadata = "https://os.unil.cloud.switch.ch/fma/fma_metadata.zip"
url_small = "https://os.unil.cloud.switch.ch/fma/fma_small.zip"
url_medium = "https://os.unil.cloud.switch.ch/fma/fma_medium.zip"
url_large = "https://os.unil.cloud.switch.ch/fma/fma_large.zip"
url_full = "https://os.unil.cloud.switch.ch/fma/fma_full.zip"

sha1sum_metadata = 'f0df49ffe5f2a6008d7dc83c6915b31835dfe733'
sha1sum_small = 'ade154f733639d52e35e32f5593efe5be76c6d70'
sha1sum_medium = 'c67b69ea232021025fca9231fc1c7c1a063ab50b'
sha1sum_large = '497109f4dd721066b5ce5e5f250ec604dc78939e'
sha1sum_full = '0f0ace23fbe9ba30ecb7e95f763e435ea802b8ab'

threads = 10

code_geass = 'https://ca-east.uploadfiles.io/get/6tnazj0v'

obj = Downloader(url=code_geass,
                 number_of_threads=threads,
                 destination_directory='D:\\Downloads')

obj.start_download()
# obj.merge_chunks()
print(obj.get_metadata())
print(obj.get_remote_crc32c())
print(obj.get_downloaded_crc32c_large())
print(obj.get_downloaded_sha1_large())
Exemplo n.º 51
0
class Sim:
    def __init__(self,
                 period='2y',
                 timedelay=100,
                 window=100,
                 timestep=5,
                 budget=5000,
                 stockPicks=10,
                 sl=-0.1,
                 tp=0.25,
                 ts=0.2,
                 avoidDowntrends=True,
                 sellAllOnCrash=True,
                 neptune=None,
                 pick_kwargs={}):
        self.neptune = neptune  # neptune.ai logging
        self.period = period
        self.timedelay = timedelay
        self.timestep = timestep
        self.budget = budget
        self.stockPicks = stockPicks
        self.sl = sl
        self.tp = tp
        self.ts = ts
        self.avoidDowntrends = avoidDowntrends
        self.sellAllOnCrash = sellAllOnCrash
        self.pick_kwargs = pick_kwargs
        self.portfolio = Portfolio(cash=budget, sl=self.sl, tp=self.tp)
        self.downloader = Downloader()
        self.downloader.download(period=self.period)
        self.current_index = timedelay
        self.is_notebook = self.isNotebook()

    def isNotebook(self):
        try:
            cfg = get_ipython().config
            return True
        except NameError:
            return False

    def progress(self, count, total=100, status=''):
        if self.is_notebook == True:
            return HTML("""
	        <progress
	            value='{value}'
	            max='{max}',
	            style='width: 50%'
	        >
	            {value}
	        </progress>
	    """.format(value=count, max=total))
        else:
            bar_len = 60
            filled_len = int(round(bar_len * count / float(total)))

            percents = round(100.0 * count / float(total), 1)
            bar = '=' * filled_len + '-' * (bar_len - filled_len)

            sys.stdout.write('[%s] %s%s\r' % (bar, percents, '%'))
            sys.stdout.flush()

    def run(self):
        #out = display(self.progress(0, 100), display_id=True)
        steps = (len(self.downloader.prices) - self.timedelay)
        stepN = 0
        self.stats = pd.DataFrame(columns=[
            'invested', 'portfolio_value', 'Positions', 'closed', 'opened',
            'cash', 'total_value', 'portfolio_resistances',
            'portfolio_supports', 'drawdown', 'drawdown_pct'
        ])
        while self.current_index + self.timestep < len(self.downloader.prices):
            output = self.tick()
            # Neptune logging
            if self.neptune is not None:
                for k in list(output.keys()):
                    self.neptune.log_metric(k, output[k])
            output['date'] = self.downloader.prices.index[self.current_index]
            self.stats = self.stats.append(output, ignore_index=True)

            stepN = stepN + 1
            pct = math.ceil((stepN / steps) * 100)
            if self.is_notebook == True:
                out.update(self.progress(pct, 100))
            else:
                self.progress(pct, 100)
            #print(str(round(pct))+"%")
        self.portfolio.sellAll(
            prices=self.downloader.prices[:self.current_index],
            date=self.downloader.prices.index[self.current_index],
            label='End Sim')
        self.stats = self.stats.set_index('date')
        return self.stats

    # Run every night after the market close
    def tick(self):
        #print('\n\n----------- tick -----------')
        # Get the latest prices & date as of today
        date = self.downloader.prices.index[self.current_index]
        prices = self.downloader.prices[:self.current_index +
                                        1]  # Include the current price

        #print("#", date)

        # Refresh the portfolio with the latest prices
        self.portfolio.refresh(prices=prices, date=date)

        # Get the portfolio Summary
        portfolio_summary = self.portfolio.summary()

        # Get the opened positions
        opened = self.portfolio.holdings[self.portfolio.holdings['status'] ==
                                         'open']

        # Get the current trend
        zt = TradingZones(prices=prices)
        zones = zt.get()

        # Activate the trailing stops
        targets = self.portfolio.getPositionsBelowTS(self.ts)
        if len(targets) > 0:
            sell_orders = self.portfolio.getSellOrdersFromSubset(
                subset=targets)
            #print("\n-----------\n",targets,"\n\n",sell_orders)
            self.portfolio.sell(sell_orders, prices, date, label='TS')

        # Activate the stop loss
        if self.sl < 0:
            poor_perfs = opened[opened['profits_pct'] <= self.sl]
            if len(poor_perfs) > 0:
                sell_list = poor_perfs['symbol'].unique()
                sell_orders = self.portfolio.getSellAllOrders(
                    symbols=sell_list)
                self.portfolio.sell(sell_orders, prices, date, label='SL')

        # Sell all on drop
        if self.sellAllOnCrash == True and zones.iloc[-1][
                'signal'] == 'DOWN' and len(opened) > 0:
            print("Sell all")
            self.portfolio.sellAll(prices, date, label='Market Drop')

        # Get the ones that reached take-profit
        if self.tp > 0:
            good_perfs = opened[opened['profits_pct'] >= self.tp]
            if len(good_perfs) > 0:
                sell_list = good_perfs['symbol'].unique()
                sell_orders = self.portfolio.getSellAllOrders(
                    symbols=sell_list)
                self.portfolio.sell(sell_orders, prices, date, label='TP')

        # Avoid trading in a general market downtrend
        if self.avoidDowntrends == False or (self.avoidDowntrends == True and
                                             zones.iloc[-1]['signal'] == 'UP'):
            # Pick the stocks
            picker = Picker(prices=prices)
            picker.pick(count=self.stockPicks, kwargs=self.pick_kwargs)
            #print(">", picker.selected)
            # Get the order size
            portfolio_summary = self.portfolio.summary()
            order = Order(prices=prices, picks=picker.selected)
            order.get(budget=portfolio_summary['cash'])
            #print("!", portfolio_summary['cash'], order.orders)
            # Update our portfolio
            self.portfolio.update(
                orders=order.orders,
                prices=prices,
                date=self.downloader.prices.index[self.current_index])
        else:
            # No new positions in a downtrend!
            # Update our portfolio
            self.portfolio.update(
                orders=pd.DataFrame(),
                prices=prices,
                date=self.downloader.prices.index[self.current_index])

        # Update the index
        if self.current_index + self.timestep < len(self.downloader.prices):
            self.current_index = self.current_index + self.timestep
        portfolio_summary = self.portfolio.summary()
        return portfolio_summary
Exemplo n.º 52
0
from Downloader import Downloader
from Builder import Builder
from Utils import loadconfig
from Controller import Controller

if __name__ == '__main__':
    # -----------------load config--------------------
    configpath = "Config/config.json"
    configs = loadconfig(configpath)
    dir_config = configs["Dir"]
    file_config = configs["File"]
    # -----------------create classes--------------
    builder = Builder(configpath)
    downloader = Downloader(dir_config["Save_Path"])
    controller = Controller(file_config["MusicList"])
    # ------------------------------------------
    controller = controller.init(downloader, builder)
    controller.AutoDownload()
Exemplo n.º 53
0
def link_crawler(seed_url,
                 link_regex=".*",
                 resource_regex=".*",
                 delay=5,
                 max_depth=-1,
                 max_urls=-1,
                 headers=None,
                 user_agent="wswp",
                 proxies=None,
                 num_retries=1,
                 download_source_callback=None,
                 cache=None,
                 data=None):
    '''
    
    '''
    #用于存储未被下载的页面
    craw_queue = [seed_url]
    #用于存储已经下载过的页面
    seen = [seed_url]
    #下载过的url数量
    num_urls = 0
    isdow = []
    downloader = Downloader(delay=delay,
                            user_agent=user_agent,
                            proxies=proxies,
                            num_retries=num_retries,
                            cache=cache)

    while craw_queue:
        url = craw_queue.pop()
        #depth = seen[]
        html = downloader(url)
        print(html)
        links = []
        isdow.append(url)
        seen.append(url)
        if download_source_callback and re.match(resource_regex, url):
            #如果是资源页面,下载页面上的所有资源
            print("属于资源,开始下载")
            download_source_callback(url, html)

        #if depth != max_depth:
        links.extend(
            [link for link in get_links(html) if re.search(link_regex, link)])
        for link in links:
            link = normalize(seed_url, link)
            if link not in seen:
                seen.append(link)
                if same_domain(seed_url, link):
                    craw_queue.append(link)
                    print(link + "符合要求加入队列,现在队列的长度是" + str(len(craw_queue)))
                    #craw_queue.append(link)
                    #print(link + "符合要求加入队列,现在队列的长度是" + str(len(craw_queue)))
                    print("当前可怕去的url数量胃" + str(len(craw_queue)))
                    print("已经发现的总url数量为", str(len(set(seen))))
                    print("已经下载过的url数量为", str(len(isdow)))
        num_urls += 1
        if num_urls == max_urls:
            break
    print(seed_url + "上所有符合要求的网站下载完毕,一共遍历了" + str(len(seen)) + "个网页")
Exemplo n.º 54
0
	def __init__( self ):
		# Appel au constructeur de la classe mere
		QtGui.QMainWindow.__init__( self )
		
		###########
		# Fenetre #
		###########
		
		###
		# Reglages de la fenetre principale
		###
		
		# Nom de la fenetre
		self.setWindowTitle( "TVDownloader" )
		# Mise en place de son icone
		self.setWindowIcon( QtGui.QIcon( "ico/TVDownloader.png" ) )
		
		###
		# Mise en place des widgets dans la fenetre
		###
		
		# Widget central qui contiendra tout
		self.centralWidget = QtGui.QWidget( self )
		
		#
		# Barre du haut
		#
		
		# Layout horizontal qui contiendra les listes deroulantes
		self.horizontalLayoutBarreHaut = QtGui.QHBoxLayout()
		
		# Liste deroulante pour choisir le site (plugin)
		self.comboBoxSite = QtGui.QComboBox( self.centralWidget )
		self.horizontalLayoutBarreHaut.addWidget( self.comboBoxSite )
		
		# Liste deroulante pour choisir une chaine du site courant
		self.comboBoxChaine = QtGui.QComboBox( self.centralWidget)
		self.horizontalLayoutBarreHaut.addWidget( self.comboBoxChaine )
		
		# Liste deroulante pour choisir une emission de la chaine courante
		self.comboBoxEmission = QtGui.QComboBox( self.centralWidget )
		self.horizontalLayoutBarreHaut.addWidget( self.comboBoxEmission )
		
		#
		# Onglets
		#
		
		# Gestionnaire onglets
		self.tabWidget = QtGui.QTabWidget( self.centralWidget )
		
		# Onglet Fichiers
		self.tabFichiers = QtGui.QSplitter( self.centralWidget ) # L'onglet Fichier contient un splitter
		self.tabWidget.addTab( self.tabFichiers, u"Choix des fichiers" )
		
		# Onglet Telechargements
		self.tabTelechargements = QtGui.QWidget( self.centralWidget )
		self.tabWidget.addTab( self.tabTelechargements, u"Téléchargements" )
		
		#
		# Liste des fichiers
		#
		
		# Layout de grille qui contient le tableau qui liste les fichiers + boutons
		self.gridLayoutFichiers = QtGui.QGridLayout( self.tabFichiers )
		
		# Tableau qui contient la liste des fichiers disponibles pour l'emission courante
		self.tableWidgetFichier = MyQTableWidget( self.tabFichiers )
		# Il a 4 colonnes et 0 ligne (pour l'instant)
		self.tableWidgetFichier.setColumnCount( 3 )
		self.tableWidgetFichier.setRowCount( 0 )
		# On ajoute les titres
		self.tableWidgetFichier.setHorizontalHeaderItem( 0,
														 self.tableWidgetFichier.creerItem( "" ) )
		self.tableWidgetFichier.setHorizontalHeaderItem( 1,
														 self.tableWidgetFichier.creerItem( "Date" ) )
		self.tableWidgetFichier.setHorizontalHeaderItem( 2,
														 self.tableWidgetFichier.creerItem( "Emission" ) )
		# On l'ajoute au layout
		self.gridLayoutFichiers.addWidget( self.tableWidgetFichier, 0, 1, 6, 1 )
		
		# Icones du tableWidget
		self.iconeFichier     = QtGui.QIcon( "ico/gtk-file.svg" )
		self.iconeAjoute      = QtGui.QIcon( "ico/gtk-add.svg" )
		self.iconeTelecharge  = QtGui.QIcon( "ico/gtk-apply.svg" )
		
		# Bouton pour ajouter tous les fichiers a la liste des telechargements
		self.pushButtonToutAjouter = MyQPushButton( self.tabFichiers )
		self.pushButtonToutAjouter.setIcon( QtGui.QIcon( "ico/gtk-add.svg" ) )
		self.pushButtonToutAjouter.setToolTip( u"Ajouter tous les fichiers à la liste des téléchargements" )
		self.gridLayoutFichiers.addWidget( self.pushButtonToutAjouter, 0, 0, 2, 1 )
		
		# Bouton pour rafraichir le plugin courant
		self.pushButtonRafraichirPlugin = MyQPushButton( self.tabFichiers )
		self.pushButtonRafraichirPlugin.setIcon( QtGui.QIcon( "ico/gtk-refresh.svg" ) )
		self.pushButtonRafraichirPlugin.setToolTip( "Rafraichir le plugin" )
		self.gridLayoutFichiers.addWidget( self.pushButtonRafraichirPlugin, 2, 0, 2, 1 )

		# Bouton pour ouvrir la fenetre des preferences du plugin courant
		self.pushButtonPreferencesPlugin = MyQPushButton( self.tabFichiers )
		self.pushButtonPreferencesPlugin.setIcon( QtGui.QIcon( "ico/gtk-preferences.svg" ) )
		self.pushButtonPreferencesPlugin.setToolTip( u"Ouvrir les préférences du plugin" )
		self.gridLayoutFichiers.addWidget( self.pushButtonPreferencesPlugin, 4, 0, 2, 1 )
		
		# On met en place ce layout sur un widget (pour le splitter)
		self.widgetFichiers = QtGui.QWidget()
		self.widgetFichiers.setLayout( self.gridLayoutFichiers )
		
		#
		# Descriptif des fichiers
		#
		
		# Layout de grille
		self.gridLayoutDescriptif = QtGui.QGridLayout()
		
		# Label pour afficher un logo
		self.logoFichierDefaut = QtGui.QPixmap()
		self.logoFichierDefaut.load( "img/gtk-dialog-question.svg" )
		
		self.labelLogo = QtGui.QLabel( self.centralWidget )
		self.labelLogo.setPixmap( self.logoFichierDefaut.scaled( QtCore.QSize( 150, 150 ), QtCore.Qt.KeepAspectRatio ) )
		
		self.gridLayoutDescriptif.addWidget( self.labelLogo, 0, 0, 1, 1 )
		
		# Zone de texte pour afficher un descriptif
		self.plainTextEdit = QtGui.QPlainTextEdit( self.centralWidget )
		self.gridLayoutDescriptif.addWidget( self.plainTextEdit, 0, 1, 1, 2 )
		
		# On met en place ce layout sur un widget (pour le splitter)
		self.widgetDescriptif = QtGui.QWidget()
		self.widgetDescriptif.setLayout( self.gridLayoutDescriptif )
		

		# Onrientation verticale du splitter
		self.tabFichiers.setOrientation( QtCore.Qt.Vertical )
		
		# On ajoute les 2 elements au splitter (qui est notre onglet)
		self.tabFichiers.addWidget( self.widgetFichiers )
		self.tabFichiers.addWidget( self.widgetDescriptif )
		
		#
		# Liste des telechargements
		#
		
		# Layout de grille qui contient le tableau qui liste les fichiers a telecharger + les boutons pour le controller
		self.gridLayoutTelechargement = QtGui.QGridLayout( self.tabTelechargements )
		
		# Tableau qui contient la liste des fichiers a telecharger
		self.tableWidgetTelechargement = MyQTableWidget( self.tabTelechargements )
		# Il a 5 colonnes et 0 ligne (pour l'instant)
		self.tableWidgetTelechargement.setColumnCount( 3 )
		self.tableWidgetTelechargement.setRowCount( 0 )
		# On ajoute le titre des 5 colonnes
		self.tableWidgetTelechargement.setHorizontalHeaderItem( 0,
																self.tableWidgetTelechargement.creerItem( "Date" ) )
		self.tableWidgetTelechargement.setHorizontalHeaderItem( 1,
																self.tableWidgetTelechargement.creerItem( "Emission" ) )
		self.tableWidgetTelechargement.setHorizontalHeaderItem( 2,
																self.tableWidgetTelechargement.creerItem( "Etat" ) )
		# On l'ajoute au layout
		self.gridLayoutTelechargement.addWidget( self.tableWidgetTelechargement, 0, 1, 4, 1 )
		
		# Bouton pour monter l'element selectionne tout en haut de la liste
		self.pushButtonExtremiteMonter = MyQPushButton( self.tabTelechargements )
		self.pushButtonExtremiteMonter.setIcon( QtGui.QIcon( "ico/gtk-jump-to-rtl.svg" ) )
		self.pushButtonExtremiteMonter.setToolTip( u"Placer l'élément sélectionné tout en haut" )
		self.gridLayoutTelechargement.addWidget( self.pushButtonExtremiteMonter, 0, 0, 1, 1 )
		
		# Bouton pour monter l'element selectionne d'un cran dans la liste
		self.pushButtonMonter = MyQPushButton( self.tabTelechargements )
		self.pushButtonMonter.setIcon( QtGui.QIcon( "ico/gtk-go-up.svg" ) )
		self.pushButtonMonter.setToolTip( u"Monter l'élément sélectionné" )
		self.gridLayoutTelechargement.addWidget( self.pushButtonMonter, 1, 0, 1, 1 )

		# Bouton pour descendre l'element selectionne d'un cran dans la liste
		self.pushButtonDescendre = MyQPushButton( self.tabTelechargements )
		self.pushButtonDescendre.setIcon( QtGui.QIcon( "ico/gtk-go-down.svg" ) )
		self.pushButtonDescendre.setToolTip( u"Descendre l'élément selectionné" )
		self.gridLayoutTelechargement.addWidget( self.pushButtonDescendre, 2, 0, 1, 1 )
		
		# Bouton pour descendre l'element selectionne tout en bas de la liste
		self.pushButtonExtremiteDescendre = MyQPushButton( self.tabTelechargements )
		self.pushButtonExtremiteDescendre.setIcon( QtGui.QIcon( "ico/gtk-jump-to-ltr.svg" ) )
		self.pushButtonExtremiteDescendre.setToolTip( u"Placer l'élément sélectionné tout en bas" )
		self.gridLayoutTelechargement.addWidget( self.pushButtonExtremiteDescendre, 3, 0, 1, 1 )
		
		# Bouton pour supprimer tous les elements de la liste
		self.pushButtonToutSupprimer = MyQPushButton( self.tabTelechargements )
		self.pushButtonToutSupprimer.setIcon( QtGui.QIcon( "ico/gtk-cancel.svg" ) )
		self.pushButtonToutSupprimer.setToolTip( u"Supprimer tous les téléchargements" )
		self.gridLayoutTelechargement.addWidget( self.pushButtonToutSupprimer, 0, 2, 1, 1 )
		
		# Bouton pour supprimer de la liste les telechargements termines
		self.pushButtonNettoyer = MyQPushButton( self.tabTelechargements )
		self.pushButtonNettoyer.setIcon( QtGui.QIcon( "ico/gtk-delete-full.svg" ) )
		self.pushButtonNettoyer.setToolTip( u"Supprimer les téléchargement terminés" )
		self.gridLayoutTelechargement.addWidget( self.pushButtonNettoyer, 1, 2, 1, 1 )
		
		# Bouton pour ouvrir le dossier des telechargements
		self.pushButtonOuvrirDossierTelechargement = MyQPushButton( self.tabTelechargements )
		self.pushButtonOuvrirDossierTelechargement.setIcon( QtGui.QIcon( "ico/gtk-folder.svg" ) )
		self.pushButtonOuvrirDossierTelechargement.setToolTip( u"Ouvrir le dossier des téléchargements" )
		self.gridLayoutTelechargement.addWidget( self.pushButtonOuvrirDossierTelechargement, 2, 2, 1, 1 )
		
		#
		# Barre progression de telechargement d'un fichier
		#
		self.progressBarTelechargementFichier = QtGui.QProgressBar( self.centralWidget )
		self.progressBarTelechargementFichier.setProperty( "value", 0 )
		
		#
		# Barre de progression de telechargement des fichiers
		#
		self.progressBarTelechargement = QtGui.QProgressBar( self.centralWidget )
		self.progressBarTelechargement.setProperty( "value", 0 )
		
		#
		# Boutons du bas pour gerer ajouter/supprimer/lancer telechargements
		#
		
		# Layout horizontal qui contiendra les boutons
		self.horizontalLayoutBarreBas = QtGui.QHBoxLayout()
		
		# Bouton pour lancer les telechargements
		self.pushButtonLancer = QtGui.QPushButton( QtGui.QIcon( "ico/gtk-media-play-ltr.svg" ), u"Lancer téléchargement", self.centralWidget )
		self.horizontalLayoutBarreBas.addWidget( self.pushButtonLancer )

		# Bouton pour stopper les telechargements
		self.pushButtonStop = QtGui.QPushButton( QtGui.QIcon( "ico/gtk-media-stop.svg" ), u"Stopper le téléchargement", self.centralWidget )
		self.pushButtonStop.setEnabled( False )
		self.horizontalLayoutBarreBas.addWidget( self.pushButtonStop )	
	
		###
		# Positionnement des differents widgets/layouts sur le layout de grille
		###
		
		# Layout de grille dans lequel on va placer nos widgets/layouts
		self.gridLayout = QtGui.QGridLayout( self.centralWidget )
		# On ajoute la barre du haut
		self.gridLayout.addLayout( self.horizontalLayoutBarreHaut, 0, 0, 1, 3 )
		# On ajoute le gestionnaire d'onglets
		self.gridLayout.addWidget( self.tabWidget, 1, 0, 1, 3 )
		# On ajoute la barre de progression de telechargement d'un fichier
		self.gridLayout.addWidget( self.progressBarTelechargementFichier, 2, 0, 1, 3 )
		# On ajoute la barre de progression de telechargement des fichiers
		self.gridLayout.addWidget( self.progressBarTelechargement, 3, 0, 1, 3 )
		# On ajoute les boutons ajouter/supprimer/lancer
		self.gridLayout.addLayout( self.horizontalLayoutBarreBas, 4, 0, 1, 3 )
		
		###
		# Mise en place le central widget dans la fenetre
		###
		self.setCentralWidget( self.centralWidget )

		###
		# Mise en place du menu
		###
		
		# Menu barre
		self.menubar = QtGui.QMenuBar( self )
		self.menubar.setGeometry( QtCore.QRect( 0, 0, 480, 25 ) )
		
		# Menu Fichier
		self.menuFichier = QtGui.QMenu( "&Fichier", self.menubar )
		self.menubar.addAction( self.menuFichier.menuAction() )
		
		# Action Fichier -> Quitter
		self.actionQuitter = QtGui.QAction( QtGui.QIcon( "ico/gtk-quit.svg" ), "&Quitter", self )
		self.actionQuitter.setIconVisibleInMenu( True )
		self.menuFichier.addAction( self.actionQuitter )
		
		# Menu Edition
		self.menuEdition = QtGui.QMenu( "&Edition", self.menubar )
		self.menubar.addAction( self.menuEdition.menuAction() )
		
		# Action Edition -> Mise a jour
		self.actionMAJ = QtGui.QAction( QtGui.QIcon( "ico/gtk-refresh.svg" ), u"&Mise à jour des plugins", self )
		self.actionMAJ.setIconVisibleInMenu( True )
		self.menuEdition.addAction( self.actionMAJ )
		
		# Action Edition -> Preferences
		self.actionPreferences = QtGui.QAction( QtGui.QIcon( "ico/gtk-preferences.svg" ), u"&Préférences", self )
		self.actionPreferences.setIconVisibleInMenu( True )
		self.menuEdition.addAction( self.actionPreferences )
		
		# Menu Aide
		self.menuAide = QtGui.QMenu( "&Aide", self.menubar )
		self.menubar.addAction( self.menuAide.menuAction() )
		
		# Action Aide -> A propos
		self.actionAPropos = QtGui.QAction( QtGui.QIcon( "ico/gtk-about.svg" ), u"À p&ropos", self )
		self.actionAPropos.setIconVisibleInMenu( True )
		self.menuAide.addAction( self.actionAPropos )
		
		# Ajout du menu a l'interface
		self.setMenuBar( self.menubar )

		###
		# Signaux provenants de l'interface
		###

		QtCore.QObject.connect( self.tableWidgetFichier,
								QtCore.SIGNAL( "cellClicked(int,int)" ),
								self.afficherInformationsFichier )	
		
		QtCore.QObject.connect( self.tableWidgetFichier,
								QtCore.SIGNAL( "cellDoubleClicked(int,int)" ),
								self.gererTelechargement )	
								
		QtCore.QObject.connect( self.pushButtonToutAjouter,
								QtCore.SIGNAL( "clicked()" ),
								self.ajouterTousLesFichiers )								

		QtCore.QObject.connect( self.pushButtonRafraichirPlugin,
								QtCore.SIGNAL( "clicked()" ),
								self.rafraichirPlugin )

		QtCore.QObject.connect( self.tableWidgetTelechargement,
								QtCore.SIGNAL( "cellDoubleClicked(int,int)" ),
								self.supprimerTelechargement )	
		
		QtCore.QObject.connect( self.pushButtonExtremiteMonter,
								QtCore.SIGNAL( "clicked()" ),
								lambda versLeHaut = True, extremite = True : self.tableWidgetTelechargement.deplacerLigne( versLeHaut, extremite ) )
		
		QtCore.QObject.connect( self.pushButtonMonter,
								QtCore.SIGNAL( "clicked()" ),
								lambda versLeHaut = True, extremite = False : self.tableWidgetTelechargement.deplacerLigne( versLeHaut, extremite ) )
		
		QtCore.QObject.connect( self.pushButtonDescendre,
								QtCore.SIGNAL( "clicked()" ),
								lambda versLeHaut = False, extremite = False : self.tableWidgetTelechargement.deplacerLigne( versLeHaut, extremite ) )
		
		QtCore.QObject.connect( self.pushButtonExtremiteDescendre,
								QtCore.SIGNAL( "clicked()" ),
								lambda versLeHaut = False, extremite = True : self.tableWidgetTelechargement.deplacerLigne( versLeHaut, extremite ) )
		
		QtCore.QObject.connect( self.pushButtonToutSupprimer,
								QtCore.SIGNAL( "clicked()" ),
								self.supprimerTousLesTelechargements )
		
		QtCore.QObject.connect( self.pushButtonNettoyer,
								QtCore.SIGNAL( "clicked()" ),
								self.nettoyer )
		
		QtCore.QObject.connect( self.pushButtonLancer,
								QtCore.SIGNAL( "clicked()" ),
								self.lancerTelechargement )

		QtCore.QObject.connect( self.pushButtonStop,
								QtCore.SIGNAL( "clicked()" ),
								self.stopperTelechargement )
		
		QtCore.QObject.connect( self.actionQuitter,
								QtCore.SIGNAL( "triggered()" ),
								self.close )						
	
		################################################
		# Instanciations + initialisation de variables #
		################################################
		
		# Fenetre About
		self.aProposDialog = None
		# Fenetre des preferences du logiciel
		self.preferencesDialog = None
		# Fenetre de mise a jour des plugins
		self.updateManagerDialog = None
		# Nom plugin courant
		self.nomPluginCourant = ""
		# Liste des fichiers
		self.listeFichiers = []
		# Liste des fichiers a telecharger
		self.listeFichiersATelecharger = []
		# Cache des images descriptive
		# Clef : urlImage Valeur : image (binaire)
		self.cacheImage = {}
		
		# On intancie le lanceur de signaux
		self.signaux = Signaux()
		# On instancie le gestionnaire de preferences
		self.preferences = Preferences()
		# On instancie le gestionnaire de preferences des plugins
		self.preferencesPluginDialog = PreferencePluginDialog( self )
		# On instancie le gestionnaire de download
		self.downloader = Downloader( self.signaux )	
		# On recupere l'instance de API
		self.api = API.getInstance()
		# On instancie le gestionnaire d'historique
		self.historique = Historique()
		# On instancie la fenetre d'attente
		self.fenetreAttenteProgressDialog = FenetreAttenteProgressDialog( self )
		# On instancie le gest								 
		
		#
		# Fenetre de confirmation pour quitter le logiciel
		#
		self.quitterMessageBox = QtGui.QMessageBox( self )
		self.quitterMessageBox.setWindowTitle( "Fermeture de TVDownloader" )
		self.quitterMessageBox.setText( u"Voulez-vous réellement quitter TVDownloader ?" )
		self.quitterMessageBox.setInformativeText( u"Votre liste de téléchargement sera perdue" )
		self.quitterMessageBox.addButton( "Oui", QtGui.QMessageBox.AcceptRole )
		self.quitterMessageBox.addButton( "Non", QtGui.QMessageBox.RejectRole )
		
		############################################################
		# On connecte les signaux des instances precedements crees #
		############################################################	

		QtCore.QObject.connect( self.pushButtonOuvrirDossierTelechargement,
								QtCore.SIGNAL( "clicked()" ),
								self.ouvrirRepertoireTelechargement )
		
		QtCore.QObject.connect( self.comboBoxSite,
								QtCore.SIGNAL( "activated(QString)" ),
								self.listerChaines )
		
		QtCore.QObject.connect( self.comboBoxChaine,
								QtCore.SIGNAL( "activated(QString)" ),
								self.listerEmissions )
		
		QtCore.QObject.connect( self.comboBoxEmission,
								QtCore.SIGNAL( "activated(QString)" ),
								self.listerFichiers )

		QtCore.QObject.connect( self.pushButtonPreferencesPlugin,
								QtCore.SIGNAL( "clicked()" ),
								self.ouvrirPreferencesPlugin )

		QtCore.QObject.connect( self.actionPreferences,
								QtCore.SIGNAL( "triggered()" ),
								self.ouvrirPreferencesLogiciel )	
								
		QtCore.QObject.connect( self.actionMAJ,
								QtCore.SIGNAL( "triggered()" ),
								self.ouvrirFenetreMiseAJour )	

		QtCore.QObject.connect( self.actionAPropos,
								QtCore.SIGNAL( "triggered()" ),
								self.ouvrirFenetreAPropos )
		
		QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "debutActualisation(PyQt_PyObject)" ) , self.fenetreAttenteProgressDialog.ouvrirFenetreAttente )
		QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "finActualisation()" ) , self.fenetreAttenteProgressDialog.fermerFenetreAttente )		
		QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "actualiserListesDeroulantes()" ) , self.actualiserListesDeroulantes )
		QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "listeChaines(PyQt_PyObject)" ) , self.ajouterChaines )
		QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "listeEmissions(PyQt_PyObject)" ) , self.ajouterEmissions )
		QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "listeFichiers(PyQt_PyObject)" ) , self.ajouterFichiers )
		QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "nouvelleImage(PyQt_PyObject)" ) , self.mettreEnPlaceImage )
		QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "debutTelechargement(int)" ) , self.debutTelechargement )
		QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "finTelechargement(int)" ) , self.finTelechargement )
		QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "finDesTelechargements()" ) , self.activerDesactiverInterface )
		QtCore.QObject.connect( self.signaux, QtCore.SIGNAL( "pourcentageFichier(int)" ) , self.progressBarTelechargementFichier.setValue )		

		#########
		# Début #
		#########			
		
		# La fenetre prend la dimension qu'elle avait a sa fermeture
		taille = self.preferences.getPreference( "tailleFenetre" )
		self.resize( taille[ 0 ], taille[ 1 ] )
		
		# Si aucun plugin n'est active, on ouvre la fenetre des preferences
		if( len( self.preferences.getPreference( "pluginsActifs" ) ) == 0 ):
			self.ouvrirPreferencesLogiciel()
		
		# On actualise tous les plugins
		self.rafraichirTousLesPlugins()
Exemplo n.º 55
0
def main(filenameList):
    lock = _threading.Lock()  # 获取线程锁,预防多个listener竞争lftp的使用
    ftp = FTPFactory().GetFTP()
    downloader = Downloader(ftp, lock, *filenameList)

    downloader.Run()
Exemplo n.º 56
0
YOUTUBE_VID_DIR = 'Youtube Videos' # Parent directory for all Yt videos !
current_user = os.getlogin()

if system()=='Windows':
    DOWNLOADS = f'C:/Users/{current_user}/Downloads' # get the currently logged in user's download dir !
elif system()=='Linux':
    DOWNLOADS = f'/home/{current_user}/Downloads'
else:
    raise Exception("Not Implemented For Current Platform !")
FINAL_PATH = os.path.join(DOWNLOADS, YOUTUBE_VID_DIR) # joining the downloads and the new vids directory !

if not os.path.isdir(FINAL_PATH):
    os.mkdir(FINAL_PATH)
    print(f'New Download Directory Created At {FINAL_PATH}')

download = Downloader()

while True:

    option = int(input('1)Video\n2)Playlist\n3)Exit\n>>'))
    if option == 1:
        link = input('Please Paste A Video Link : ')
        download.download_single_video(FINAL_PATH, link)

    elif option == 2:
        playlist_link = input('Please Paste A Playlist Link :')
        download.download_playlist(FINAL_PATH, playlist_link=playlist_link)

    elif option == 3:
	    print('Exiting........!')
	    break
Exemplo n.º 57
0
from sentinelsat import read_geojson, geojson_to_wkt
from Downloader import Downloader
import StringProcessor as process
import zipfile
import os

username = '******'
password = '******'
path = '/Users/DavidLei/PycharmProjects/untitled/testfile.geojson'
link: str = 'https://scihub.copernicus.eu/apihub/'
from_satelite: str = 'Sentinel-2'
package_info = list()
keys = []

downloader = Downloader(username, password, link)
wkt = geojson_to_wkt(read_geojson(path))
downloader.search_polygon(wkt,
                          '20190501',
                          '20190503',
                          str_platform_name=from_satelite,
                          percentage=(0, 100))
f = open('FileList.txt', 'w+')
for key, value in downloader.products.items():
    if "S2B_MSIL1C" in str(value):
        # print('downloading')
        # print(key)
        # print(value)
        package_info.append(str(value))
        keys.append(str(key))
        print(value['title'])
        downloader.download_one(
Exemplo n.º 58
0
 def download(self):
     self.downloader = Downloader(self, self.url, self.path)
     self.thread = threading.Thread(target = self.downloader.download)
     self.thread.start()
Exemplo n.º 59
0
    def __init__(self, uuid):
        # Movement_has_Category.delete().execute()
        # Movement.delete().execute()

        downloader = Downloader(uuid)
        account_statements = downloader.download_account_data()

        for statement in account_statements:

            try:
                movement = Movement.get(hash=statement['id'])

                if movement.id:
                    continue
            except:
                date = datetime.strptime(statement['postDate'], '%Y-%m-%d')

                if statement['__typename'] == 'BarcodePaymentEvent':
                    type = 'OUTCOME'
                    original_name = statement['title'] + " " + statement[
                        'detail']
                elif statement['__typename'] == 'TransferOutEvent':
                    type = 'OUTCOME'
                    original_name = statement['title'] + " " + statement[
                        'destinationAccount']['name']
                elif statement['__typename'] == 'TransferInEvent':
                    original_name = statement['title']

                    if statement['originAccount']:
                        original_name = original_name + " " + statement[
                            'originAccount']['name']

                    type = 'INCOME'

                if statement['__typename'] == 'TransferOutReversalEvent':
                    value = float(statement['detail'].split(
                        'R$ ', 1)[1].replace('.', '').replace(',', '.'))
                    type = 'REVERSAL'
                else:
                    value = statement['amount']

                Movement.create(original_name=original_name,
                                date=date,
                                value=value,
                                status='SHOW',
                                type=type,
                                hash=statement['id'])

        card_statements = downloader.download_credit_card_data()

        for statement in card_statements:

            try:
                movement = Movement.get(hash=statement['id'])
                if movement.id:
                    continue
            except:
                rest = statement['time'].split(
                    'T', 1)[0]  # todo Haw ficou de corrigir para a data iso

                date = datetime.strptime(rest, '%Y-%m-%d')
                Movement.create(original_name=statement['description'],
                                date=date,
                                value=statement['amount'] / 100,
                                status='SHOW',
                                type='OUTCOME',
                                hash=statement['id'])