def load_data(self): data = Downloader('data.txt') weapons = WeaponDownloader('weapons.txt') quests = QuestDownloader('quests.txt') self.weapon_list = weapons.load() self.character_namespace = data.load() self.quests = quests.load()
def download_update(): global console, check_flash # Если в меню выбрано обнвление if listbox.curselection(): # получаем выбранное значение value = listbox.curselection() status_bar['value'] = 20 status_bar.update() # получаем имя обновления которое было выбрано name = listbox.get(value) status_bar['value'] = 40 status_bar.update() logger.info("Key {}".format(name)) console = False check_flash = False # качаем выбранное обновление Downloader.download_upd(check_flash, [name.split(".zip")[0]], ftp_path_upd) status_bar['value'] = 60 status_bar.update() # разархивируем Install.unzip_upd(tmp_catalog_for_install_updates) status_bar['value'] = 80 status_bar.update() # Устанавливаем Install.search_local_upd_for_install( check_flash, tmp_catalog_for_install_updates) status_bar['value'] = 100 status_bar.update() else: Gui.show_message('Ошибка', 'Не выбрано обновление для установки.')
def status(): downloader = Downloader(app.config.config) downloads = Download.select().where( (Download.status != consts.STATUS_PROCESSED) & (Download.status != consts.STATUS_CANCELLED)).order_by( Download.started_at.desc()) return jsonify({'items': [downloader.get_status(d) for d in downloads]})
def auto_install(): global console, check_flash # поиск обновлений без указанного имени logger.info("Not found key") console = False status_bar['value'] = 20 status_bar.update() # Проверка установлена ли флешка check_flash = search_flash(system_path_media, upd_path_in_flash) status_bar['value'] = 30 status_bar.update() # поиск обновления name_update = Downloader.get_update_name(check_flash, system_name, ftp_path_task) status_bar['value'] = 40 status_bar.update() if name_update: # если было задание на сервере Downloader.download_upd(check_flash, name_update, ftp_path_upd) status_bar['value'] = 60 status_bar.update() Install.unzip_upd(tmp_catalog_for_install_updates) status_bar['value'] = 80 status_bar.update() Install.search_local_upd_for_install( check_flash, tmp_catalog_for_install_updates) else: Gui.show_message("Ошибка", "Обновления для этой кассы\nне найдены") status_bar['value'] = 100 status_bar.update()
def fetch(cls, pageNum): """ Instantiate a Page from its page number. The page HTML will be fetched and parsed. However, the image will not yet be downloaded. Parameters: pageNum (int): The page number. """ soup = None url = Page.getPageUrl(pageNum) logger.debug('Fetching page %d...', pageNum) response, err = Downloader.get(url) if err is not None: logger.error('Failed to fetch %s, %s', url, Downloader.getErrorString(err)) return None else: logger.debug('Successfully fetched page %s', pageNum) try: soup = BeautifulSoup(response.text, 'html.parser') except Exception as err: # pylint: disable=broad-except logger.error('Failed to parse %s to HTML soup, %s', url, err) return None return Page.fromSoup(pageNum, soup)
def downloadVideo(self, url, title): common.log('Trying to download video ' + str(url)) # check url if url.startswith('plugin'): common.log('Video is not downloadable') return None path = common.getSetting('download_path') if not path: path = common.browseFolders(common.translate(30017)) common.setSetting('download_path', path) title = getKeyboard(default = fu.cleanFilename(title),heading='SportsDevil') if title == None or title == '': return None downloader = Downloader() downloaded_file = downloader.downloadMovie(url, path, fu.cleanFilename(title), '.flv') if downloaded_file == None: common.log ('Download cancelled') else: common.log('Video ' + url + " downloaded to '" + downloaded_file + "'") return downloaded_file
def main(argv): print("Platform: {0}".format(plt.system())) print("Argument: {0}".format(str(argv))) down = Downloader(plt.system()) down.downloadDriver(str(argv)) print("file_exec_path: {0}".format(down.file_exec_path)) Environment(plt.system(), down.file_exec_path)
def cli_entry(download): downloader = Downloader() exporter = XlsExporter() downloader.save_all_mutual_fund_info(exporter) exporter.save_file()
def test_encoding_gb18030_20160619b(): url = "http://zhidao.baidu.com/question/130330820.html" agt = Downloader("test", "http://127.0.0.1", request=True) content_unicode = agt.request_download(url) print content_unicode[:500] assert (type(content_unicode) == unicode)
def __init__(self, link, **kwargs): super(Crawler, self).__init__() self.lookup = {} self.q = Queue() self.feed = link.replace('https://', 'http://') if self.feed.endswith('/'): self.feed = self.feed[0:len(self.feed) - 1] self.downloader = Downloader(verify=kwargs.get('verify', True)) link = link.replace('http://', '') link = link.replace('https://', '') self.path = kwargs.get( 'output_path', os.path.join( os.path.expanduser('~'), link.translate({ord(c): "" for c in "<>:/\\\"|?*"}) + '.txt')) self.links_count = 0 self.parser = Spider() log_level = kwargs.get('LOG', Crawler.INFO_LOG) if not os.path.exists(os.path.join(os.path.expanduser('~'), '.crawler')): os.makedirs(os.path.join(os.path.expanduser('~'), '.crawler')) logging.basicConfig(filename=os.path.join( os.path.expanduser('~'), '.crawler', link.translate({ord(c): "" for c in "<>:/\\\"|?*"}) + '.log'), format='%(asctime)s %(levelname)s %(message)s', level=log_level) logging.getLogger().addHandler(logging.StreamHandler())
def main(): usage = "Please use -s to specify a search term, and -r to specify" + \ " the desired result name" try: opts, args = getopt.getopt(sys.argv[1:], "s:r:h") except getopt.GetoptError: print usage sys.exit(1) search_term = None desired_item_name = None for opt, arg in opts: if opt == '-s': search_term = arg elif opt == '-r': desired_item_name = arg elif opt == '-h': print usage return 0 if not desired_item_name or not search_term: try: search_term = raw_input('Search Term: ') desired_item_name = raw_input('Desired Item Name: ') except EOFError: return 0 downloader = Downloader() downloader.download(search_term, desired_item_name) return 0
def test_fetch_latest_onpoints(self): basedir = self.standardpath downloader = Downloader() downloader.fs = FileSystem() subs = Subscriptions(downloader, self.standardpath) asub = subs.find("wbur") self.assertNotEqual(None,asub)
def main(): parser = argparse.ArgumentParser() parser.add_argument('-s', '--settings', dest='settings_path', metavar='PATH', default='', help='path to the settings file') parser.add_argument( '-i', '--ignore', dest='ignore_path', metavar='PATH', default='', help= 'path to the file that has a list od symbols that should be ignored') parser.add_argument('mode', metavar='MODE', choices=[IMPORT, UPDATE], help='\'{0}\' or \'{1}\''.format(IMPORT, UPDATE)) options = parser.parse_args() load_settings(options.settings_path) symbols = get_symbols(options.ignore_path) downloader = Downloader(symbols, options.mode) downloader.run()
def __init__(self): super(MainWindow, self).__init__() self.setupUi(self) # Menu self.dark_theme.triggered.connect(self.on_dark_theme) self.theme() # Terminal self.term = Terminal() sys.stdout = self.term self.term.event.connect(self.update_terminal) self.dl = Downloader() Thread(target=self.dl.run, daemon=True).start() # Browse self.lineEdit_file_path.setText(self.dl.output_path) self.lineEdit_file_path.textEdited.connect(self.dl.output) self.lineEdit_file_path.textEdited.connect(self.dl.output) self.browse_button.clicked.connect(self.browse) # Buttons self.firefox_button.clicked.connect(self.dl.from_firefox) self.chrome_button.clicked.connect(self.dl.from_chrome) self.text_button.clicked.connect( self.from_text) # Intercept signal to add str # Logic self.dl.added_row.connect(self.add_row) self.dl.dl_hook.connect(self.update_row) self.dl.removed_row.connect(self.remove_row)
def setUpClass(cls): id = '68428' d = Downloader() url = d.goodreads_url(id) results = d.download_ram(url) cls.soup = BeautifulSoup(results, "html.parser") cls.grs = GoodreadsScraper(cls.soup)
def download(name,url): file_name = urllib.unquote(name) file_url = urllib.unquote(url) dst_file = file_name + "." + file_url.split("/")[-1].split(".")[-1] #Download file downloader = Downloader() downloader.downloadall(os.path.join(kodiutils.get_setting("downloadPath"),dst_file),file_url,file_name)
def downloadVideo(self, url, title): common.log("Trying to download video " + str(url)) # check url if url.startswith("plugin"): common.log("Video is not downloadable") return None path = common.getSetting("download_path") if not path: path = common.browseFolders(common.translate(30017)) common.setSetting("download_path", path) title = getKeyboard(default=fu.cleanFilename(title), heading="Dragon Streams") if title == None or title == "": return None downloader = Downloader() downloaded_file = downloader.downloadMovie(url, path, fu.cleanFilename(title), ".flv") if downloaded_file == None: common.log("Download cancelled") else: common.log("Video " + url + " downloaded to '" + downloaded_file + "'") return downloaded_file
def downloadVideo(self, url, title): common.log('Trying to download video ' + str(url)) # check url if url.startswith('plugin'): common.log('Video is not downloadable') return None path = common.getSetting('download_path') if not path: path = common.browseFolders(common.translate(30017)) common.setSetting('download_path', path) title = getKeyboard(default = fu.cleanFilename(title),heading='Filmkodi') if title == None or title == '': return None downloader = Downloader() downloaded_file = downloader.downloadMovie(url, path, fu.cleanFilename(title), '.flv') if downloaded_file == None: common.log ('Download cancelled') else: common.log('Video ' + url + " downloaded to '" + downloaded_file + "'") return downloaded_file
class PyCast: def __init__( self, configFile ): self.configFile = configFile #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # def fullReset( self ): self.setting = Settings( self.configFile ) self.grabber = Grabber( self.setting ) self.scraper = Scraper( self.setting, self.grabber.xmls ) self.lister = Lister( self.setting, self.scraper.items ) self.downloader = Downloader( self.setting, self.lister.lists, self.scraper.items ) #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # def download( self, verbose = False, dryRun = False ): self.downloader.downloadAll( verbose, dryRun )
class Main: def __init__(self): self.soupHandler = SoupHandler(SystemInfo.initUri) self.downloader = Downloader() self.fileHandler = FileHandler() self.path = SystemInfo.path self.name = SystemInfo.name def introduce(self): print("kocw 다운로드 프로그램.") print("대학교 자체 cdn 전용입니다.") print("url 수정은 소스코드를 직접 수정하세요.") def end(self): print("다운로드 완료 되었습니다.") def run(self): self.introduce() self.fileHandler.safeMkdir("%s/output" % self.path) self.fileHandler.safeMkdir("%s/output/%s" % (self.path, self.name)) table = self.soupHandler.findTable() parsedOnClickStrings = self.soupHandler.parseOnClick(table) self.downloader.startIteratingDownload(parsedOnClickStrings) self.end()
def __distribute(self): while self.__isRunning: if len(self.__threadList) < 3: # download 3 videos simultaneously if not self.__queue.empty(): dl = Downloader(self.__frame, self.__plm, self.__queue.get(), self.__dir, self._lock) self.__threadList.append(dl) dl.start() sleep(WAIT_TIME) for t in self.__threadList: if t.is_alive( ): # if video is still being downloaded, update status t.updateStatus() else: # otherwise, remove it from list for next video to be downloaded self.__threadList.remove(t) sleep(WAIT_TIME) if len(self.__threadList) <= 0 and self.__queue.empty( ): # when every video is completed with self._lock: self.__frame.setFinished() self.__isRunning = False
def main(): dbc = DBController() dec = Decoder([TLEDecoder(), TLEListDecoder()]) dlc = None try: dlc = Downloader() except DownloaderError as e: print("failed to initialize downloader: " + str(e)) sys.exit(1) for esat in dlc.get_data(): sats = [] try: sats = dec.decode(esat.fmt, esat.data) except DecoderError as e: print("failed to decode: " + str(e)) try: for sat in sats: dbc.add(sat) dbc.sync() except DBError as e: print("failed to insert into db: " + str(e))
def add_song_to_queue(song_url, source_type): if source_type == 'youtube': song_info = Downloader.get_youtube_song_metadata(song_url) else: song_info = Downloader.get_soundcloud_song_metadata(song_url) _shared_queue.append(song_info)
class Dataset: def __init__(self, settings: Settings, base_path, size, path_csv, is_test): self.settings = settings self.base_path = base_path self.size = int(size) self.path_csv = path_csv self.is_test = is_test if self.size > 0: self.data = pd.read_csv(self.path_csv).head(self.size) else: self.data = pd.read_csv(self.path_csv) self.downloader = Downloader(settings) if is_test: folders_names = ['tmp', ''] else: folders_names = self.data['label'].unique().tolist() + ['tmp'] self.label_to_dir = create_file_structure(self.base_path, folders_names) def download(self): try: for _, row in self.data.iterrows(): self.downloader.execute(row, self.label_to_dir, test=self.is_test) finally: shutil.rmtree(self.label_to_dir['tmp'])
class TestDownloaded(unittest.TestCase): '''Using mock object to replace real file reading and network accessing''' def setUp(self): self.urls = [ "https://www.pornhub.com/view_video.php?viewkey=ph5ebd5cf2658e8", "https://www.pornhub.com/view_video.php?viewkey=ph5e5866a54b458", "https://www.pornhub.com/view_video.php?viewkey=ph5e9618d3baddf", "https://www.pornhub.com/view_video.php?viewkey=ph5ea206352bd49" ] self.demo_downloader = Downloader(self.urls) self.filename = "test_downloaded.json" with open(self.filename, "w"): # create a json file for testing pass def test_whether_new_urls_are_downloaded(self): for url in self.urls: with self.subTest(url=url): self.assertEqual( self.demo_downloader.is_downloaded(url, self.filename), False) def test_whether_old_urls_are_downloaded(self): for url in self.urls: self.demo_downloader.mark_as_downloaded(url, self.filename) for url in self.urls: with self.subTest(url=url): self.assertEqual( self.demo_downloader.is_downloaded(url, self.filename), True) def tearDown(self): self.demo_downloader.driver.quit() os.remove(self.filename)
def get_properties(self): url = '' d = Downloader() print d.get_sub_pages_links('http://www.wp.pl/', '') pass
def fetchIcon(self, iconName): iconFileName = None if self.weatherIconId >= 200: iconFileName, description = WeatherIconMapper.convertIcon( self.weatherIconId, self.sunrise.time(), self.sunset.time()) print("Icon file name: {}, Description: {}".format( iconFileName, description)) if iconFileName is not None: self.icon = UiUtility.loadWeatherIcon(iconFileName) self.icon.fill(UiColors.GRAY, special_flags=pygame.BLEND_RGB_ADD) else: # This weather icon ID is not mapped to a weather icon. In this case, # fetch the icon from OpenWeatherMap downloader = Downloader(None) # TODO: Do in either a background thread, or a coroutine url = "http://openweathermap.org/img/w/{}.png".format(iconName) downloader.download(url) image = downloader.getData() # Does image need to be processed before it can be used by Pygame? memFileObj = io.BytesIO(image) self.icon = pygame.image.load(memFileObj)
def post(self): if not request.json: return {'message': 'Problems parsing JSON'}, 400 if request.json.get('urls', None) is None: return {'message': 'Validation Failed', 'errors': [{'resource:': 'Package', 'field': 'urls', 'code': 'missing_field'}]}, 422 if not isinstance(request.json['urls'], list): return {'message': 'Validation Failed', 'errors': [{'resource:': 'Package', 'field': 'urls', 'code': 'invalid'}]}, 422 try: parent_conn, child_conn = multiprocessing.Pipe() d = Downloader(child_conn, app.config['BASEPATH'], request.json['urls']) d.start() r = parent_conn.recv() d.join() p = Package(ip=request.remote_addr, path=r['zipname'], date_created=arrow.now().datetime) db.session.add(p) db.session.commit() except Exception as e: logger.error(e) return {'package': 'http://{0}/{1}'.format( app.config['SERVER_ADDR'], r['name'])}, 201
def _networkM3U8(self, url: str) -> None: url = url.replace("www.", "").removeprefix("https://") filmName = re.findall( r"[altadefinizione\..*|altadefinizionecommunity\.net]\/(.*/)", url) serieName = re.findall(r"seriehd\..*\/(.*/)", url) if filmName != []: out = self.output() self.root.destroy() film = Film(filmName[0], out) if film.init[0] == 'created': Downloader(film.outputFile) elif film.init[0] == "continuare": messagebox.showinfo("4K Downloader", "Download già iniziato, verrà ripreso") Downloader(film.outputFile) elif film.init[0] == "sostituire": s = messagebox.askyesno( "4K Downloader", "Nome già esistente ma i file sono diversi.\nSi -> Indicizza file\nNo -> Sostituisci file" ) if s: film.outputFile = film.outputFile.duplicate() else: film.outputFile.temp().remove() Data.delete(film.outputFile.name) Film.initialize(film.outputFile, film.m3u8Path) Downloader(film.outputFile)
def __init__(self, ): # - Allow ourselves to use any Downloader methods Downloader.__init__(self, ) # - Local directory for saving data self.refcat_dir = self._fetch_refcat_data_directory() # - Anticipated filepath for compiled refcat code self.refcat_codename = 'refcat' self.refcat_filepath = os.path.join(self.refcat_dir, self.refcat_codename) # - Set of refcat catalogues on mast self.downloadable_files = { '16': 'https://archive.stsci.edu/hlsps/atlas-refcat2/orig/hlsp_atlas-refcat2_atlas_ccd_00-m-16_multi_v1_cat.tbz', '17': 'https://archive.stsci.edu/hlsps/atlas-refcat2/orig/hlsp_atlas-refcat2_atlas_ccd_16-m-17_multi_v1_cat.tbz', '18': 'https://archive.stsci.edu/hlsps/atlas-refcat2/orig/hlsp_atlas-refcat2_atlas_ccd_17-m-18_multi_v1_cat.tbz', '19': 'https://archive.stsci.edu/hlsps/atlas-refcat2/orig/hlsp_atlas-refcat2_atlas_ccd_18-m-19_multi_v1_cat.tbz', '20': 'https://archive.stsci.edu/hlsps/atlas-refcat2/orig/hlsp_atlas-refcat2_atlas_ccd_19-m-20_multi_v1_cat.tbz', 'code': 'https://archive.stsci.edu/prepds/atlas-refcat2/refcat.c', 'man': 'https://archive.stsci.edu/prepds/atlas-refcat2/refcat.man', }
class Client: def __init__(self, config: ConfigurationFile): self.config = config self.database = Database(self.config.db_file) self.organizer = Organizer(self.config, self.config.db_file) self.downloader = Downloader(self.config.db_file, self.organizer, self.config) self.tracker = Tracker(self.config.db_file, self.downloader, self.config.update_period) self.tracker.start() self.downloader.start() self.organizer.start() def add_tvshow(self, tvshow_id: int): tvshow_name = showrss.get_name(tvshow_id) self.database.put_tvshow(TVShow(tvshow_id, tvshow_name)) def remove_tvshow(self, tvshow_id: int): self.database.remove_tvshow(tvshow_id) def list_tvshows(self): return self.database.tvshows() def list_episodes(self, state: EpisodeState = None): return self.database.episodes(state) def download_progress(self): return self.downloader.downloads() def exit(self): self.tracker.stop() self.downloader.stop()
def __init__(self): reload(sys) sys.setdefaultencoding('UTF-8') self.title = 'Nada' self.model = ['luoo 落网', 'echo 回声', 'nada 收藏', '关于'] self.view = 'menu' self.ctrl = 'menu' self.offset = 0 self.index = 0 self.step = 10 self.play_id = -1 self.play_vol = -1 self.present = [] self.stack = [] self.player = Player() self.ui = UI() self.luoo = Luoo() self.echo = Echo() self.downloader = Downloader() self.database = Database() self.database.load() self.collections = self.database.data['collections'][0] self.screen = curses.initscr() self.screen.keypad(1)
def main(): args = parse_argument() try: if args.socks5[0] and args.socks5[1]: if args.proxy: logger.error('invalid proxy protocol count.') raise SystemExit socks.set_default_proxy(socks.SOCKS5, args.socks5[0], int(args.socks5[1]), True, args.socks5[2], args.socks5[3]) socket.socket = socks.socksocket except Exception as e: logger.error('invalid socks5 proxy arguments.') raise SystemExit t = Thread(args.board, args.thread) if not args.downloading: thread_info = t.thread_info() logger.info('/{}/ - {} - {}'.format(args.board, thread_info['sub'], const.BOARDS[args.board])) logger.info('total images - {}'.format(thread_info['images'])) else: downloader = Downloader(path=args.path, threads=args.threads, timeout=args.timeout, is_thumb=args.thumb) q = t.detail_queue() downloader.fetch(q)
class Spider(object): start_urls = [''] def __init__(self): self.count = 0 # 用于记录xls的行数 self.downloader = Downloader() self.next_url = None self.mutex = Lock() # 创建一个锁 用于文件的读写 def __enter__(self): # 创建 xls 文件对象 self.wb = xlwt.Workbook() # 新增一个表单 self.sheet = self.wb.add_sheet('Sheet 1') self.start_request() def __exit__(self, exc_type, exc_val, exc_tb): self.wb.save('data.xls') def start_request(self): for url in self.start_urls: self.downloader.add_job(priority_number=1, job=url, handle=self.parse) self.downloader.create_threads() def parse(self, response): pass
def get(self, type, img): retImg = self.__createYzmImage(img, type) d = Downloader() data = dict( JsdmYzmPlugin.jsdm_info, **{ "captchaData": retImg, "captchaType": 1023, "captchaMinLength": 0, "captchaMaxLength": 0 }) dataPost = json.dumps(data) res = d.request_data_from_url('https://v2-api.jsdama.com/upload', 'post', dataPost, timeout=-1) j_res = json.loads(res) dataret = j_res.get('data') if dataret: recognition = dataret['recognition'] else: recognition = None self.getdmResult = res print('验证码自动识别:[', recognition, ']') return recognition
def __init__(self, config): """Weibo类初始化""" self.config = config # change cookie from string to dict if type(self.config['cookie']) == type(u''): self.config['cookie'] = { t.strip().split("=")[0]: t.strip().split("=")[1] for t in self.config['cookie'].split(";") } if type(self.config['user_id_list']) == type(u""): user_id_list = self.config['user_id_list'] if not os.path.isabs(user_id_list): user_id_list = os.path.split( os.path.realpath(__file__))[0] + os.sep + user_id_list self.config['user_id_list'] = user_id_list with open(self.config['user_id_list'], 'rb') as f: lines = f.read().splitlines() lines = [line.decode('utf-8') for line in lines] self.config['user_id_list'] = [ line.split(' ')[0] for line in lines if len(line.split(' ')) > 0 and line.split(' ')[0].isdigit() ] if type(self.config['since_date']) == type(0): self.config['since_date'] = str( date.today() - timedelta(self.config['since_date'])) self.validator = Validator(self.config) self.validator.validate() self.printer = Printer() self.writer = Writer(self.config) self.downloader = Downloader(self.config) self.parser = Parser(self.config)
def __init__(self): super(Main, self).__init__(load_config, NAME_FILE_CONFIG_PATH, NAME_FILE_LOG_PATH, NAME_FILE_CSV_PATH) self.loading_args() self.log_configuration() # General variable self.config = self.loading_file_config() self.errors = [] self.fields_csv = self.config.get("GENERAL", "fields_csv").split(",") list_links = parser_cvs(self.args.csv, self.fields_csv) http = Downloader() report = Report() for data in list_links: download = http.download(data["link"]) report.add_elements(download) report.print_report() logging.info("Script Completado.")
class Listener(Thread): def __init__(self, target): Thread.__init__(self) self._address = 'amqp://*****:*****@orangutan.rmq.cloudamqp.com/qxyexflk' self._queue = 'mytube' self._timeout = 5 self._downloader = Downloader(target) def run(self): parameters = pika.URLParameters(self._address) parameters.socket_timeout = self._timeout connection = pika.BlockingConnection(parameters) channel = connection.channel() channel.queue_declare(queue=self._queue, durable=True) channel.basic_qos(prefetch_count=1) channel.basic_consume(self.on_message, queue=self._queue) channel.start_consuming() print("Waiting for messages...") def on_message(self, channel, method, properties, body): print('Received Message') try: self._downloader.download(body.decode('ascii')) channel.basic_ack(delivery_tag=method.delivery_tag) print('Processed Message') except: channel.basic_nack(delivery_tag=method.delivery_tag, requeue=False) print('Failed To Process')
def __init__(self, plugin, layerDef, creditVisibility=1): QgsPluginLayer.__init__(self, TileLayer.LAYER_TYPE, layerDef.title) self.plugin = plugin self.iface = plugin.iface self.layerDef = layerDef self.creditVisibility = 1 if creditVisibility else 0 # set custom properties self.setCustomProperty("title", layerDef.title) self.setCustomProperty("credit", layerDef.credit) # TODO: need to remove self.setCustomProperty("serviceUrl", layerDef.serviceUrl) self.setCustomProperty("yOriginTop", layerDef.yOriginTop) self.setCustomProperty("zmin", layerDef.zmin) self.setCustomProperty("zmax", layerDef.zmax) if layerDef.bbox: self.setCustomProperty("bbox", layerDef.bbox.toString()) self.setCustomProperty("creditVisibility", self.creditVisibility) # create a QgsCoordinateReferenceSystem instance if plugin has no instance yet if plugin.crs3857 is None: plugin.crs3857 = QgsCoordinateReferenceSystem(3857) self.setCrs(plugin.crs3857) if layerDef.bbox: self.setExtent( BoundingBox.degreesToMercatorMeters( layerDef.bbox).toQgsRectangle()) else: self.setExtent( QgsRectangle(-layerDef.TSIZE1, -layerDef.TSIZE1, layerDef.TSIZE1, layerDef.TSIZE1)) self.setValid(True) self.tiles = None self.useLastZoomForPrint = False self.canvasLastZoom = 0 self.setTransparency(LayerDefaultSettings.TRANSPARENCY) self.setBlendModeByName(LayerDefaultSettings.BLEND_MODE) self.setSmoothRender(LayerDefaultSettings.SMOOTH_RENDER) self.downloader = Downloader(self) self.downloader.userAgent = "QGIS/{0} QuickMapServices Plugin".format( QGis.QGIS_VERSION ) # , self.plugin.VERSION) # not written since QGIS 2.2 self.downloader.DEFAULT_CACHE_EXPIRATION = QSettings().value( "/qgis/defaultTileExpiry", 24, type=int) QObject.connect(self.downloader, SIGNAL("replyFinished(QString, int, int)"), self.networkReplyFinished) # multi-thread rendering self.eventLoop = None QObject.connect(self, SIGNAL("fetchRequest(QStringList)"), self.fetchRequest) if self.iface: QObject.connect(self, SIGNAL("showMessage(QString, int)"), self.showStatusMessageSlot) QObject.connect( self, SIGNAL("showBarMessage(QString, QString, int, int)"), self.showBarMessageSlot)
def getRssInfo(rss): downLoader = Downloader() try: (t_url, header, html) = downLoader.open(rss) print "download ", rss except Exception, msg: getLogger().error("feed download error : %s %s", msg, rss) return None
def __download(self): downloader = Downloader(self.url, self.downloadFile, \ OnlineUpdater.__getModifiedTime(self.recipeFile)) try: retval = downloader.download() finally: downloader.close() return retval
def test_id_from_downloads(self): query = 'Well of Ascension' d = Downloader() url = d.goodreads_id_query(query) results = d.download_ram(url) soup = BeautifulSoup(results, 'html.parser') gs = GoogleScraper(soup) self.assertEqual(gs.id, '68429')
def main(): if len(sys.argv) <= 1: print "No url given!" sys.exit(1) finder = FuxFinder(sys.argv[1]) downloader = Downloader(finder.find_movie(), finder.find_title() + '.mp4') downloader.download()
class Project(object): """docstring for Project""" def __init__(self, configs, path): super(Project, self).__init__() self.name = 'name' in configs and configs['name'] or os.path.basename(path) self.configs = configs['configs'] self._path = path self._spiders_cls = {} self._spiders_cfg = {} if 'spiders' in self.configs: cfgs = self.configs['spiders'] for scfg in cfgs: name = scfg.get('id') or scfg.get('name') self._spiders_cfg[name] = scfg self._downloader = None def load_spider(self, name, params=None): spider_cls = None if name in self._spiders_cls: spider_cls = self._spiders_cls[name] else: for fn in os.listdir(self._path): if fn in ('.', '..'): continue fp = os.path.join(self._path, fn) if os.path.isfile(fp): mod = loader.load_module(fp) if mod and name in mod: spider_cls = mod[name] self._spiders_cls[name] = spider_cls if spider_cls: s = spider_cls() p = self._spiders_cfg.get(name, {}) if params: p.update(params) if p: s.set_params(**p) return s return None def get_downloader(self): if self._downloader is None: thread = self.configs.get('thread', 4) headers = self.configs.get('headers') self._downloader = Downloader(thread) if headers: self._downloader.add_headers(headers) return self._downloader def get_crawler(self): crawler = Crawler(self) return crawler
def add(self, path, url, file_name): """""" if file_name not in self.active_downloads: if self.get_slot(): logger.info("Started %s" % (file_name)) th = Downloader(path, url, file_name, self.link_parser) self.active_downloads[file_name] = th th.start() return True
def ncd_loop(doInit, dlThreadNum): ndutil.setTimezone() #read config cnfManager = CnfManager() cnfManager.load('./ndc.cnf') cnfData = cnfManager.getCnfData() #check dirs ndutil.enableDir(cnfData['dirWorking']) ndutil.enableDir(cnfData['dirStore']) #ndlcom logger = Logger('nDroid-Crawler', cnfData['ndlComHost'], cnfData['ndlComPort']) logger.logger('Initiating') dbManager = DbManager(cnfData['dbHost'], cnfData['dbUser'], cnfData['dbPass'], cnfData['dbName']) if doInit: dbManager.create_table() os.system('rm -f %s/*' % cnfData['dirWorking']) os.system('rm -f %s/*' % cnfData['dirStore']) #logger.logger('Customizing Spiders') #spiderGenerator = SpiderGenerator('template', 'spider/spiders') #for spider in cnfData['spiders']: # spiderGenerator.gen_spider(spider, cnfData[spider]['startPage'], cnfData[spider]['stopPage']) rpQueue = Queue() pdQueue = Queue() dpQueue = Queue() pdLock = threading.Lock() rpcMonitor = RpcMonitor(logger, cnfData['rpcPort'], cnfData['rpcAuth'], 'RpcMonitor') rpcMonitor.setGlobalInfo(ndutil.getCurrentTimeStr(), 'Standalone', dlThreadNum) rpcMonitor.setDownloadTotal(pdQueue.qsize()) rpcMonitor.setPdQueueSize(pdQueue.qsize()) botScheduler = BotScheduler(logger, rpcMonitor, cnfData['spiders'], cnfData['spiderCnfs'], 'BotScheduler') receiver = Receiver(logger, rpcMonitor, rpQueue, cnfData['receiverPort'], 'Receiver') preProcessor = PreProcessor(logger, rpcMonitor, rpQueue, pdQueue, pdLock, dbManager, 'PreProcessor') downloader = Downloader([logger, rpcMonitor, pdQueue, dpQueue, pdLock, dlThreadNum, cnfData['dirWorking']], 'Downloader') processor = Processor(logger, rpcMonitor, dpQueue, pdLock, pdQueue, dbManager, cnfData['dirWorking'], cnfData['dirStore'], 'Processor') logger.logger('Starting Threads') rpcMonitor.start() botScheduler.start() receiver.start() preProcessor.start() downloader.start() processor.start() processor.join() downloader.join() preProcessor.join() receiver.join() botScheduler.join() rpcMonitor.join()
def download(self, args): downloader = Downloader() if args.name == "All": for configFile in os.listdir("config/"): res = downloader.download("config/" + configFile) self.printResult(res, configFile.split(".")[0]) else: res = downloader.download("config/" + args.name + ".json") self.printResult(res, args.name)
class DriDaemon(DaemonBase): """ Manages all functions that need to take place on the router on a regular basis """ def __init__(self, options): DaemonBase.__init__(self, options) self.kill_switch = False self.loops = 0 self.downloader = Downloader(options) self.uploader = Uploader(options) self.allowed_traffic = self.downloader.get_allowed_traffic() self.policy_mgr = PolicyMgr("/tmp/dnsmasq.log", self.allowed_traffic, self.options) self.policy_mgr.prep_system() self.policy_mgr.initial_load() self.policy_mgr.rotate_log() def main_loop(self): """ Runs forever. We're a daemon """ self.log('Starting dri...') while not self.kill_switch: start_time = time.time() while time.time() < (start_time + INTER_LOOP_SLEEP): try: has_more = self.policy_mgr.check_for_new_stuff() if not has_more: time.sleep(1) except (PolicyMgrException, CommandException): self.log('Help! Policy Manager') if MAX_LOOPS: self.loops += 1 if self.loops > MAX_LOOPS: sys.exit(0) try: self.downloader.get_addresses() allowed_traffic = self.downloader.get_allowed_traffic() self.policy_mgr.process_new_allowed(allowed_traffic) log_open_files("downloader") except (DownloadException, CommandException): self.log('Help! Downloading') try: self.uploader.upload_arp_table() log_open_files("uploader") except (UploadException, CommandException): self.log('Help! Uploading') #print "I LIVE" def terminate(self): self.kill_switch = True print "dying"
def donwload_playlist(playlist): youtube = Youtube_List() lista = youtube.list_playlist(playlist) #Check if some files are already downloaded list_to_download = youtube.getDiferences(playlist,lista) download = Downloader() list_errors = download.download_list(list_to_download) #Create the logs files youtube.save_playlist_on_file(playlist + ".err",list_errors) youtube.save_playlist_on_file(playlist,lista)
def pack(data): downloads_folder = fetch_folder_path(DOWNLOADS_FOLDER) archives_folder = fetch_folder_path(ARCHIVES_FOLDER) downloader = Downloader(downloads_folder, data) if downloader.download(): archivator = Archivator(downloader.destination, os.path.join(archives_folder, "{}.zip".format(downloader.uid))) archivator.archive() return os.path.basename(archivator.destination)
def downloadAll(self): utils.mkdir(baseDirPath, self.comicTitle) for i in range(threadCount): t = Downloader() t.start() for url in self.chapterURLs[:2]: getChapterInfo(url, self.comicTitle) time.sleep(1) Downloader.isStop = True Tkinter.Label(self.window, text=u"下载完成").pack()
def getTistoryId(url): downLoader = Downloader() attr_dict = dict() attr_dict["tid"] = "livere_blogurl = '****.tistory.com';" attr_dict["tid2"] = """__addParam("author","****");""" try: (t_url, header, html) = downLoader.open(url) print "download", url except Exception, msg: getLogger().error("feed download error : %s %s", msg, rss) return None
def test_WithCacheDirCAndEpT1x1DownloadingDirA_MoveFromCBarATpCBarT1x1( self, mock_move): config = ConfigurationFile() config.cache_dir = 'c' downloader = Downloader(None, None, config) episode = Episode('t', 1, 1, None, None) downloading_dir = 'a' downloader._moveto_downloaded_dir(downloading_dir, episode) mock_move.assert_called_once_with( 'c/a', 'c/t1x1')
def start(): year = input("Please give the year you want to download: ") parser = NostalgiaMachineParser() songs = parser.get_list_of_songs_from_year(year) dl = Downloader() #print(songs[0]) for s in songs: print("Downloading: " + str(s)) dl.get_song(s)
def http_downloader(url): from downloader import Downloader downer = Downloader() hostpagehtml = downer.getContent(url) if hostpagehtml: root = fromstring(hostpagehtml) msgdivs = root.xpath("//div[@class='poi-tile__info']") for msg in msgdivs: msgdiv = tostring(msg, encoding='utf-8') print msgdiv ParseMsg(msgdiv) pass print url, '\n is over'
def run(self): print 'ENGINE STSRT' self.is_running=True for i in range(settings.S_thre_num): t=Downloader(i,self.scheduler,self) self.threads.append(t) for t in self.threads: t.start() time.sleep(10) for t in self.threads: t.join() self.stop()