def refresh(self, file=None): if file is None: # blocking network operation file = download(self.url, raise_error=False) self.webdoc.update(file) return self.webdoc.status
def build(path_or_url, name, no_flicker_patch, patch): if is_url(path_or_url): path_or_url = download(path_or_url, config.DOWNLOAD_TO) wine_src_dir = extract(path_or_url, os.path.dirname(path_or_url)) default_name = f'WS10{os.path.basename(wine_src_dir)}' try: if patch: for p in patch: apply_patch(wine_src_dir, p) if no_flicker_patch: if not name: name = f'{default_name}NoFlicker' path = os.path.join(config.ROOT_DIR, 'patches', 'wine', 'no-flickering-wine-2.14.diff') apply_patch(wine_src_dir, path) config_writer(wine_src_dir, name or default_name) subprocess.run([ config.WINESKIN_ENGINE_BUILD_SCRIPT ], check=True) except: raise click.ClickException(f'Unexpected error: {sys.exc_info()[:-1]}') finally: remove_tree(wine_src_dir)
def execute(self, engine): """ :type engine: engine.Engine """ engine.log.info(u"Start checking for <b>rutor.org</b>") with DBSession() as db: topics = db.query(RutorOrgTopic).all() db.expunge_all() for topic in topics: topic_name = topic.display_name try: engine.log.info(u"Check for changes <b>%s</b>" % topic_name) torrent_content, filename = download(self.tracker.get_download_url(topic.url)) engine.log.downloaded(u"Torrent <b>%s</b> downloaded" % filename or topic_name, torrent_content) torrent = Torrent(torrent_content) if torrent.info_hash != topic.hash: engine.log.info(u"Torrent <b>%s</b> was changed" % topic_name) existing_torrent = engine.find_torrent(torrent.info_hash) if existing_torrent: engine.log.info(u"Torrent <b>%s</b> already added" % filename or topic_name) elif engine.add_torrent(torrent_content): old_existing_torrent = engine.find_torrent(topic.hash) if old_existing_torrent: engine.log.info(u"Updated <b>%s</b>" % filename or topic_name) else: engine.log.info(u"Add new <b>%s</b>" % filename or topic_name) if old_existing_torrent: if engine.remove_torrent(topic.hash): engine.log.info(u"Remove old torrent <b>%s</b>" % old_existing_torrent['name']) else: engine.log.failed(u"Can't remove old torrent <b>%s</b>" % old_existing_torrent['name']) existing_torrent = engine.find_torrent(torrent.info_hash) if existing_torrent: last_update = existing_torrent['date_added'] else: last_update = datetime.datetime.now() with DBSession() as db: db.add(topic) topic.hash = torrent.info_hash topic.last_update = last_update db.commit() else: engine.log.info(u"Torrent <b>%s</b> not changed" % topic_name) except Exception as e: engine.log.failed(u"Failed update <b>%s</b>.\nReason: %s" % (topic_name, e.message)) engine.log.info(u"Finish checking for <b>rutor.org</b>")
def execute(self, engine): engine.log.info(u"Start checking for <b>lostfilm.tv</b>") try: if not self._login_to_tracker(engine): engine.log.failed('Login to <b>lostfilm.tv</b> failed') return cookies = self.tracker.get_cookies() with DBSession() as db: db_series = db.query(LostFilmTVSeries).all() series = map(row2dict, db_series) series_names = {s['search_name'].lower(): s for s in series} d = feedparser.parse(u'http://www.lostfilm.tv/rssdd.xml') engine.log.info( u'Download <a href="http://www.lostfilm.tv/rssdd.xml">rss</a>') try: for entry in d.entries: info = self.tracker.parse_rss_title(entry.title) if not info: engine.log.failed( u'Can\'t parse title: <b>{0}</b>'.format( entry.title)) continue original_name = info['original_name'] serie = series_names.get(original_name.lower(), None) if not serie: engine.log.info( u'Not watching series: {0}'.format(original_name)) continue if (info['season'] < serie['season']) or \ (info['season'] == serie['season'] and info['episode'] <= serie['episode']): engine.log.info( u"Series <b>{0}</b> not changed".format( original_name)) continue if info['quality'] != serie['quality']: engine.log.info( u'Skip <b>{0}</b> by quality filter. Searching for {1} by get {2}' .format(original_name, serie['quality'], info['quality'])) continue try: torrent_content, filename = download(entry.link, cookies=cookies) except Exception as e: engine.log.failed( u"Failed to download from <b>{0}</b>.\nReason: {1}" .format(entry.link, e.message)) continue torrent = Torrent(torrent_content) engine.log.downloaded( u'Download new series: {0} ({1})'.format( original_name, info['episode_info']), torrent_content) existing_torrent = engine.find_torrent(torrent.info_hash) if existing_torrent: engine.log.info( u"Torrent <b>%s</b> already added" % filename or original_name) elif engine.add_torrent(torrent_content): engine.log.info(u"Add new <b>%s</b>" % filename or original_name) existing_torrent = engine.find_torrent( torrent.info_hash) if existing_torrent: last_update = existing_torrent['date_added'] else: last_update = datetime.datetime.now() with DBSession() as db: db_serie = db.query(LostFilmTVSeries)\ .filter(LostFilmTVSeries.id == serie['id'])\ .first() db_serie.last_update = last_update db_serie.season = info['season'] db_serie.episode = info['episode'] db.commit() except Exception as e: engine.log.failed( u"Failed update <b>lostfilm</b>.\nReason: {0}".format( e.message)) finally: engine.log.info(u"Finish checking for <b>lostfilm.tv</b>")
from utils import downloader from utils import md5 downloader.download("https://www.baidu.com/img/bd_logo1.png") print md5.encode("angus")
def execute(self, engine): engine.log.info(u"Start checking for <b>lostfilm.tv</b>") try: if not self._login_to_tracker(engine): engine.log.failed('Login to <b>lostfilm.tv</b> failed') return cookies = self.tracker.get_cookies() with DBSession() as db: db_series = db.query(LostFilmTVSeries).all() series = map(row2dict, db_series) series_names = {s['search_name'].lower(): s for s in series} d = feedparser.parse(u'http://www.lostfilm.tv/rssdd.xml') engine.log.info(u'Download <a href="http://www.lostfilm.tv/rssdd.xml">rss</a>') try: for entry in d.entries: info = self.tracker.parse_rss_title(entry.title) if not info: engine.log.failed(u'Can\'t parse title: <b>{0}</b>'.format(entry.title)) continue original_name = info['original_name'] serie = series_names.get(original_name.lower(), None) if not serie: engine.log.info(u'Not watching series: {0}'.format(original_name)) continue if (info['season'] < serie['season']) or \ (info['season'] == serie['season'] and info['episode'] <= serie['episode']): engine.log.info(u"Series <b>{0}</b> not changed".format(original_name)) continue if info['quality'] != serie['quality']: engine.log.info(u'Skip <b>{0}</b> by quality filter. Searching for {1} by get {2}' .format(original_name, serie['quality'], info['quality'])) continue try: torrent_content, filename = download(entry.link, cookies=cookies) except Exception as e: engine.log.failed(u"Failed to download from <b>{0}</b>.\nReason: {1}" .format(entry.link, e.message)) continue torrent = Torrent(torrent_content) engine.log.downloaded(u'Download new series: {0} ({1})' .format(original_name, info['episode_info']), torrent_content) existing_torrent = engine.find_torrent(torrent.info_hash) if existing_torrent: engine.log.info(u"Torrent <b>%s</b> already added" % filename or original_name) elif engine.add_torrent(torrent_content): engine.log.info(u"Add new <b>%s</b>" % filename or original_name) existing_torrent = engine.find_torrent(torrent.info_hash) if existing_torrent: last_update = existing_torrent['date_added'] else: last_update = datetime.datetime.now() with DBSession() as db: db_serie = db.query(LostFilmTVSeries)\ .filter(LostFilmTVSeries.id == serie['id'])\ .first() db_serie.last_update = last_update db_serie.season = info['season'] db_serie.episode = info['episode'] db.commit() except Exception as e: engine.log.failed(u"Failed update <b>lostfilm</b>.\nReason: {0}".format(e.message)) finally: engine.log.info(u"Finish checking for <b>lostfilm.tv</b>")
def download(self): if self.src is None: archive_path = downloader.download(self.url, self.download_to) self.src = archiver.extract(archive_path, self.extract_to) return self.src
def _get_original_link(link): resp = download(link, follow_redirect=False) if resp is None: return link return resp.original_url or link
import urllib2 from lxml import html import os from utils import dirs from utils import downloader basedir = "python_docs" root = "https://docs.python.org/2.7/" index = urllib2.urlopen(root).read() dirs.mkdirs(basedir) node = html.fromstring(index) for i in node.xpath("//body//a"): url = i.get("href") downloader.download(root, url, basedir)
def execute(self, engine): """ :type engine: engine.Engine """ engine.log.info(u"Start checking for <b>rutor.org</b>") with DBSession() as db: topics = db.query(RutorOrgTopic).all() db.expunge_all() for topic in topics: topic_name = topic.display_name try: engine.log.info(u"Check for changes <b>%s</b>" % topic_name) torrent_content, filename = download( self.tracker.get_download_url(topic.url)) engine.log.downloaded( u"Torrent <b>%s</b> downloaded" % filename or topic_name, torrent_content) torrent = Torrent(torrent_content) if torrent.info_hash != topic.hash: engine.log.info(u"Torrent <b>%s</b> was changed" % topic_name) existing_torrent = engine.find_torrent(torrent.info_hash) if existing_torrent: engine.log.info( u"Torrent <b>%s</b> already added" % filename or topic_name) elif engine.add_torrent(torrent_content): old_existing_torrent = engine.find_torrent(topic.hash) if old_existing_torrent: engine.log.info(u"Updated <b>%s</b>" % filename or topic_name) else: engine.log.info(u"Add new <b>%s</b>" % filename or topic_name) if old_existing_torrent: if engine.remove_torrent(topic.hash): engine.log.info( u"Remove old torrent <b>%s</b>" % old_existing_torrent['name']) else: engine.log.failed( u"Can't remove old torrent <b>%s</b>" % old_existing_torrent['name']) existing_torrent = engine.find_torrent( torrent.info_hash) if existing_torrent: last_update = existing_torrent['date_added'] else: last_update = datetime.datetime.now() with DBSession() as db: db.add(topic) topic.hash = torrent.info_hash topic.last_update = last_update db.commit() else: engine.log.info(u"Torrent <b>%s</b> not changed" % topic_name) except Exception as e: engine.log.failed(u"Failed update <b>%s</b>.\nReason: %s" % (topic_name, e.message)) engine.log.info(u"Finish checking for <b>rutor.org</b>")