def load_data(self):
data = Downloader('data.txt')
weapons = WeaponDownloader('weapons.txt')
quests = QuestDownloader('quests.txt')
self.weapon_list = weapons.load()
self.character_namespace = data.load()
self.quests = quests.load()
def download_update():
global console, check_flash
# Если в меню выбрано обнвление
if listbox.curselection():
# получаем выбранное значение
value = listbox.curselection()
status_bar['value'] = 20
status_bar.update()
# получаем имя обновления которое было выбрано
name = listbox.get(value)
status_bar['value'] = 40
status_bar.update()
logger.info("Key {}".format(name))
console = False
check_flash = False
# качаем выбранное обновление
Downloader.download_upd(check_flash, [name.split(".zip")[0]],
ftp_path_upd)
status_bar['value'] = 60
status_bar.update()
# разархивируем
Install.unzip_upd(tmp_catalog_for_install_updates)
status_bar['value'] = 80
status_bar.update()
# Устанавливаем
Install.search_local_upd_for_install(
check_flash, tmp_catalog_for_install_updates)
status_bar['value'] = 100
status_bar.update()
else:
Gui.show_message('Ошибка',
'Не выбрано обновление для установки.')
def status():
downloader = Downloader(app.config.config)
downloads = Download.select().where(
(Download.status != consts.STATUS_PROCESSED)
& (Download.status != consts.STATUS_CANCELLED)).order_by(
Download.started_at.desc())
return jsonify({'items': [downloader.get_status(d) for d in downloads]})
def auto_install():
global console, check_flash
# поиск обновлений без указанного имени
logger.info("Not found key")
console = False
status_bar['value'] = 20
status_bar.update()
# Проверка установлена ли флешка
check_flash = search_flash(system_path_media, upd_path_in_flash)
status_bar['value'] = 30
status_bar.update()
# поиск обновления
name_update = Downloader.get_update_name(check_flash, system_name,
ftp_path_task)
status_bar['value'] = 40
status_bar.update()
if name_update:
# если было задание на сервере
Downloader.download_upd(check_flash, name_update, ftp_path_upd)
status_bar['value'] = 60
status_bar.update()
Install.unzip_upd(tmp_catalog_for_install_updates)
status_bar['value'] = 80
status_bar.update()
Install.search_local_upd_for_install(
check_flash, tmp_catalog_for_install_updates)
else:
Gui.show_message("Ошибка",
"Обновления для этой кассы\nне найдены")
status_bar['value'] = 100
status_bar.update()
def fetch(cls, pageNum):
"""
Instantiate a Page from its page number.
The page HTML will be fetched and parsed.
However, the image will not yet be downloaded.
Parameters:
pageNum (int): The page number.
"""
soup = None
url = Page.getPageUrl(pageNum)
logger.debug('Fetching page %d...', pageNum)
response, err = Downloader.get(url)
if err is not None:
logger.error('Failed to fetch %s, %s', url,
Downloader.getErrorString(err))
return None
else:
logger.debug('Successfully fetched page %s', pageNum)
try:
soup = BeautifulSoup(response.text, 'html.parser')
except Exception as err: # pylint: disable=broad-except
logger.error('Failed to parse %s to HTML soup, %s', url, err)
return None
return Page.fromSoup(pageNum, soup)
def downloadVideo(self, url, title):
common.log('Trying to download video ' + str(url))
# check url
if url.startswith('plugin'):
common.log('Video is not downloadable')
return None
path = common.getSetting('download_path')
if not path:
path = common.browseFolders(common.translate(30017))
common.setSetting('download_path', path)
title = getKeyboard(default = fu.cleanFilename(title),heading='SportsDevil')
if title == None or title == '':
return None
downloader = Downloader()
downloaded_file = downloader.downloadMovie(url, path, fu.cleanFilename(title), '.flv')
if downloaded_file == None:
common.log ('Download cancelled')
else:
common.log('Video ' + url + " downloaded to '" + downloaded_file + "'")
return downloaded_file
def main(argv):
print("Platform: {0}".format(plt.system()))
print("Argument: {0}".format(str(argv)))
down = Downloader(plt.system())
down.downloadDriver(str(argv))
print("file_exec_path: {0}".format(down.file_exec_path))
Environment(plt.system(), down.file_exec_path)
def cli_entry(download):
downloader = Downloader()
exporter = XlsExporter()
downloader.save_all_mutual_fund_info(exporter)
exporter.save_file()
def test_encoding_gb18030_20160619b():
url = "http://zhidao.baidu.com/question/130330820.html"
agt = Downloader("test", "http://127.0.0.1", request=True)
content_unicode = agt.request_download(url)
print content_unicode[:500]
assert (type(content_unicode) == unicode)
def __init__(self, link, **kwargs):
super(Crawler, self).__init__()
self.lookup = {}
self.q = Queue()
self.feed = link.replace('https://', 'http://')
if self.feed.endswith('/'):
self.feed = self.feed[0:len(self.feed) - 1]
self.downloader = Downloader(verify=kwargs.get('verify', True))
link = link.replace('http://', '')
link = link.replace('https://', '')
self.path = kwargs.get(
'output_path',
os.path.join(
os.path.expanduser('~'),
link.translate({ord(c): ""
for c in "<>:/\\\"|?*"}) + '.txt'))
self.links_count = 0
self.parser = Spider()
log_level = kwargs.get('LOG', Crawler.INFO_LOG)
if not os.path.exists(os.path.join(os.path.expanduser('~'),
'.crawler')):
os.makedirs(os.path.join(os.path.expanduser('~'), '.crawler'))
logging.basicConfig(filename=os.path.join(
os.path.expanduser('~'), '.crawler',
link.translate({ord(c): ""
for c in "<>:/\\\"|?*"}) + '.log'),
format='%(asctime)s %(levelname)s %(message)s',
level=log_level)
logging.getLogger().addHandler(logging.StreamHandler())
def main():
usage = "Please use -s to specify a search term, and -r to specify" + \
" the desired result name"
try:
opts, args = getopt.getopt(sys.argv[1:], "s:r:h")
except getopt.GetoptError:
print usage
sys.exit(1)
search_term = None
desired_item_name = None
for opt, arg in opts:
if opt == '-s':
search_term = arg
elif opt == '-r':
desired_item_name = arg
elif opt == '-h':
print usage
return 0
if not desired_item_name or not search_term:
try:
search_term = raw_input('Search Term: ')
desired_item_name = raw_input('Desired Item Name: ')
except EOFError:
return 0
downloader = Downloader()
downloader.download(search_term, desired_item_name)
return 0
def test_fetch_latest_onpoints(self):
basedir = self.standardpath
downloader = Downloader()
downloader.fs = FileSystem()
subs = Subscriptions(downloader, self.standardpath)
asub = subs.find("wbur")
self.assertNotEqual(None,asub)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-s',
'--settings',
dest='settings_path',
metavar='PATH',
default='',
help='path to the settings file')
parser.add_argument(
'-i',
'--ignore',
dest='ignore_path',
metavar='PATH',
default='',
help=
'path to the file that has a list od symbols that should be ignored')
parser.add_argument('mode',
metavar='MODE',
choices=[IMPORT, UPDATE],
help='\'{0}\' or \'{1}\''.format(IMPORT, UPDATE))
options = parser.parse_args()
load_settings(options.settings_path)
symbols = get_symbols(options.ignore_path)
downloader = Downloader(symbols, options.mode)
downloader.run()
def __init__(self):
super(MainWindow, self).__init__()
self.setupUi(self)
# Menu
self.dark_theme.triggered.connect(self.on_dark_theme)
self.theme()
# Terminal
self.term = Terminal()
sys.stdout = self.term
self.term.event.connect(self.update_terminal)
self.dl = Downloader()
Thread(target=self.dl.run, daemon=True).start()
# Browse
self.lineEdit_file_path.setText(self.dl.output_path)
self.lineEdit_file_path.textEdited.connect(self.dl.output)
self.lineEdit_file_path.textEdited.connect(self.dl.output)
self.browse_button.clicked.connect(self.browse)
# Buttons
self.firefox_button.clicked.connect(self.dl.from_firefox)
self.chrome_button.clicked.connect(self.dl.from_chrome)
self.text_button.clicked.connect(
self.from_text) # Intercept signal to add str
# Logic
self.dl.added_row.connect(self.add_row)
self.dl.dl_hook.connect(self.update_row)
self.dl.removed_row.connect(self.remove_row)
def setUpClass(cls):
id = '68428'
d = Downloader()
url = d.goodreads_url(id)
results = d.download_ram(url)
cls.soup = BeautifulSoup(results, "html.parser")
cls.grs = GoodreadsScraper(cls.soup)
def download(name,url):
file_name = urllib.unquote(name)
file_url = urllib.unquote(url)
dst_file = file_name + "." + file_url.split("/")[-1].split(".")[-1]
#Download file
downloader = Downloader()
downloader.downloadall(os.path.join(kodiutils.get_setting("downloadPath"),dst_file),file_url,file_name)
def downloadVideo(self, url, title):
common.log("Trying to download video " + str(url))
# check url
if url.startswith("plugin"):
common.log("Video is not downloadable")
return None
path = common.getSetting("download_path")
if not path:
path = common.browseFolders(common.translate(30017))
common.setSetting("download_path", path)
title = getKeyboard(default=fu.cleanFilename(title), heading="Dragon Streams")
if title == None or title == "":
return None
downloader = Downloader()
downloaded_file = downloader.downloadMovie(url, path, fu.cleanFilename(title), ".flv")
if downloaded_file == None:
common.log("Download cancelled")
else:
common.log("Video " + url + " downloaded to '" + downloaded_file + "'")
return downloaded_file
def downloadVideo(self, url, title):
common.log('Trying to download video ' + str(url))
# check url
if url.startswith('plugin'):
common.log('Video is not downloadable')
return None
path = common.getSetting('download_path')
if not path:
path = common.browseFolders(common.translate(30017))
common.setSetting('download_path', path)
title = getKeyboard(default = fu.cleanFilename(title),heading='Filmkodi')
if title == None or title == '':
return None
downloader = Downloader()
downloaded_file = downloader.downloadMovie(url, path, fu.cleanFilename(title), '.flv')
if downloaded_file == None:
common.log ('Download cancelled')
else:
common.log('Video ' + url + " downloaded to '" + downloaded_file + "'")
return downloaded_file
class PyCast:
def __init__( self, configFile ):
self.configFile = configFile
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#
def fullReset( self ):
self.setting = Settings( self.configFile )
self.grabber = Grabber( self.setting )
self.scraper = Scraper(
self.setting,
self.grabber.xmls
)
self.lister = Lister(
self.setting,
self.scraper.items
)
self.downloader = Downloader(
self.setting,
self.lister.lists,
self.scraper.items
)
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#
def download( self, verbose = False, dryRun = False ):
self.downloader.downloadAll( verbose, dryRun )
class Main:
def __init__(self):
self.soupHandler = SoupHandler(SystemInfo.initUri)
self.downloader = Downloader()
self.fileHandler = FileHandler()
self.path = SystemInfo.path
self.name = SystemInfo.name
def introduce(self):
print("kocw 다운로드 프로그램.")
print("대학교 자체 cdn 전용입니다.")
print("url 수정은 소스코드를 직접 수정하세요.")
def end(self):
print("다운로드 완료 되었습니다.")
def run(self):
self.introduce()
self.fileHandler.safeMkdir("%s/output" % self.path)
self.fileHandler.safeMkdir("%s/output/%s" % (self.path, self.name))
table = self.soupHandler.findTable()
parsedOnClickStrings = self.soupHandler.parseOnClick(table)
self.downloader.startIteratingDownload(parsedOnClickStrings)
self.end()
def setUpClass(cls):
id = '68428'
d = Downloader()
url = d.goodreads_url(id)
results = d.download_ram(url)
cls.soup = BeautifulSoup(results, "html.parser")
cls.grs = GoodreadsScraper(cls.soup)
def __distribute(self):
while self.__isRunning:
if len(self.__threadList) < 3: # download 3 videos simultaneously
if not self.__queue.empty():
dl = Downloader(self.__frame, self.__plm,
self.__queue.get(), self.__dir, self._lock)
self.__threadList.append(dl)
dl.start()
sleep(WAIT_TIME)
for t in self.__threadList:
if t.is_alive(
): # if video is still being downloaded, update status
t.updateStatus()
else: # otherwise, remove it from list for next video to be downloaded
self.__threadList.remove(t)
sleep(WAIT_TIME)
if len(self.__threadList) <= 0 and self.__queue.empty(
): # when every video is completed
with self._lock:
self.__frame.setFinished()
self.__isRunning = False
def main():
dbc = DBController()
dec = Decoder([TLEDecoder(), TLEListDecoder()])
dlc = None
try:
dlc = Downloader()
except DownloaderError as e:
print("failed to initialize downloader: " + str(e))
sys.exit(1)
for esat in dlc.get_data():
sats = []
try:
sats = dec.decode(esat.fmt, esat.data)
except DecoderError as e:
print("failed to decode: " + str(e))
try:
for sat in sats:
dbc.add(sat)
dbc.sync()
except DBError as e:
print("failed to insert into db: " + str(e))
def add_song_to_queue(song_url, source_type):
if source_type == 'youtube':
song_info = Downloader.get_youtube_song_metadata(song_url)
else:
song_info = Downloader.get_soundcloud_song_metadata(song_url)
_shared_queue.append(song_info)
class Dataset:
def __init__(self, settings: Settings, base_path, size, path_csv, is_test):
self.settings = settings
self.base_path = base_path
self.size = int(size)
self.path_csv = path_csv
self.is_test = is_test
if self.size > 0:
self.data = pd.read_csv(self.path_csv).head(self.size)
else:
self.data = pd.read_csv(self.path_csv)
self.downloader = Downloader(settings)
if is_test:
folders_names = ['tmp', '']
else:
folders_names = self.data['label'].unique().tolist() + ['tmp']
self.label_to_dir = create_file_structure(self.base_path, folders_names)
def download(self):
try:
for _, row in self.data.iterrows():
self.downloader.execute(row, self.label_to_dir, test=self.is_test)
finally:
shutil.rmtree(self.label_to_dir['tmp'])
class TestDownloaded(unittest.TestCase):
'''Using mock object to replace real file reading and network accessing'''
def setUp(self):
self.urls = [
"https://www.pornhub.com/view_video.php?viewkey=ph5ebd5cf2658e8",
"https://www.pornhub.com/view_video.php?viewkey=ph5e5866a54b458",
"https://www.pornhub.com/view_video.php?viewkey=ph5e9618d3baddf",
"https://www.pornhub.com/view_video.php?viewkey=ph5ea206352bd49"
]
self.demo_downloader = Downloader(self.urls)
self.filename = "test_downloaded.json"
with open(self.filename, "w"): # create a json file for testing
pass
def test_whether_new_urls_are_downloaded(self):
for url in self.urls:
with self.subTest(url=url):
self.assertEqual(
self.demo_downloader.is_downloaded(url, self.filename),
False)
def test_whether_old_urls_are_downloaded(self):
for url in self.urls:
self.demo_downloader.mark_as_downloaded(url, self.filename)
for url in self.urls:
with self.subTest(url=url):
self.assertEqual(
self.demo_downloader.is_downloaded(url, self.filename),
True)
def tearDown(self):
self.demo_downloader.driver.quit()
os.remove(self.filename)
def get_properties(self):
url = ''
d = Downloader()
print d.get_sub_pages_links('http://www.wp.pl/', '')
pass
def fetchIcon(self, iconName):
iconFileName = None
if self.weatherIconId >= 200:
iconFileName, description = WeatherIconMapper.convertIcon(
self.weatherIconId, self.sunrise.time(), self.sunset.time())
print("Icon file name: {}, Description: {}".format(
iconFileName, description))
if iconFileName is not None:
self.icon = UiUtility.loadWeatherIcon(iconFileName)
self.icon.fill(UiColors.GRAY, special_flags=pygame.BLEND_RGB_ADD)
else:
# This weather icon ID is not mapped to a weather icon. In this case,
# fetch the icon from OpenWeatherMap
downloader = Downloader(None)
# TODO: Do in either a background thread, or a coroutine
url = "http://openweathermap.org/img/w/{}.png".format(iconName)
downloader.download(url)
image = downloader.getData()
# Does image need to be processed before it can be used by Pygame?
memFileObj = io.BytesIO(image)
self.icon = pygame.image.load(memFileObj)
def post(self):
if not request.json:
return {'message': 'Problems parsing JSON'}, 400
if request.json.get('urls', None) is None:
return {'message': 'Validation Failed',
'errors': [{'resource:': 'Package', 'field': 'urls',
'code': 'missing_field'}]}, 422
if not isinstance(request.json['urls'], list):
return {'message': 'Validation Failed',
'errors': [{'resource:': 'Package', 'field': 'urls',
'code': 'invalid'}]}, 422
try:
parent_conn, child_conn = multiprocessing.Pipe()
d = Downloader(child_conn, app.config['BASEPATH'],
request.json['urls'])
d.start()
r = parent_conn.recv()
d.join()
p = Package(ip=request.remote_addr, path=r['zipname'],
date_created=arrow.now().datetime)
db.session.add(p)
db.session.commit()
except Exception as e:
logger.error(e)
return {'package': 'http://{0}/{1}'.format(
app.config['SERVER_ADDR'], r['name'])}, 201
def _networkM3U8(self, url: str) -> None:
url = url.replace("www.", "").removeprefix("https://")
filmName = re.findall(
r"[altadefinizione\..*|altadefinizionecommunity\.net]\/(.*/)", url)
serieName = re.findall(r"seriehd\..*\/(.*/)", url)
if filmName != []:
out = self.output()
self.root.destroy()
film = Film(filmName[0], out)
if film.init[0] == 'created':
Downloader(film.outputFile)
elif film.init[0] == "continuare":
messagebox.showinfo("4K Downloader",
"Download già iniziato, verrà ripreso")
Downloader(film.outputFile)
elif film.init[0] == "sostituire":
s = messagebox.askyesno(
"4K Downloader",
"Nome già esistente ma i file sono diversi.\nSi -> Indicizza file\nNo -> Sostituisci file"
)
if s:
film.outputFile = film.outputFile.duplicate()
else:
film.outputFile.temp().remove()
Data.delete(film.outputFile.name)
Film.initialize(film.outputFile, film.m3u8Path)
Downloader(film.outputFile)
def __init__(self, ):
# - Allow ourselves to use any Downloader methods
Downloader.__init__(self, )
# - Local directory for saving data
self.refcat_dir = self._fetch_refcat_data_directory()
# - Anticipated filepath for compiled refcat code
self.refcat_codename = 'refcat'
self.refcat_filepath = os.path.join(self.refcat_dir,
self.refcat_codename)
# - Set of refcat catalogues on mast
self.downloadable_files = {
'16':
'https://archive.stsci.edu/hlsps/atlas-refcat2/orig/hlsp_atlas-refcat2_atlas_ccd_00-m-16_multi_v1_cat.tbz',
'17':
'https://archive.stsci.edu/hlsps/atlas-refcat2/orig/hlsp_atlas-refcat2_atlas_ccd_16-m-17_multi_v1_cat.tbz',
'18':
'https://archive.stsci.edu/hlsps/atlas-refcat2/orig/hlsp_atlas-refcat2_atlas_ccd_17-m-18_multi_v1_cat.tbz',
'19':
'https://archive.stsci.edu/hlsps/atlas-refcat2/orig/hlsp_atlas-refcat2_atlas_ccd_18-m-19_multi_v1_cat.tbz',
'20':
'https://archive.stsci.edu/hlsps/atlas-refcat2/orig/hlsp_atlas-refcat2_atlas_ccd_19-m-20_multi_v1_cat.tbz',
'code': 'https://archive.stsci.edu/prepds/atlas-refcat2/refcat.c',
'man': 'https://archive.stsci.edu/prepds/atlas-refcat2/refcat.man',
}
class Client:
def __init__(self, config: ConfigurationFile):
self.config = config
self.database = Database(self.config.db_file)
self.organizer = Organizer(self.config, self.config.db_file)
self.downloader = Downloader(self.config.db_file, self.organizer,
self.config)
self.tracker = Tracker(self.config.db_file, self.downloader,
self.config.update_period)
self.tracker.start()
self.downloader.start()
self.organizer.start()
def add_tvshow(self, tvshow_id: int):
tvshow_name = showrss.get_name(tvshow_id)
self.database.put_tvshow(TVShow(tvshow_id, tvshow_name))
def remove_tvshow(self, tvshow_id: int):
self.database.remove_tvshow(tvshow_id)
def list_tvshows(self):
return self.database.tvshows()
def list_episodes(self, state: EpisodeState = None):
return self.database.episodes(state)
def download_progress(self):
return self.downloader.downloads()
def exit(self):
self.tracker.stop()
self.downloader.stop()
def __init__(self):
reload(sys)
sys.setdefaultencoding('UTF-8')
self.title = 'Nada'
self.model = ['luoo 落网', 'echo 回声', 'nada 收藏', '关于']
self.view = 'menu'
self.ctrl = 'menu'
self.offset = 0
self.index = 0
self.step = 10
self.play_id = -1
self.play_vol = -1
self.present = []
self.stack = []
self.player = Player()
self.ui = UI()
self.luoo = Luoo()
self.echo = Echo()
self.downloader = Downloader()
self.database = Database()
self.database.load()
self.collections = self.database.data['collections'][0]
self.screen = curses.initscr()
self.screen.keypad(1)
def main():
args = parse_argument()
try:
if args.socks5[0] and args.socks5[1]:
if args.proxy:
logger.error('invalid proxy protocol count.')
raise SystemExit
socks.set_default_proxy(socks.SOCKS5, args.socks5[0], int(args.socks5[1]),
True, args.socks5[2], args.socks5[3])
socket.socket = socks.socksocket
except Exception as e:
logger.error('invalid socks5 proxy arguments.')
raise SystemExit
t = Thread(args.board, args.thread)
if not args.downloading:
thread_info = t.thread_info()
logger.info('/{}/ - {} - {}'.format(args.board, thread_info['sub'], const.BOARDS[args.board]))
logger.info('total images - {}'.format(thread_info['images']))
else:
downloader = Downloader(path=args.path, threads=args.threads, timeout=args.timeout,
is_thumb=args.thumb)
q = t.detail_queue()
downloader.fetch(q)
class Spider(object):
start_urls = ['']
def __init__(self):
self.count = 0 # 用于记录xls的行数
self.downloader = Downloader()
self.next_url = None
self.mutex = Lock() # 创建一个锁 用于文件的读写
def __enter__(self):
# 创建 xls 文件对象
self.wb = xlwt.Workbook()
# 新增一个表单
self.sheet = self.wb.add_sheet('Sheet 1')
self.start_request()
def __exit__(self, exc_type, exc_val, exc_tb):
self.wb.save('data.xls')
def start_request(self):
for url in self.start_urls:
self.downloader.add_job(priority_number=1,
job=url,
handle=self.parse)
self.downloader.create_threads()
def parse(self, response):
pass
def get(self, type, img):
retImg = self.__createYzmImage(img, type)
d = Downloader()
data = dict(
JsdmYzmPlugin.jsdm_info, **{
"captchaData": retImg,
"captchaType": 1023,
"captchaMinLength": 0,
"captchaMaxLength": 0
})
dataPost = json.dumps(data)
res = d.request_data_from_url('https://v2-api.jsdama.com/upload',
'post',
dataPost,
timeout=-1)
j_res = json.loads(res)
dataret = j_res.get('data')
if dataret:
recognition = dataret['recognition']
else:
recognition = None
self.getdmResult = res
print('验证码自动识别：[', recognition, ']')
return recognition
def __init__(self, config):
"""Weibo类初始化"""
self.config = config
# change cookie from string to dict
if type(self.config['cookie']) == type(u''):
self.config['cookie'] = {
t.strip().split("=")[0]: t.strip().split("=")[1]
for t in self.config['cookie'].split(";")
}
if type(self.config['user_id_list']) == type(u""):
user_id_list = self.config['user_id_list']
if not os.path.isabs(user_id_list):
user_id_list = os.path.split(
os.path.realpath(__file__))[0] + os.sep + user_id_list
self.config['user_id_list'] = user_id_list
with open(self.config['user_id_list'], 'rb') as f:
lines = f.read().splitlines()
lines = [line.decode('utf-8') for line in lines]
self.config['user_id_list'] = [
line.split(' ')[0] for line in lines if
len(line.split(' ')) > 0 and line.split(' ')[0].isdigit()
]
if type(self.config['since_date']) == type(0):
self.config['since_date'] = str(
date.today() - timedelta(self.config['since_date']))
self.validator = Validator(self.config)
self.validator.validate()
self.printer = Printer()
self.writer = Writer(self.config)
self.downloader = Downloader(self.config)
self.parser = Parser(self.config)
def __init__(self):
super(Main, self).__init__(load_config, NAME_FILE_CONFIG_PATH, NAME_FILE_LOG_PATH, NAME_FILE_CSV_PATH)
self.loading_args()
self.log_configuration()
# General variable
self.config = self.loading_file_config()
self.errors = []
self.fields_csv = self.config.get("GENERAL", "fields_csv").split(",")
list_links = parser_cvs(self.args.csv, self.fields_csv)
http = Downloader()
report = Report()
for data in list_links:
download = http.download(data["link"])
report.add_elements(download)
report.print_report()
logging.info("Script Completado.")
class Listener(Thread):
def __init__(self, target):
Thread.__init__(self)
self._address = 'amqp://*****:*****@orangutan.rmq.cloudamqp.com/qxyexflk'
self._queue = 'mytube'
self._timeout = 5
self._downloader = Downloader(target)
def run(self):
parameters = pika.URLParameters(self._address)
parameters.socket_timeout = self._timeout
connection = pika.BlockingConnection(parameters)
channel = connection.channel()
channel.queue_declare(queue=self._queue, durable=True)
channel.basic_qos(prefetch_count=1)
channel.basic_consume(self.on_message, queue=self._queue)
channel.start_consuming()
print("Waiting for messages...")
def on_message(self, channel, method, properties, body):
print('Received Message')
try:
self._downloader.download(body.decode('ascii'))
channel.basic_ack(delivery_tag=method.delivery_tag)
print('Processed Message')
except:
channel.basic_nack(delivery_tag=method.delivery_tag, requeue=False)
print('Failed To Process')
def __init__(self, plugin, layerDef, creditVisibility=1):
QgsPluginLayer.__init__(self, TileLayer.LAYER_TYPE, layerDef.title)
self.plugin = plugin
self.iface = plugin.iface
self.layerDef = layerDef
self.creditVisibility = 1 if creditVisibility else 0
# set custom properties
self.setCustomProperty("title", layerDef.title)
self.setCustomProperty("credit",
layerDef.credit) # TODO: need to remove
self.setCustomProperty("serviceUrl", layerDef.serviceUrl)
self.setCustomProperty("yOriginTop", layerDef.yOriginTop)
self.setCustomProperty("zmin", layerDef.zmin)
self.setCustomProperty("zmax", layerDef.zmax)
if layerDef.bbox:
self.setCustomProperty("bbox", layerDef.bbox.toString())
self.setCustomProperty("creditVisibility", self.creditVisibility)
# create a QgsCoordinateReferenceSystem instance if plugin has no instance yet
if plugin.crs3857 is None:
plugin.crs3857 = QgsCoordinateReferenceSystem(3857)
self.setCrs(plugin.crs3857)
if layerDef.bbox:
self.setExtent(
BoundingBox.degreesToMercatorMeters(
layerDef.bbox).toQgsRectangle())
else:
self.setExtent(
QgsRectangle(-layerDef.TSIZE1, -layerDef.TSIZE1,
layerDef.TSIZE1, layerDef.TSIZE1))
self.setValid(True)
self.tiles = None
self.useLastZoomForPrint = False
self.canvasLastZoom = 0
self.setTransparency(LayerDefaultSettings.TRANSPARENCY)
self.setBlendModeByName(LayerDefaultSettings.BLEND_MODE)
self.setSmoothRender(LayerDefaultSettings.SMOOTH_RENDER)
self.downloader = Downloader(self)
self.downloader.userAgent = "QGIS/{0} QuickMapServices Plugin".format(
QGis.QGIS_VERSION
) # , self.plugin.VERSION) # not written since QGIS 2.2
self.downloader.DEFAULT_CACHE_EXPIRATION = QSettings().value(
"/qgis/defaultTileExpiry", 24, type=int)
QObject.connect(self.downloader,
SIGNAL("replyFinished(QString, int, int)"),
self.networkReplyFinished)
# multi-thread rendering
self.eventLoop = None
QObject.connect(self, SIGNAL("fetchRequest(QStringList)"),
self.fetchRequest)
if self.iface:
QObject.connect(self, SIGNAL("showMessage(QString, int)"),
self.showStatusMessageSlot)
QObject.connect(
self, SIGNAL("showBarMessage(QString, QString, int, int)"),
self.showBarMessageSlot)
def getRssInfo(rss):
downLoader = Downloader()
try:
(t_url, header, html) = downLoader.open(rss)
print "download ", rss
except Exception, msg:
getLogger().error("feed download error : %s %s", msg, rss)
return None
def __download(self):
downloader = Downloader(self.url, self.downloadFile, \
OnlineUpdater.__getModifiedTime(self.recipeFile))
try:
retval = downloader.download()
finally:
downloader.close()
return retval
def test_id_from_downloads(self):
query = 'Well of Ascension'
d = Downloader()
url = d.goodreads_id_query(query)
results = d.download_ram(url)
soup = BeautifulSoup(results, 'html.parser')
gs = GoogleScraper(soup)
self.assertEqual(gs.id, '68429')
def main():
if len(sys.argv) <= 1:
print "No url given!"
sys.exit(1)
finder = FuxFinder(sys.argv[1])
downloader = Downloader(finder.find_movie(), finder.find_title() + '.mp4')
downloader.download()
class Project(object):
"""docstring for Project"""
def __init__(self, configs, path):
super(Project, self).__init__()
self.name = 'name' in configs and configs['name'] or os.path.basename(path)
self.configs = configs['configs']
self._path = path
self._spiders_cls = {}
self._spiders_cfg = {}
if 'spiders' in self.configs:
cfgs = self.configs['spiders']
for scfg in cfgs:
name = scfg.get('id') or scfg.get('name')
self._spiders_cfg[name] = scfg
self._downloader = None
def load_spider(self, name, params=None):
spider_cls = None
if name in self._spiders_cls:
spider_cls = self._spiders_cls[name]
else:
for fn in os.listdir(self._path):
if fn in ('.', '..'): continue
fp = os.path.join(self._path, fn)
if os.path.isfile(fp):
mod = loader.load_module(fp)
if mod and name in mod:
spider_cls = mod[name]
self._spiders_cls[name] = spider_cls
if spider_cls:
s = spider_cls()
p = self._spiders_cfg.get(name, {})
if params:
p.update(params)
if p:
s.set_params(**p)
return s
return None
def get_downloader(self):
if self._downloader is None:
thread = self.configs.get('thread', 4)
headers = self.configs.get('headers')
self._downloader = Downloader(thread)
if headers:
self._downloader.add_headers(headers)
return self._downloader
def get_crawler(self):
crawler = Crawler(self)
return crawler
def add(self, path, url, file_name):
""""""
if file_name not in self.active_downloads:
if self.get_slot():
logger.info("Started %s" % (file_name))
th = Downloader(path, url, file_name, self.link_parser)
self.active_downloads[file_name] = th
th.start()
return True
def ncd_loop(doInit, dlThreadNum):
ndutil.setTimezone()
#read config
cnfManager = CnfManager()
cnfManager.load('./ndc.cnf')
cnfData = cnfManager.getCnfData()
#check dirs
ndutil.enableDir(cnfData['dirWorking'])
ndutil.enableDir(cnfData['dirStore'])
#ndlcom
logger = Logger('nDroid-Crawler', cnfData['ndlComHost'], cnfData['ndlComPort'])
logger.logger('Initiating')
dbManager = DbManager(cnfData['dbHost'], cnfData['dbUser'], cnfData['dbPass'], cnfData['dbName'])
if doInit:
dbManager.create_table()
os.system('rm -f %s/*' % cnfData['dirWorking'])
os.system('rm -f %s/*' % cnfData['dirStore'])
#logger.logger('Customizing Spiders')
#spiderGenerator = SpiderGenerator('template', 'spider/spiders')
#for spider in cnfData['spiders']:
# spiderGenerator.gen_spider(spider, cnfData[spider]['startPage'], cnfData[spider]['stopPage'])
rpQueue = Queue()
pdQueue = Queue()
dpQueue = Queue()
pdLock = threading.Lock()
rpcMonitor = RpcMonitor(logger, cnfData['rpcPort'], cnfData['rpcAuth'], 'RpcMonitor')
rpcMonitor.setGlobalInfo(ndutil.getCurrentTimeStr(), 'Standalone', dlThreadNum)
rpcMonitor.setDownloadTotal(pdQueue.qsize())
rpcMonitor.setPdQueueSize(pdQueue.qsize())
botScheduler = BotScheduler(logger, rpcMonitor, cnfData['spiders'], cnfData['spiderCnfs'], 'BotScheduler')
receiver = Receiver(logger, rpcMonitor, rpQueue, cnfData['receiverPort'], 'Receiver')
preProcessor = PreProcessor(logger, rpcMonitor, rpQueue, pdQueue, pdLock, dbManager, 'PreProcessor')
downloader = Downloader([logger, rpcMonitor, pdQueue, dpQueue, pdLock, dlThreadNum, cnfData['dirWorking']], 'Downloader')
processor = Processor(logger, rpcMonitor, dpQueue, pdLock, pdQueue, dbManager, cnfData['dirWorking'], cnfData['dirStore'], 'Processor')
logger.logger('Starting Threads')
rpcMonitor.start()
botScheduler.start()
receiver.start()
preProcessor.start()
downloader.start()
processor.start()
processor.join()
downloader.join()
preProcessor.join()
receiver.join()
botScheduler.join()
rpcMonitor.join()
def download(self, args):
downloader = Downloader()
if args.name == "All":
for configFile in os.listdir("config/"):
res = downloader.download("config/" + configFile)
self.printResult(res, configFile.split(".")[0])
else:
res = downloader.download("config/" + args.name + ".json")
self.printResult(res, args.name)
class DriDaemon(DaemonBase):
"""
Manages all functions that need to take place on the router
on a regular basis
"""
def __init__(self, options):
DaemonBase.__init__(self, options)
self.kill_switch = False
self.loops = 0
self.downloader = Downloader(options)
self.uploader = Uploader(options)
self.allowed_traffic = self.downloader.get_allowed_traffic()
self.policy_mgr = PolicyMgr("/tmp/dnsmasq.log", self.allowed_traffic,
self.options)
self.policy_mgr.prep_system()
self.policy_mgr.initial_load()
self.policy_mgr.rotate_log()
def main_loop(self):
"""
Runs forever. We're a daemon
"""
self.log('Starting dri...')
while not self.kill_switch:
start_time = time.time()
while time.time() < (start_time + INTER_LOOP_SLEEP):
try:
has_more = self.policy_mgr.check_for_new_stuff()
if not has_more:
time.sleep(1)
except (PolicyMgrException, CommandException):
self.log('Help! Policy Manager')
if MAX_LOOPS:
self.loops += 1
if self.loops > MAX_LOOPS:
sys.exit(0)
try:
self.downloader.get_addresses()
allowed_traffic = self.downloader.get_allowed_traffic()
self.policy_mgr.process_new_allowed(allowed_traffic)
log_open_files("downloader")
except (DownloadException, CommandException):
self.log('Help! Downloading')
try:
self.uploader.upload_arp_table()
log_open_files("uploader")
except (UploadException, CommandException):
self.log('Help! Uploading')
#print "I LIVE"
def terminate(self):
self.kill_switch = True
print "dying"
def donwload_playlist(playlist):
youtube = Youtube_List()
lista = youtube.list_playlist(playlist)
#Check if some files are already downloaded
list_to_download = youtube.getDiferences(playlist,lista)
download = Downloader()
list_errors = download.download_list(list_to_download)
#Create the logs files
youtube.save_playlist_on_file(playlist + ".err",list_errors)
youtube.save_playlist_on_file(playlist,lista)
def pack(data):
downloads_folder = fetch_folder_path(DOWNLOADS_FOLDER)
archives_folder = fetch_folder_path(ARCHIVES_FOLDER)
downloader = Downloader(downloads_folder, data)
if downloader.download():
archivator = Archivator(downloader.destination, os.path.join(archives_folder, "{}.zip".format(downloader.uid)))
archivator.archive()
return os.path.basename(archivator.destination)
def downloadAll(self):
utils.mkdir(baseDirPath, self.comicTitle)
for i in range(threadCount):
t = Downloader()
t.start()
for url in self.chapterURLs[:2]:
getChapterInfo(url, self.comicTitle)
time.sleep(1)
Downloader.isStop = True
Tkinter.Label(self.window, text=u"下载完成").pack()
def getTistoryId(url):
downLoader = Downloader()
attr_dict = dict()
attr_dict["tid"] = "livere_blogurl = '****.tistory.com';"
attr_dict["tid2"] = """__addParam("author","****");"""
try:
(t_url, header, html) = downLoader.open(url)
print "download", url
except Exception, msg:
getLogger().error("feed download error : %s %s", msg, rss)
return None
def test_WithCacheDirCAndEpT1x1DownloadingDirA_MoveFromCBarATpCBarT1x1(
self, mock_move):
config = ConfigurationFile()
config.cache_dir = 'c'
downloader = Downloader(None, None, config)
episode = Episode('t', 1, 1, None, None)
downloading_dir = 'a'
downloader._moveto_downloaded_dir(downloading_dir, episode)
mock_move.assert_called_once_with(
'c/a', 'c/t1x1')
def start():
year = input("Please give the year you want to download: ")
parser = NostalgiaMachineParser()
songs = parser.get_list_of_songs_from_year(year)
dl = Downloader()
#print(songs[0])
for s in songs:
print("Downloading: " + str(s))
dl.get_song(s)
def http_downloader(url):
from downloader import Downloader
downer = Downloader()
hostpagehtml = downer.getContent(url)
if hostpagehtml:
root = fromstring(hostpagehtml)
msgdivs = root.xpath("//div[@class='poi-tile__info']")
for msg in msgdivs:
msgdiv = tostring(msg, encoding='utf-8')
print msgdiv
ParseMsg(msgdiv)
pass
print url, '\n is over'
def run(self):
print 'ENGINE STSRT'
self.is_running=True
for i in range(settings.S_thre_num):
t=Downloader(i,self.scheduler,self)
self.threads.append(t)
for t in self.threads:
t.start()
time.sleep(10)
for t in self.threads:
t.join()
self.stop()