def run(self):
'''Task dispatcher loop'''
for job in iter(self.queue.get, self.QUIT):
status_data = Downloader.ClipData()
status_data.URL = job.url
status_data.status = Downloader.Status.INPROGRESS
self.progress.notify(status_data)
try:
job.download()
status_data = Downloader.ClipData()
status_data.URL = job.url
status_data.status = Downloader.Status.DONE
self.progress.notify(status_data)
self.queue.task_done()
except Exception as e:
status_data = Downloader.ClipData()
status_data.URL = job.url
status_data.status = Downloader.Status.ERROR
self.progress.notify(status_data)
self.queue.task_done()
self.queue.put(self.CANCEL)
for job in iter(self.queue.get, self.CANCEL):
job.cancel()
self.queue.task_done()
self.queue.task_done()
def main():
delay = 0.5 # First iteration will set delay=1 or multiply by 2
num_short_delays = 0
while True:
try:
Downloader.status()
if Downloader.run():
# files were successfully downloaded
delay=1
num_short_delays = 0
#elif num_short_delays <= 8:
#elif num_short_delays <= 4:
# No files successfully download this iteration
# Increase sleep time
# delay = min((delay*2, 4))
# num_short_delays += 1
else:
delay = min((delay*2,32))
except Exception, e:
if config.email.send_on_crash:
msg = '*** Downloader has crashed! ***\n\n'
msg += 'Fatal error occured while running downloader: %s\n\n' % str(e)
msg += ''.join(traceback.format_exception(*sys.exc_info()))
notification = mailer.ErrorMailer(msg, subject="Downloader crash!")
notification.send()
sys.stderr.write("Fatal error occurred!\n")
raise
print "Will sleep for %d seconds" % (config.background.sleep*delay)
sys.exit()
time.sleep(config.background.sleep*delay)
def run(start, end):
i = start
try:
results = select_paper_data(i, end)
for result in results:
i += 1
print i
if i > end:
return
detail_url = "%s&time=%.f" % (result[0], time.time() * 1000)
source_url = result[1]
category = result[2]
data_h = result[3]
device_name = get_device_name(data_h)
device_path = "%s/%s/" % (root_path, device_name)
suffix = source_url.split(".")[-1]
if not os.path.exists(device_path):
os.mkdir(device_path)
category_path = "%s/%s/%s" % (root_path, device_name, category)
if not os.path.exists(category_path):
os.mkdir(category_path)
max_img_path = "%s/%s/%s/%s_max.%s" % (
root_path, device_name, category,
Downloader.get_md5(source_url), suffix)
min_img_path = "%s/%s/%s/%s_min.%s" % (
root_path, device_name, category,
Downloader.get_md5(source_url), suffix)
print max_img_path
print min_img_path
print detail_url
download(detail_url, max_img_path)
download(source_url, min_img_path)
except Exception:
run(i, end)
def testgetHtml(self):
product = input("Enter prodcut name:")
f = d.FlipkartDownloader(product) # encode will be caleed in here.
content = d.getHtml(f.link)
print(content)
with open("temp.html", "wb") as f:
f.write(content)
def start(login_data):
with requests.Session() as session:
session.post(login_url, login_data)
ids = Suppe.getCourseID(session)
Downloader.dowload(ids, session)
def run(start):
i = start
try:
results = select_paper_data(i, 100000)
for result in results:
i += 1
print i
detail_url = result[0]
source_url = result[1]
category = result[2]
data_h = result[3]
device_name = get_device_name(data_h)
device_path = "%s/%s/" % (root_path, device_name)
suffix = detail_url.split(".")[-1]
if not os.path.exists(device_path):
os.mkdir(device_path)
category_path = "%s/%s/%s" % (root_path, device_name, category)
if not os.path.exists(category_path):
os.mkdir(category_path)
max_img_path = "%s/%s/%s/%s_max.%s" % (
root_path, device_name, category,
Downloader.get_md5(source_url), suffix)
min_img_path = "%s/%s/%s/%s_min.%s" % (
root_path, device_name, category,
Downloader.get_md5(source_url), suffix)
print max_img_path
print min_img_path
main(max_img_path, detail_url)
main(min_img_path, source_url)
except Exception:
run(i)
def downloader_test():
global dl
ss = SvtStream(json.load(open('test/episode.json')))
ss.downloadcommand = "/bin/echo %(url)s %(output)s"
dl = Downloader()
di = DownloadItem(ss, config['output_dir'])
assert di.command.args == [
u'/bin/echo',
u'http://svtplay3q-f.akamaihd.net/i/world/open/20140210/1322836-005A/LABYRINT-005A-b6149effda4e5c1d_,900,348,564,1680,2800,.mp4.csmil/index_4_av.m3u8?null=',
u'/tmp/labyrint-del-5-av-10.mp4'
]
dl.add(di)
assert di.command.status == ("Queued" or "Running")
count = 0
maxwait = 30
while di.command.status != "Completed" and count < maxwait:
# Wait for thread to finish
time.sleep(0.1)
count += 1
assert di.command.status == "Completed"
assert di.command.stdout.rstrip(
) == "http://svtplay3q-f.akamaihd.net/i/world/open/20140210/1322836-005A/LABYRINT-005A-b6149effda4e5c1d_,900,348,564,1680,2800,.mp4.csmil/index_4_av.m3u8?null= /tmp/labyrint-del-5-av-10.mp4"
def update():
logging.info(u"Cron job started")
Downloader.download_and_insert_last_well_completion()
move_data_to_wells_apis()
check_and_create_table_if_not_exist()
# clear_last_year_table()
with open('cron_apis_by_year','w+') as fh:
year = date.today().year
Downloader.check_necessary_tables_and_create_if_not_exists(year)
download_and_insert_data_by_all_apis_by_year(year, fh)
logging.info(u"Cron job over")
def schedule(iter_start, iter_end):
for iter in range(iter_start, iter_end, batch_size):
try:
Downloader.download_batch(iter,
iter + batch_size,
"DownloadErrorLog_%d_to_%d.txt" % (iter, iter + batch_size - 1),
target_path='D:\Downloads\midi',
working_directory='D:/code/repository/chromedriver/chromedriver.exe')
except:
err_log_file = open("DownloadErrorLog_%d_to_%d.txt" % (iter, iter + batch_size - 1), "a+")
err_log = "Download failed: " + str(iter)
err_log_file.write(err_log + "\n")
print(err_log)
def run(self, url, html):
if not url.find("?"):
return False
BOOLEAN_TESTS = (" AND %d=%d", " OR NOT (%d=%d)")
DBMS_ERRORS = {
"MySQL": (r"SQL syntax.*MySQL", r"Warning.*mysql_.*",
r"valid MySQL result", r"MySqlClient\."),
"PostgreSQL": (r"PostgreSQL.*ERROR", r"Warning.*\Wpg_.*",
r"valid PostgreSQL result", r"Npgsql\."),
"Microsoft SQL Server":
(r"Driver.* SQL[\-\_\ ]*Server", r"OLE DB.* SQL Server",
r"(\W|\A)SQL Server.*Driver", r"Warning.*mssql_.*",
r"(\W|\A)SQL Server.*[0-9a-fA-F]{8}",
r"(?s)Exception.*\WSystem\.Data\.SqlClient\.",
r"(?s)Exception.*\WRoadhouse\.Cms\."),
"Microsoft Access":
(r"Microsoft Access Driver", r"JET Database Engine",
r"Access Database Engine"),
"Oracle":
(r"\bORA-[0-9][0-9][0-9][0-9]", r"Oracle error", r"Oracle.*Driver",
r"Warning.*\Woci_.*", r"Warning.*\Wora_.*"),
"IBM DB2": (r"CLI Driver.*DB2", r"DB2 SQL error", r"\bdb2_\w+\("),
"SQLite":
(r"SQLite/JDBCDriver", r"SQLite.Exception",
r"System.Data.SQLite.SQLiteException", r"Warning.*sqlite_.*",
r"Warning.*SQLite3::", r"\[SQLITE_ERROR\]"),
"Sybase": (r"(?i)Warning.*sybase.*", r"Sybase message",
r"Sybase.*Server message.*"),
}
_url = url + "%29%28%22%27" #)("'的url编码
#get返回的html
_content = Downloader.get(_url)
for (dbms, regex) in ((dbms, regex) for dbms in DBMS_ERRORS
for regex in DBMS_ERRORS[dbms]):
if re.search(regex, _content):
return True
content = {}
content["origin"] = Downloader.get(url)
for test_payload in BOOLEAN_TESTS:
RANDINT = random.randint(1, 255)
_url = url + test_payload % (RANDINT, RANDINT)
content["true"] = Downloader.get(_url)
_url = url + test_payload % (RANDINT, RANDINT + 1)
content["false"] = Downloader.get(_url)
if content["origin"] == content["true"] != content["false"]:
return "sql found: %" % url
def crawland(url):
downloader = Downloader.Downloader()
html_cont, code = downloader.download(url)
soup = BeautifulSoup(
html_cont,
'html.parser',
)
records = soup.select("table.tab4 > tr")
lands = []
for record in records:
data = {}
r2 = record.select('td') # 这里使用了再次解析
data['landNo'] = (r2[0].contents[0].get('title'))
data['href'] = (r2[0].contents[0].get('href'))
data['use'] = (r2[1].contents[1].get('title'))
data['address'] = (r2[2].contents[0].get('title'))
data['date'] = (r2[3].get('title'))
data['acreage'] = (r2[4].get('title'))
if len(r2) == 8: # 对于工业用地，缺少“建筑面积”一列
data['floorArea'] = (r2[5].get('title'))
data['price'] = (r2[6].get_text())
data['user'] = (r2[7].get('title'))
else:
data['floorArea'] = data['acreage']
data['price'] = (r2[5].get_text())
data['user'] = (r2[6].get('title'))
lands.append(data)
return lands
def get_contents(pageIndex, category, type_id, device_name, device_id):
url = "http://app.tongbu.com/bizhi/%s-cateshow-%s-%s" % (
device_name, type_id, pageIndex)
page = get_page(url)
#print page
if page is None:
return
next_re = re.compile('<a.*?">下一页.*?</a>', re.S)
result = re.search(next_re, page)
if result is None:
return None
pattern = re.compile(
'<li>.*?class="pic".*?<img alt="(.*?)".*?src="(.*?)".*?<div class="downnum">(.*?)<.*?href="(.*?)">',
re.S)
items = re.findall(pattern, page)
data_str = ""
for item in items:
source_url = item[1]
detail_url = item[3]
data_w = device_dic[device_id][1]
data_h = device_dic[device_id][0]
favorite = item[2].split(":")[-1]
tag = item[0]
like = favorite
author = 'test321546'
md5 = Downloader.get_md5(source_url)
#print data_w, data_h, detail_url, source_url, tag, favorite, md5, '\n'
data_str += get_insert_data(data_w, data_h, detail_url, source_url,
tag, like, favorite, author, category, md5)
#contents.append([data_w, data_h, detail_url, source_url, tag, like, favorite, comment, author])
return data_str
def __init__(self):
# Check the OS first
if not str(sys.platform) == "darwin":
self.head("Incompatible System")
print(" ")
print("This script can only be run from macOS/OS X.")
print(" ")
print("The current running system is \"{}\".".format(sys.platform))
print(" ")
self.grab("Press [enter] to quit...")
print(" ")
exit(1)
self.dl = Downloader.Downloader()
self.web_drivers = None
self.os_build_number = None
self.os_number = None
self.wd_loc = None
self.sip_checked = False
self.installed_version = "Not Installed!"
self.bdmesg = os.path.join(os.path.dirname(os.path.realpath(__file__)), "bdmesg")
if not os.path.exists(self.bdmesg):
self.bdmesg = None
self.get_manifest()
self.get_system_info()
def link_crawler(seed_url, link_regex, max_depth=2, user_agent='wswp', proxies=None, scrape_callback=None, catch=None):
crawler_queue = [seed_url]
seen = {seed_url: 0}
rp = get_robots(seed_url)
D = Downloader(delay=5, user_agent=user_agent, proxies=proxies, num_tries=1, catch=catch)
while crawler_queue:
url = crawler_queue.pop()
depth = seen[url]
if rp.can_fetch('GoodCrawler', url):
html = D(url)
if html is None:
continue
if depth != max_depth:
for link in get_links(html):
if re.match(link_regex, link):
link = urlparse.urljoin(seed_url, link)
if link not in seen:
seen[link] = depth + 1
crawler_queue.append(link)
links = []
html1 = D(link)
if scrape_callback:
links.append(scrape_callback(link, html1) or [])
else:
print 'Blocked by robot.txt:', url
def post_experiment():
'''Experiment where we try generating a series of posts'''
Poster = DL.PostDownloader()
Selector = PS.PostSelector()
if not os.path.exists('experiment_posts'):
os.mkdir('experiment_posts')
for postnumber in range(100):
try:
posttype = np.random.choice(['news','general'],p=[0.2,0.8])
print('Running download')
downloaded, story, download_type, pt = Poster.rundownload(posttype=posttype)
print('Selecting post meta')
selected_post_meta = Selector.select_post(download_type=download_type)
print(downloaded,story,download_type,pt,selected_post_meta)
post_image = 'post_%03d.jpg' %postnumber
os.system('mv debug_test.png experiment_posts/%s' %post_image)
print('Done post number %i' %postnumber)
print('----------------------------------------------------------------')
except:
print('Wait 10 seconds, try again')
time.sleep(10)
def download():
for down in downloads:
(readerLink, anime, saison, episode) = down
print("| Downloading : {} > {} > {}".format(anime, saison, episode))
readerLink = d.download(readerLink)
if(readerLink == ''):
downloadLog.log('| Error to download : {} > {} > {}'.format(anime, saison, episode))
continue
stream = r.get(readerLink, stream=True)
if not os.path.exists(folder + anime):
os.mkdir(folder + anime)
if not os.path.exists(folder + anime + "/" + saison):
os.mkdir(folder + anime + "/" + saison)
filePath = folder + anime + "/" + saison + "/" + episode + ".mp4"
if(os.path.exists(filePath)):
continue
f = open(filePath, 'wb')
try:
f.write(stream.content)
except:
downloadLog.log('| ERROR during (writing error) download : {} > {} > {}'.format(anime, saison, episode))
print(c.Back.RED + "| Download failed !" + c.Back.BLACK)
def setUp(self):
self.downloader = Downloader.Downloader(verbosity=DEBUG)
self.transferer = Transferer.Transferer(id="transferer",
verbosity=DEBUG_TRANSFERER)
self.configFileTransmission = "tests/downloaderTransmission.json"
self.configFileSynology = "tests/downloader3.json"
self.configFileNone = "tests/downloaderNone.json"
self.configFileTvShowSchedule = "tests/tvShowSchedule.json"
self.testTransmission = os.path.isfile(self.configFileTransmission)
self.tmpdir1 = unicode(tempfile.mkdtemp())
self.tmpdir2 = unicode(tempfile.mkdtemp())
self.transfererData = {
"enable": True,
"source": {
"path": self.tmpdir1,
"protocol": "file"
},
"destination": {
"path": self.tmpdir2,
"protocol": "file"
},
"delete_after": False,
"pathPattern": "{seriesname}/season {seasonnumber}"
}
self.ts = torrentSearch.torrentSearch(
id="torrentSearch",
dataFile="tests/torrentSearch2.json",
verbosity=DEBUG_TORRENT_SEARCH)
self.t = myTvDB.myTvDB(debug=DEBUG, cache=False)
def add(self, callback, url):
'''Add new task to queue'''
status_data = Downloader.ClipData()
status_data.URL = url
status_data.status = Downloader.Status.PENDING
self.queue.put(Job(callback, url))
self.progress.notify(status_data)
def test_synology_start_fail(self):
self.d = Downloader.Downloader(verbosity=DEBUG)
self.d.loadConfig(self.configFile3)
with self.assertLogs(logger=self.d.logger,level='ERROR'):
with self.assertRaises(Downloader.DownloaderSlotNotExists):
self.d.start_torrent('dbid_161')
def get_charts(self):
# Make instance of downloader
self.downloader = Downloader.Downloader(
self.system, self.settings.resources_list,
self.settings.dest_folder, self.progress_dialog, self.status_bar,
self.icao_edit.text().upper().split(), self.settings.view_notify,
self.settings.open_file, self.system_tray)
# Set downloader thread
self.downloader_thread = QThread(self)
# On thread start will execute download method
self.downloader_thread.started.connect(self.downloader.download)
# Move downloader to thread
self.downloader.moveToThread(self.downloader_thread)
# Connect PyQt5 slot and signals
self.downloader.start_download_process.connect(
self.start_download_process)
self.downloader.show_dialog.connect(self.show_dialog)
self.downloader.set_progress.connect(self.set_progress)
self.downloader.reset_dialog.connect(self.progress_dialog.reset)
self.downloader.finish_download_process.connect(
self.finish_download_process)
# Start thread
self.downloader_thread.start()
def __init__(self):
super().__init__()
# Menu Window
self.help_window = Mw.HelpWindow()
self.setting_window = Mw.MenuWindow()
# Ini File Setting
self.config = configparser.ConfigParser()
self.config['setting'] = {}
# Youtube Downloader
self.yt_downloader = Downloader.Downloader()
# Api Setting
self.api_lb = Qt.QLabel("Api key: ")
self.api_box = Qt.QHBoxLayout()
self.api_key = Qt.QLineEdit()
self.api_btn = Qt.QPushButton("적용")
self.is_valid_api = False
# Directory to download
self.dir_box = Qt.QHBoxLayout()
self.dir_path = Qt.QLineEdit()
self.dir_button = Qt.QPushButton("...")
# Search
self.search_label = Qt.QLabel("검색어: ")
self.search_line = Qt.QLineEdit(self)
self.search_btn = Qt.QPushButton("검색")
self.search_box = Qt.QHBoxLayout()
# Download Option
self.opt_lb = Qt.QLabel("검색 방식: ")
self.opt_url = Qt.QRadioButton("주소")
self.opt_word = Qt.QRadioButton("검색어")
self.is_url = True
# Result
self.scrollBox = Qt.QScrollArea()
self.download_btns = []
self.download_urls = []
self.initUI()
def __init__(self, userStorageDirectory, torrentUrl, params={}):
print '!!!!!!!!!!!!!!!!!! BORN ' + self.__class__.__name__
self.userStorageDirectory = userStorageDirectory
self.torrentUrl = torrentUrl
xbmc.Player.__init__(self)
log("[TorrentPlayer] Initalized")
self.params = params
self.get = self.params.get
self.contentId = int(self.get("url"))
self.torrent = Downloader.Torrent(self.userStorageDirectory,
self.torrentUrl,
self.torrentFilesDirectory).player
try:
if self.get("url2"):
self.ids_video = urllib.unquote_plus(
self.get("url2")).split(',')
else:
self.ids_video = self.get_ids()
except:
pass
self.init()
self.setup_torrent()
if self.buffer():
while True:
if self.setup_play():
debug('************************************* GOING LOOP')
self.torrent.startSession()
self.torrent.continueSession(self.contentId)
self.loop()
else:
break
debug('************************************* GO NEXT?')
if self.next_dl and self.next_dling and isinstance(
self.next_contentId, int) and self.iterator == 100:
self.contentId = self.next_contentId
continue
debug('************************************* NO! break')
break
self.torrent.stopSession()
self.torrent.threadComplete = True
self.torrent.checkThread()
if '1' != self.__settings__.getSetting(
"keep_files"
) and 'Saved Files' not in self.userStorageDirectory:
xbmc.sleep(1000)
clearStorage(self.userStorageDirectory)
else:
if self.seeding_status:
showMessage(
self.localize('Information'),
self.localize(
'Torrent is seeding. To stop it use Download Status.'),
forced=True)
else:
if self.seeding: self.db_delete()
showMessage(self.localize('Information'),
self.localize('Torrent downloading is stopped.'),
forced=True)
def test_transmission_start_fail(self): self.d = Downloader.Downloader(verbosity=DEBUG) self.d.loadConfig(self.configFileTransmission) with self.assertLogs(logger=self.d.logger,level='ERROR'): with self.assertRaises(Downloader.DownloaderSlotNotExists): self.d.start_torrent(5) self.assertEqual(len(wwwoman.wwwoman.latest_requests),2)
def kill(self, name, current=None):
'''Elimina un servidor de descargaa si existe y lo elimina de la lista
de servidores de descargas'''
if name not in self.registry:
raise Downloader.SchedulerNotFound()
print('Delete scheduler: ' + str(name))
current.adapter.remove(Ice.stringToIdentity(name))
del self.registry[name]
def mapper_html2json(self, _, line_path):
downloader = d.Downloader(line_path)
downloader.run()
if downloader.pieces:
for piece in downloader.pieces:
yield downloader.title_name, piece
else:
yield downloader.title_name, None
def loadFullConfig(self):
self.confFilename = "tests/fullConfig.json"
self.downloader = Downloader.Downloader(verbosity=DEBUG_DOWNLOADER)
self.downloader.loadConfig(self.confFilename)
self.transferer = Transferer.Transferer(id="transferer",verbosity=DEBUG)
self.transfererData = {"enable":True,"source": {"path": self.tmpdir1, "protocol": "file"}, "destination": {"path": self.tmpdir2, "protocol": "file"}}
self.transferer.addData(self.confFilename)
self.transferer.setValue(self.transfererData)
self.torrentSearch = torrentSearch.torrentSearch(id="torrentSearch",dataFile=self.confFilename,verbosity=DEBUG)
def playTorrent(torrentUrl, StorageDirectory):
if 0 != len(torrentUrl):
contentId = 0 #int(urllib.unquote_plus(url))
torrent = Downloader.Torrent(torrentUrl, StorageDirectory)
torrent.startSession(contentId)
iterator = 0
progressBar = xbmcgui.DialogProgress()
progressBar.create('Подождите', 'Идёт поиск сидов.')
downloadedSize = 0
while downloadedSize < (44 * 1024 * 1024): #not torrent.canPlay:
time.sleep(0.1)
progressBar.update(iterator)
iterator += 1
if iterator == 100:
iterator = 0
downloadedSize = torrent.torrentHandle.file_progress()[contentId]
dialogText = 'Preloaded: ' + str(
downloadedSize / 1024 / 1024) + ' MB / ' + str(
torrent.getContentList()[contentId].size / 1024 /
1024) + ' MB'
peersText = ' [%s: %s; %s: %s]' % (
'Seeds', str(torrent.torrentHandle.status().num_seeds),
'Peers', str(torrent.torrentHandle.status().num_peers))
speedsText = '%s: %s Mbit/s; %s: %s Mbit/s' % (
'Downloading',
str(torrent.torrentHandle.status().download_payload_rate * 8 /
1000000), 'Uploading',
str(torrent.torrentHandle.status().upload_payload_rate * 8 /
1000000))
progressBar.update(iterator, 'Seeds searching.' + peersText,
dialogText, speedsText)
if progressBar.iscanceled():
progressBar.update(0)
progressBar.close()
torrent.threadComplete = True
return
#progressBar.update(0)
#progressBar.close()
playlist = xbmc.PlayList(xbmc.PLAYLIST_VIDEO)
playlist.clear()
listitem = xbmcgui.ListItem(torrent.getContentList()[contentId].path)
playlist.add('file:///' + torrent.getFilePath(contentId), listitem)
#progressBar.close()
xbmc.Player().play(playlist)
progressBar.close()
#time.sleep(15)
while 1 == xbmc.Player().isPlayingVideo():
torrent.fetchParts()
torrent.checkThread()
time.sleep(1)
xbmc.executebuiltin("Notification(%s, %s)" %
('Информация', 'Загрузка торрента прекращена.'))
torrent.threadComplete = True
else:
print " Unexpected access to method playTorrent() without torrent content"
def start(self):
self.get_link = str(self.play_lista_link.text())
full_link = [
self.get_link,
]
folder = "ZA SAMPLOVANJE"
self.max_Counter = Downloader.index_finder(full_link)
ar = (full_link, folder)
self.starter = Thread(target=self.download, args=ar)
self.starter.start()
def main(output, url):
start_time = time.time()
FileInfo = namedtuple('FileInfo', 'url name size lastmodified')
#url = "https://pixabay.com/get/e833b20c2cf5093ecd1f4600ee45479fe16ae3d11eb6134290f0c578/auto-1633418.jpg"
print url
code = Downloader.download(url, output)
if code == 400:
return 400
print '下载时间: %ds' % int(time.time() - start_time)
return code
def make(self, name, current=None):
'''Crea servidores de decarga dado un nombre y lo mete en una lista de
servidores de descarga'''
servant = DownloadSchedulerI(self.queue, DOWNLOADED_FILES)
if name in self.registry:
raise Downloader.SchedulerAlreadyExists()
proxy = current.adapter.add(servant, Ice.stringToIdentity(name))
print('New scheduler: ' + str(name) + ', ' + str(proxy))
self.registry[name] = proxy
return Downloader.DownloadSchedulerPrx.checkedCast(proxy)
def main():
# Initialize Configuration
inputSearchText = raw_input(
'1. Please input the key words you want to search.[default: ' + Configuration.runtime_search_text + ']\n')
if inputSearchText.strip() != "":
Configuration.runtime_search_text = inputSearchText
inputMaxPageNumber = raw_input('2. Please input the maximum page number you want to search.[default: ' + str(
Configuration.runtime_search_max_page_number) + ']\n')
if inputMaxPageNumber.strip() != "":
Configuration.runtime_search_max_page_number = int(inputMaxPageNumber)
downloadPath = raw_input(
'3. Please input the output folder of these videos.[default: ' + Configuration.runtime_download_root_path + ']\n')
if downloadPath.strip() != "":
Configuration.runtime_download_root_path = downloadPath
inputDownloadThread = raw_input('4. Please input download threads number.[default: ' + str(
Configuration.runtime_num_of_download_worker) + ']\n')
if inputDownloadThread.strip() != "":
Configuration.runtime_num_of_download_worker = int(inputDownloadThread)
disableExtractor = raw_input('5. Disable video link extractor(y/n).[default: ' + (
'y' if Configuration.runtime_disable_extractor else 'n') + ']\n')
if disableExtractor.strip() == "y" or disableExtractor.strip() == "Y":
Configuration.runtime_disable_extractor = True
if not os.path.exists(Configuration.getDownloadPath()):
mkdirs(Configuration.getDownloadPath())
DownloadTaskProducer.start()
Downloader.start()
if not Configuration.runtime_disable_extractor:
VideoLinkExtractor.start()
else:
logging.info('Video link extractor is disabled.')
logging.info('App started ...')
while True:
pass
def download(self):
#find out where to get it
(uri, branch) = self._get_download_location()
if(uri):
#now get it
dirsmade = Downloader.download(self.appdir, uri, branch)
#this trace isn't used yet but could be
self.tracewriter.downloaded(self.appdir, uri)
#this trace is used to remove the dirs created
self.tracewriter.dir_made(*dirsmade)
return self.tracedir
def downloader_test():
global dl
ss = SvtStream(json.load(open('test/episode.json')))
ss.downloadcommand="/bin/echo %(url)s %(output)s"
dl = Downloader()
di = DownloadItem(ss, config['output_dir'])
assert di.command.args == [ u'/bin/echo', u'http://svtplay3q-f.akamaihd.net/i/world/open/20140210/1322836-005A/LABYRINT-005A-b6149effda4e5c1d_,900,348,564,1680,2800,.mp4.csmil/index_4_av.m3u8?null=', u'/tmp/labyrint-del-5-av-10.mp4' ]
dl.add(di)
assert di.command.status == ( "Queued" or "Running" )
count = 0
maxwait = 30
while di.command.status != "Completed" and count < maxwait:
# Wait for thread to finish
time.sleep(0.1)
count += 1
assert di.command.status == "Completed"
assert di.command.stdout.rstrip() == "http://svtplay3q-f.akamaihd.net/i/world/open/20140210/1322836-005A/LABYRINT-005A-b6149effda4e5c1d_,900,348,564,1680,2800,.mp4.csmil/index_4_av.m3u8?null= /tmp/labyrint-del-5-av-10.mp4"
def main():
delay = 0.5 # First iteration will set delay=1 or multiply by 2
while True:
try:
Downloader.status()
if Downloader.run():
# files were successfully downloaded
delay = 1
else:
# No files successfully download this iteration
# Increase sleep time
delay = min((delay * 2, 32))
except Exception, e:
if config.email.send_on_crash:
msg = "*** Downloader has crashed! ***\n\n"
msg += "Fatal error occured while running downloader: %s\n\n" % str(e)
msg += "".join(traceback.format_exception(*sys.exc_info()))
notification = mailer.ErrorMailer(msg, subject="Downloader crash!")
notification.send()
sys.stderr.write("Fatal error occurred!\n")
raise
print "Will sleep for %d seconds" % (config.background.sleep * delay)
time.sleep(config.background.sleep * delay)
import time
import sys
import traceback
import mailer
import Downloader
import config.background
import config.email
while True:
try:
Downloader.status()
Downloader.run()
except Exception, e:
if config.email.send_on_crash:
msg = '*** Downloader has crashed! ***\n\n'
msg += 'Fatal error occured while running downloader: %s\n\n' % str(e)
msg += ''.join(traceback.format_exception(*sys.exc_info()))
notification = mailer.ErrorMailer(msg).send()
sys.stderr.write("Fatal error occurred!\n")
raise
time.sleep(config.background.sleep)
from pytradelib.utils import batchfrom pytradelib.downloader import Downloader
from urllib import urlencodetry: import simplejson as jsonexcept ImportError: import json
def get_yql_url(yql): base_url = 'http://query.yahooapis.com/v1/public/yql?' url = base_url + urlencode({'q': yql, 'env': 'store://datatables.org/alltableswithkeys', 'format': 'json'}) if len(url) > (2 * 1024): raise Exception('URL must be shorter than 2048 characters') return url
_EXCHANGES = { 'ASE': 'AMEX', 'NYQ': 'NYSE', 'NMS': 'NASDAQ', # 'NasdaqGS', 'NGM': 'NASDAQ', # 'NasdaqGM', 'NCM': 'NASDAQ', # 'NasdaqCM',}
def _convert_result(result): """ Converts a YQL result dictionary into PyTradeLib format :param result: dict :return: dict with keys lowercased """ r = {} for k, v in result.items(): if k == 'StockExchange': k = 'exchange' v = _EXCHANGES.get(v, None) if k == 'ErrorIndicationreturnedforsymbolchangedinvalid': k = 'error' if k == 'LastTradeDate': k = 'last_trade_date' r[k.lower()] = v return r
def get_symbols_info(symbols, keys=None): if not isinstance(symbols, (list, tuple)): symbols = list(symbols) keys = keys or ['Symbol', 'Name', 'StockExchange', 'LastTradeDate'] yql = 'select %(keys)s from yahoo.finance.quotes where symbol in (%(symbols)s)'
urls = [] for batched_symbols in batch(symbols, 100): csv_symbols = ','.join(['"%s"' % s.upper() for s in batched_symbols]) urls.append(get_yql_url(yql % {'keys': ','.join(keys), 'symbols': csv_symbols})) downloader = Downloader()
results = [] for url, text in downloader.download(urls): json_ = json.loads(text) for result in json_['query']['results']['quote']: results.append(_convert_result(result)) return results
@app.route('/jobs/<int:jobId>/stop', methods=['POST'])
def stop_job(jobId):
command = downloader.downloaditems[jobId].command
if command.status == 'Running':
command.kill()
return make_response('', 200)
else:
return make_response('Cannot stop job %s' % jobId, 400)
try:
config = dict(CONFIG_DEFAULT.items() + yaml.load(open('config.yml')).items())
except:
print "Could not load config.yml"
sys.exit(1)
if config['debug']:
downloader = Downloader()
app.debug = True
app.run(host=config['host'], port=config['port'])
downloader.stop()
else:
stdout = open("server.out", "w")
stderr = open("server.err", "w")
stdin = open("/dev/null", "r")
with daemon.DaemonContext(stdout=stdout, stderr=stderr, stdin=stdin,
working_directory='.'):
downloader = Downloader()
app.run(host=config['host'], port=config['port'])
downloader.stop()
