def fetchIcon(self, iconName):
iconFileName = None
if self.weatherIconId >= 200:
iconFileName, description = WeatherIconMapper.convertIcon(
self.weatherIconId, self.sunrise.time(), self.sunset.time())
print("Icon file name: {}, Description: {}".format(
iconFileName, description))
if iconFileName is not None:
self.icon = UiUtility.loadWeatherIcon(iconFileName)
self.icon.fill(UiColors.GRAY, special_flags=pygame.BLEND_RGB_ADD)
else:
# This weather icon ID is not mapped to a weather icon. In this case,
# fetch the icon from OpenWeatherMap
downloader = Downloader(None)
# TODO: Do in either a background thread, or a coroutine
url = "http://openweathermap.org/img/w/{}.png".format(iconName)
downloader.download(url)
image = downloader.getData()
# Does image need to be processed before it can be used by Pygame?
memFileObj = io.BytesIO(image)
self.icon = pygame.image.load(memFileObj)
class Listener(Thread):
def __init__(self, target):
Thread.__init__(self)
self._address = 'amqp://*****:*****@orangutan.rmq.cloudamqp.com/qxyexflk'
self._queue = 'mytube'
self._timeout = 5
self._downloader = Downloader(target)
def run(self):
parameters = pika.URLParameters(self._address)
parameters.socket_timeout = self._timeout
connection = pika.BlockingConnection(parameters)
channel = connection.channel()
channel.queue_declare(queue=self._queue, durable=True)
channel.basic_qos(prefetch_count=1)
channel.basic_consume(self.on_message, queue=self._queue)
channel.start_consuming()
print("Waiting for messages...")
def on_message(self, channel, method, properties, body):
print('Received Message')
try:
self._downloader.download(body.decode('ascii'))
channel.basic_ack(delivery_tag=method.delivery_tag)
print('Processed Message')
except:
channel.basic_nack(delivery_tag=method.delivery_tag, requeue=False)
print('Failed To Process')
def main():
usage = "Please use -s to specify a search term, and -r to specify" + \
" the desired result name"
try:
opts, args = getopt.getopt(sys.argv[1:], "s:r:h")
except getopt.GetoptError:
print usage
sys.exit(1)
search_term = None
desired_item_name = None
for opt, arg in opts:
if opt == '-s':
search_term = arg
elif opt == '-r':
desired_item_name = arg
elif opt == '-h':
print usage
return 0
if not desired_item_name or not search_term:
try:
search_term = raw_input('Search Term: ')
desired_item_name = raw_input('Desired Item Name: ')
except EOFError:
return 0
downloader = Downloader()
downloader.download(search_term, desired_item_name)
return 0
def main():
if len(sys.argv) <= 1:
print "No url given!"
sys.exit(1)
finder = FuxFinder(sys.argv[1])
downloader = Downloader(finder.find_movie(), finder.find_title() + '.mp4')
downloader.download()
def download(self, args):
downloader = Downloader()
if args.name == "All":
for configFile in os.listdir("config/"):
res = downloader.download("config/" + configFile)
self.printResult(res, configFile.split(".")[0])
else:
res = downloader.download("config/" + args.name + ".json")
self.printResult(res, args.name)
def download(self, args):
downloader = Downloader()
if args.name == "All":
for configFile in os.listdir("config/"):
res = downloader.download("config/" + configFile)
self.printResult(res, configFile.split(".")[0])
else:
res = downloader.download("config/" + args.name + ".json")
self.printResult(res, args.name)
def main():
root = Tk()
root.withdraw()
# Check for output directory
logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s',
level=logging.INFO,
datefmt='%Y-%m-%d %H:%M:%S')
output_path = askdirectory()
root.update()
if output_path == '':
logging.warning("No output path selected. Using current directory")
output_path = '.'
browser, refresh_rate_min, sid, token, to, from_ = _parse_conf("conf.toml")
downloader = Downloader(browser, sid, token, to, from_, output_path)
def handle_sigint(_sig, _frame):
print()
global exit_signal
global exit_lock
if downloader is None:
logging.info('Shutting down...')
logging.shutdown()
sys.exit(0)
exit_lock.acquire()
t = Thread(target=downloader.download)
t.daemon = True
if exit_signal:
exit_lock.release()
logging.info('Shutting down...')
logging.shutdown()
sys.exit(0)
else:
exit_signal = True
exit_lock.release()
logging.info("Checking now...")
downloader.load_cookie_jar()
t.start()
sleep(0.75)
exit_lock.acquire()
exit_signal = False
exit_lock.release()
signal(SIGINT, handle_sigint)
while True:
try:
downloader.download()
except Exception as e:
logging.exception(e)
logging.info(f"Sleeping for {refresh_rate_min} minutes.")
sleep(refresh_rate_min * 60)
def FetchPodcastFiles(self):
path = os.path.normpath(QtGui.QFileDialog.getExistingDirectory())
myScraper = Scraper(
"http://{}.libsyn.com/webpage/page/1/size/{}".format(
self.textbox.text(), self.comboBox.currentText()))
myScraper.scrape()
for pod in myScraper.PodList:
downloader = Downloader(pod)
downloader.download(path)
file = PodFile(downloader.DownloadedFile)
file.writeTag(pod)
def fetchCurrentMoonPhase(self):
downloader = Downloader(None)
currentTimestamp = datetime.now().timestamp()
url = "http://api.farmsense.net/v1/moonphases/?d={}".format(
int(currentTimestamp))
downloader.download(url)
moonPhaseJson = downloader.getDataAsString()
self.parseMoonPhaseJson(moonPhaseJson)
self._notifyListeners()
def download_history(past_days, logger):
downloader = Downloader(logger)
retry_download(downloader, past_days)
yesterday_id = utils.get_previous_working_day_date_id()
for date_id in range(max(1, yesterday_id - past_days + 1), yesterday_id + 1):
for file_type in files_to_download:
downloader.download(
files_to_download[file_type].format(date_id),
str(date_id)
)
class Scheduler:
def __init__(self):
self.__url_manager = URLManager()
self.__downloader = Downloader()
self.__parser = XMLParser()
pass
def __str__(self):
pass
def run(self):
start_url = ['http://www.iplaypy.com/jichu/set.html']
self.__url_manager.put_url(list_url=start_url)
while not self.__url_manager.empty():
url = self.__url_manager.get_url()
print(url)
try:
html = self.__downloader.download(url)
except Exception as e:
print(e)
while not self.__url_manager.empty():
print(self.__url_manager.get_url())
self.__parser.parser(html=html)
self.__url_manager.put_url(list_url=self.__parser.get_url())
pass
pass
def __init__(self):
super(Main, self).__init__(load_config, NAME_FILE_CONFIG_PATH, NAME_FILE_LOG_PATH, NAME_FILE_CSV_PATH)
self.loading_args()
self.log_configuration()
# General variable
self.config = self.loading_file_config()
self.errors = []
self.fields_csv = self.config.get("GENERAL", "fields_csv").split(",")
list_links = parser_cvs(self.args.csv, self.fields_csv)
http = Downloader()
report = Report()
for data in list_links:
download = http.download(data["link"])
report.add_elements(download)
report.print_report()
logging.info("Script Completado.")
def __download(self):
downloader = Downloader(self.url, self.downloadFile, \
OnlineUpdater.__getModifiedTime(self.recipeFile))
try:
retval = downloader.download()
finally:
downloader.close()
return retval
def download(self, msg):
url = msg["url"];
title = msg["title"];
fname = os.path.basename(url)
extName = fname.split(".")[1];
filename, ok2 = QFileDialog.getSaveFileName(dialog, title, "C:/"+fname, 'File (*.'+extName+')');
log.info(filename);
data={};
data["type"] = 10;
data["id"] = msg["id"];
pos = filename.rfind("/")
if not filename[:pos] =="":
# //download
downloader = Downloader(url, filename)
downloader.download()
os.startfile(filename[:pos]);#打开目录
self._socket.sendTextMessage(json.dumps(data))
def download(self):
self.btnDownload.setDisabled(True)
self.statusLabel = QLabel('Connecting')
self.progressBar = QProgressBar()
self.progressBar.setMinimum(0)
self.progressBar.setMaximum(100)
self.myStatusBar.addWidget(self.statusLabel, 2)
self.myStatusBar.addWidget(self.progressBar, 1)
self.progressBar.setValue(1)
self.settings.beginGroup('URLS')
initialLink = self.settings.value('CategoriesDetailed/VahaSelected/InitialLink')
pagingLink = self.settings.value('CategoriesDetailed/VahaSelected/PagingLink')
self.settings.endGroup()
downloader = Downloader(initialLink, pagingLink, 25, 5)
downloader.messenger.downloadProgressChanged.connect(self.onDownloadProgressChanged)
downloader.messenger.downloadComplete.connect(self.onDownloadComplete)
downloader.download()
def __download(self):
downloader = Downloader(self.url, self.downloadFile, \
OnlineUpdater.__getModifiedTime(self.recipeFile))
try:
retval = downloader.download()
finally:
downloader.close()
return retval
def run(self):
downloader = Downloader()
filename, file_size = downloader.download(
self.items[0], # name
self.items[1], # data url
self.items[2], # key url
self.items[3], # referer url
progress=self.download_progress.emit)
self.download_finished.emit(filename, file_size)
class ClientHandler:
def __init__(self, port):
ip = getCurrMachineIp()
self.username, self.password = None, None
self.socket = zhelper.newServerSocket(zmq.REP, ip, port)
self.db = Db("TrackerDB")
self.uploader = Uploader(self.socket, self.db)
self.downloader = Downloader(self.socket, self.db)
self.fileExplorer = FileExplorer(self.socket, self.db)
def handleRequest(self):
"""
syncrounous (blocking) function, waits until receiving an action, handle it and return
"""
clientRequestInfo = self.socket.recv_json()
userToken = clientRequestInfo.get("token")
if not self._authenticate(userToken):
print("authentication error")
return
function = clientRequestInfo.get("function")
if function == "ls":
self.fileExplorer.explore(self.socket, self.username)
elif function == "download":
fileName = clientRequestInfo.get("fileName")
self.uploader.upload(fileName, self.username)
elif function == "upload":
self.downloader.download()
else:
print("not correct function: ", function, " should be one of 'download, upload, ls'")
self.socket.send_string("Either you don't send a function or it is not a valid function")
def _authenticate(self, userToken):
if not userToken:
print("no user token is supplied: ", userToken)
self.socket.send_string("you should provide a token")
return False
self.username, self.password = decodeToken(userToken)
if not self.username:
print("can't decode the token: ", userToken)
self.socket.send_string("Invalid token")
return False
return True
class SpiderMain(object):
def __init__(self):
# 初始化程序
self.download = Downloader()
self.parser = HtmlParser()
self.mysql = Mysqldb()
def run(self, url, database):
response = self.download.download(url)
self.parser.parser(response, database)
def pack(data):
downloads_folder = fetch_folder_path(DOWNLOADS_FOLDER)
archives_folder = fetch_folder_path(ARCHIVES_FOLDER)
downloader = Downloader(downloads_folder, data)
if downloader.download():
archivator = Archivator(downloader.destination, os.path.join(archives_folder, "{}.zip".format(downloader.uid)))
archivator.archive()
return os.path.basename(archivator.destination)
def main():
# 多页图片doujinshi，测试
doujinshi_url = "https://e-hentai.org/g/980918/a3e70a1ca8/"
# 单页图片doujinshi，测试
doujinshi_url = "https://e-hentai.org/g/1178073/30bc24b00a/"
_ = doujinshi_parser(doujinshi_url)
# 如果jap标题为空，就用eng标题
folder = _["jap_name"]
if _["jap_name"] == "":
folder = _["eng_name"]
from downloader import Downloader
downloader = Downloader()
url_queue = _["img_list"]
print(url_queue)
downloader.download(url_queue, folder=folder)
def crawl(init_url):
url_pool = UrlManager()
downloader = Downloader()
parser = HtmlParser()
outputer = Outputer()
temp_url = init_url
while temp_url:
driver = downloader.download(temp_url)
content, temp_url = parser.parse(driver)
outputer.write(content)
outputer.close()
class FileManager:
def __init__(self):
self.course_dict = {}
self.downloader = Downloader()
def __getattribute__(self, name):
from loguru import logger
attr = super().__getattribute__(name)
if hasattr(attr, "__call__"):
def log_func(*args, **kwargs):
logger.debug("called {} with {} and {}".format(
name, args, kwargs))
ret = attr(*args, **kwargs)
return ret
return log_func
else:
return attr
def add_course_list(self, course_list):
for course in course_list:
self.course_dict[course.name] = course
def get_all_files(self, course_name):
def get_all_files_recsive(course_element):
childern = course_element.childern
files = [ch for ch in childern if ch.type == "File"]
childern_files = [
child for ch in childern for child in get_all_files_recsive(ch)
]
return files + childern_files
course = self.course_dict[course_name]
files = get_all_files_recsive(course.course_element)
def download_files(self, driver, course_name):
files = self.get_all_files(course_name)
f = files[0]
self.downloader.update_cookies(driver.get_cookies())
self.downloader.download(f)
def on_feed_updated(self, feed, result):
if not self.process_error(result):
return
self.name = feed.name
last_label = self.ui.get_object('last_label')
last_label.set_markup(
_('''\
This guide has finished.\n\
You can change the name of this comic. Press "apply" to add it.'''))
name_entry = self.ui.get_object('name_entry')
name_entry.set_text(self.name)
# Can we skip the image page?
images = feed.get_unique_images()
if len(images) > 1:
# Download images for the image list
self.model.clear()
for image in images:
iterator = self.model.append((image, None))
downloader = Downloader(image[1])
downloader.connect('completed',
self.on_image_download_completed, iterator)
downloader.download()
self.ui.get_object('image_page').show()
else:
self.ui.get_object('image_page').hide()
try:
self.image = images[0]
except IndexError:
self.process_error(Feed.DOWNLOAD_NOT_FEED)
return
# Complete page
self.assistant.set_page_complete(self.ui.get_object('wait_page'), True)
self.ui.get_object('wait_label').set_markup(
_('Found <b>%(name)s</b>!\n\n<i>%(description)s</i>') % {
'name': self.name,
'description': self.feed.description
})
def main(port):
#? TODO: when to use udp protocol
trackerSocket = zhelper.newSocket(zmq.REQ, TRACKER_IP,
TRACKER_PORTS_KEEPERS)
downloadUploadSocket = zhelper.newServerSocket(zmq.REP, getCurrMachineIp(),
port)
downloader = Downloader(downloadUploadSocket, trackerSocket, port)
uploader = Uploader(downloadUploadSocket, trackerSocket, port)
while True:
request = downloadUploadSocket.recv_json()
print("request received:", request)
username = authenticate(downloadUploadSocket, request.get("token"))
downloadUploadSocket.send_string("ACK")
if not username:
continue
del request["token"]
request["username"] = username
if request.get("function") == "download":
uploader.upload(request)
elif request.get("function") == "upload":
downloader.download(request)
def main():
""" Main function that is executed when running the program """
args = get_args()
init_logger()
global downloader
downloader = Downloader(room=args.room,
password=args.password,
output_dir=args.output_dir,
max_allowed_size=args.max_allowed_size,
do_log=args.no_logs,
archive=args.archive,
archive_type=args.archive_type,
chat_log=args.chat_log)
if args.loop:
downloader.downloadLoop(args.loop_delay)
else:
downloader.download()
def on_feed_updated(self, feed, result):
if not self.process_error(result):
return
self.name = feed.name
last_label = self.ui.get_object('last_label')
last_label.set_markup(_('''\
This guide has finished.\n\
You can change the name of this comic. Press "apply" to add it.'''))
name_entry = self.ui.get_object('name_entry')
name_entry.set_text(self.name)
# Can we skip the image page?
images = feed.get_unique_images()
if len(images) > 1:
# Download images for the image list
self.model.clear()
for image in images:
iterator = self.model.append((image, None))
downloader = Downloader(image[1])
downloader.connect('completed',
self.on_image_download_completed, iterator)
downloader.download()
self.ui.get_object('image_page').show()
else:
self.ui.get_object('image_page').hide()
try:
self.image = images[0]
except IndexError:
self.process_error(Feed.DOWNLOAD_NOT_FEED)
return
# Complete page
self.assistant.set_page_complete(self.ui.get_object('wait_page'),
True)
self.ui.get_object('wait_label').set_markup(
_('Found <b>%(name)s</b>!\n\n<i>%(description)s</i>')
% {'name': self.name,
'description': self.feed.description})
class Manager(object):
def __init__(self, settings):
self.changer = Changer()
self.downloader = Downloader()
self.settings = settings
self._setup_schedule()
def _setup_schedule(self):
change_interval = self.settings['changeInterval']
download_interval = self.settings['downloadInterval']
self._next = schedule.every(change_interval).seconds.do(run_threaded, self.change)
self._download_more = schedule.every(download_interval).seconds.do(run_threaded, self.download)
def download(self):
client = Wallhaven()
urls = client.get_random(count=self.settings['count'])
for url in urls:
filename = url.split('/')[-1]
file = os.path.join(self.settings['path'], filename)
self.downloader.download(url, file)
def change(self):
self.changer.next(self.settings['path'])
def download_more(self):
self._download_more.run()
def next(self):
self._next.run()
def delete(self):
run_threaded(self.changer.delete)
self.next()
def run_pending(self):
schedule.run_pending()
def run_all(self):
schedule.run_all()
class SpiderMain(object):
def __init__(self):
# 初始化程序
self.download = Downloader()
self.parser = HtmlParser()
self.save = SaveData()
self.workbook = Workbook()
self.ch = Choice()
print('初始化完成...')
def run(self):
while True:
try:
p = int(input('想要爬多少页的数据？' + '\n'))
break
except ValueError:
print('输入错误！请输入数字')
page = p + 1
print("================================")
print(' A.原创发布区 B.精品软件区 ')
print(' C.脱壳破解区 D.移动安全区 ')
print(' E.病毒分析区 F.编程语言区 ')
print(' G.软件调试区 H.动画发布区 ')
print(' I.逆向资源区 J.安全工具区 ')
print(' K.招聘求职区 ')
print("================================")
while True:
choice = input("选择爬取的专区，输入 Q 退出程序(输入的字母必须大写)：")
half_url, name = self.ch.make_the_arrg(choice)
if name != 'Error':
break
print(half_url + '\n' + name)
self.save.createfile(name)
for i in range(1, page):
url = half_url + str(i) + '.html'
response = self.download.download(url)
self.parser.parser(response, name)
sleep = random.randint(2, 10)
print('爬取第' + str(i) + '页完成，程序休息' + str(sleep) + '秒')
time.sleep(sleep) # 程序睡眠
if i != page - 1:
print('-----------------------------')
print(' 下一页 ')
print('-----------------------------')
print('数据写入完成，正在进行数据去重...')
self.save.delete_same_data()
try:
self.workbook.save('将csv的数据导入此表.xlsx')
except:
print('创建xlsx文件失败，请手动创建')
print('程序运行完毕')
def get_response(self):
int_len = 4
header = self.s.recv(7)
packet_length = int_len.from_bytes(header[2:5], 'big')
data = self.recvall(self.s, packet_length)
r = Reader(data)
code = r.readUInt32()
if code == 7 or code == 8:
self.fingerprint = json.loads(r.readFinger()) # fingerprint
r.readInt32()
r.readShort()
self.assets_url = r.readString() # assets url
r.skip(23)
r.readString()
r.skip(2)
r.readString()
d = Downloader(self.fingerprint, self.assets_url)
d.download()
else:
_(f"Recived code {code} - returning!")
return
def start(id):
try:
download = Download.get_by_id(id)
except:
abort(404)
# now run download process
downloader = Downloader(app.config.config)
if downloader.download(download) is False:
abort(500)
# done
return jsonify(model_to_dict(download))
class Crawler(object):
def __init__(self):
self.down_load = Downloader()
self.site_parse = Parser()
self.vail_data = Vaildater()
def crawl(self):
for config in configs:
urls = config['urls']
parse_config = config
offset = 0
for url in urls:
offset += 1
print(f"抓取{config['site']}的第{offset}页...")
proxy_list = self.site_parse.who_parse(
self.down_load.download(url), parse_config)
for proxy in proxy_list:
self.vail_data.valid(proxy, 'crawl')
print("抓取完成...")
def download():
# for now
#url = urllib.parse.unquote(url)
url = request.args.get('url')
if len(url) < 1:
abort(400)
# try to get more info
downloader = Downloader(app.config.config)
download = downloader.get_download_info(url, request.args.get('dest'))
if download is None:
abort(500)
# now run download process
if downloader.download(download) is False:
abort(500)
# done
return jsonify(model_to_dict(download))
def upload_link():
url_to_download = request.args.get('url_to_download')
if url_to_download == '':
return redirect('/')
else:
try:
dwnloader = Downloader()
output_data = dwnloader.download({'url': url_to_download})
file_name = getPureFileName(output_data['dir_uid'],
output_data['file_name'])
response = make_response(
send_file(os.path.join('Files', output_data['file_name']),
attachment_filename=file_name,
as_attachment=True))
response.headers[
'Access-Control-Expose-Headers'] = 'Content-Disposition'
response.headers['X-file-name'] = file_name
return response
except:
raise InvalidUsage('There was an error processing the request.',
status_code=500)
class Client:
def __init__(self):
self.socket = zhelper.newSocket(zmq.REQ, TRACKER_IP, TRACKER_PORTS)
self.downloader = Downloader()
self.uploader = Uploader()
self.fileExplorer = FileExplorer()
def listFiles(self, token):
files = self.fileExplorer.explore(self.socket, token)
print("listFiles: files are")
print("\n".join(files))
return files
def downloadFile(self, token, fileName, savePathDir="."):
downloadedFile = self.downloader.download(self.socket, token, fileName)
if (not downloadedFile):
return False
utils.writeVideo(downloadedFile, f"{savePathDir}/{fileName}_rec.mp4")
return downloadedFile
def uploadLocalFile(self, token, filePath):
return self.uploader.upload(self.socket, token, filePath)
def filter(self, update, context):
if not hasattr(update.message, 'sticker'):
return
if self.s3_client.sticker_exists(update.message.sticker.set_name) \
and not update.effective_chat.username in self.admin_users:
update.message.reply_text("Este pack ya existe en toDus! " +
update.message.sticker.set_name)
return
update.message.reply_text("Procesando " +
update.message.sticker.set_name)
downloader = Downloader(context.bot)
sticker_dict = downloader.download(update.message.sticker.set_name)
for file_path in glob.iglob(sticker_dict['name'] + '/**/*.*',
recursive=True):
self.s3_client.upload_file(file_path, file_path)
storage = Storage()
storage.zip_dir(sticker_dict['name'], sticker_dict['name'] + '.zip')
self.s3_client.upload_file(sticker_dict['name'] + '.zip',
sticker_dict['name'] + "/pack.zip")
shutil.rmtree(sticker_dict['name'], True)
os.remove(sticker_dict['name'] + '.zip')
self.pg_client.insert_sticker_pack(sticker_dict['name'],
sticker_dict['title'],
sticker_dict['thumb'],
sticker_dict['animated'])
update.message.reply_text("Añadido " + update.message.sticker.set_name)
if update.effective_chat.username in self.admin_users:
self.pg_client.set_recommended_pack(sticker_dict['name'])
update.message.reply_text("Marcado como recomendado " +
update.message.sticker.set_name)
class Menu:
def __init__(self):
reload(sys)
sys.setdefaultencoding('UTF-8')
self.title = 'Nada'
self.model = ['最新期刊', '分类期刊', '搜索期刊', '收藏夹', '关于']
self.view = 'menu'
self.ctrl = 'menu'
self.offset = 0
self.index = 0
self.step = 10
self.play_id = -1
self.play_vol = -1
self.present = []
self.stack = []
self.player = Player()
self.ui = UI()
self.luoo = Luoo()
self.downloader = Downloader()
self.database = Database()
self.database.load()
self.collections = self.database.data['collections'][0]
self.screen = curses.initscr()
self.screen.keypad(1)
def start(self):
self.ui.build(self.title, self.model, self.view, self.offset, self.index, self.step, self.play_vol,
self.play_id)
self.stack.append([self.title, self.model, self.view, self.ctrl, self.offset, self.index])
while True:
title = self.title
model = self.model
view = self.view
ctrl = self.ctrl
offset = self.offset
index = idx = self.index
step = self.step
key = self.screen.getch()
self.ui.screen.refresh()
if view == 'songs':
length = len(model['songs'])
else:
length = len(model)
if key == ord('q'):
break
elif key == ord('k'):
if idx == offset:
if offset == 0:
continue
self.offset -= step
self.index = offset - 1
else:
self.index = carousel(offset, min(length, offset + step) - 1, idx - 1)
elif key == ord('j'):
if idx == min(length, offset + step) - 1:
if offset + step >= length:
continue
self.offset += step
self.index = offset + step
else:
self.index = carousel(offset, min(length, offset + step) - 1, idx + 1)
elif key == ord('u'):
if offset == 0:
continue
self.offset -= step
self.index = (index - step) // step * step
elif key == ord('i'):
if offset + step >= length:
continue
self.offset += step
self.index = (index + step) // step * step
elif key == ord('l'):
if self.view == 'songs':
continue
self.ui.loading()
self.control(idx)
self.index = 0
self.offset = 0
elif key == ord('h'):
if len(self.stack) == 1:
continue
up = self.stack.pop()
self.title = up[0]
self.model = up[1]
self.view = up[2]
self.ctrl = up[3]
self.offset = up[4]
self.index = up[5]
elif key == ord(' '):
if view == 'songs':
self.present = [title, model, view, ctrl, offset]
self.player.play_song(view, model, idx)
elif key == ord(']'):
self.player.next_song()
if view == 'songs':
self.index = self.player.play_id % length
if idx == min(length, offset + step) - 1:
if offset + step >= length:
continue
self.offset += step
time.sleep(0.1)
elif key == ord('['):
self.player.prev_song()
if view == 'songs':
self.index = self.player.play_id % length
if idx == offset:
if offset == 0:
continue
self.offset -= step
time.sleep(0.1)
elif key == ord('p'):
if len(self.present) == 0:
continue
self.stack.append([self.title, self.model, self.view, self.ctrl, self.offset, self.index])
self.title = self.present[0]
self.model = self.present[1]
self.view = self.present[2]
self.ctrl = self.present[3]
self.index = idx = self.player.play_id
self.offset = idx // step * step
elif key == ord('m'):
if view != 'menu':
self.stack.append([self.title, self.model, self.view, self.ctrl, self.offset, self.index])
self.title = self.stack[0][0]
self.model = self.stack[0][1]
self.view = self.stack[0][2]
self.ctrl = self.stack[0][3]
self.offset = self.stack[0][4]
self.index = self.stack[0][5]
elif key == ord('a'):
if view == 'songs' and ctrl is not 'collections':
self.ui.status(model['songs'][idx]['name'], 'Add', 5)
self.collections.append(model['songs'][idx])
elif key == ord('r'):
if ctrl == 'collections':
if length != 0:
self.ui.status(model['songs'][idx]['name'], 'Remove', 3)
self.model['songs'].pop(idx)
self.index = carousel(offset, min(length, offset + step) - 1, idx)
elif key == ord('c'):
self.stack.append([self.title, self.model, self.view, self.ctrl, self.offset, self.index])
self.title = 'Nada > nada 收藏'
self.collection()
elif key == ord('d'):
if view == 'songs':
self.downloader.download(model['songs'][idx])
self.play_vol = self.player.play_vol
self.play_id = self.player.play_id
self.ui.build(self.title, self.model, self.view, self.offset, self.index, self.step, self.play_vol,
self.play_id)
self.ui.screen.refresh()
self.player.stop()
self.database.save()
curses.endwin()
def control(self, idx):
self.stack.append([self.title, self.model, self.view, self.ctrl, self.offset, self.index])
ctrl = self.ctrl
if ctrl == 'menu':
self.menu(idx)
elif ctrl == 'vols':
self.vols(idx)
elif ctrl == 'vtype':
self.vtype(idx)
self.offset = 0
self.index = 0
def menu(self, idx):
if idx == 0:
self.title += ' > 最新期刊'
self.model = self.luoo.new()
self.view = 'vols'
self.ctrl = 'vols'
elif idx == 1:
self.title += ' > 分类期刊'
self.model = self.luoo.vtype()
self.view = 'vols'
self.ctrl = 'vtype'
elif idx == 2:
vol_id = self.ui.search('搜索期刊: ')
try:
self.model = self.luoo.vol(vol_id)
self.title += ' > 搜索期刊 > Vol.' + self.model['id'] + ' ' + self.model['title']
self.view = 'songs'
self.ctrl = 'songs'
except Exception, e:
self.model = []
elif idx == 3:
self.title += ' > 收藏夹'
self.collection()
import httplib2
while True:
try:
fp = open('config.json', 'r')
config = json.loads(fp.read())
sync = config['sync']
sync_rate = config['sync_rate']
fp.close()
except:
print 'Read config.json failed!'
exit(1)
# Download
D = Downloader()
D.download()
# Analyze
# -----------------
A = Analyzer()
# Visualize
# -----------------
A.exportjs()
A.coinhtml()
if not sync:
webbrowser.open('./html/main.html', new=0, autoraise=True)
exit(0)
def main(args):
"""
Main function - launches the program.
:param args:
The Parser arguments
:type args:
Parser object
:returns:
List
:example:
>>> ["The latitude and longitude values must be valid numbers", 1]
"""
v = VerbosityMixin()
if args:
if args.subs == 'process':
verbose = True if args.verbose else False
force_unzip = True if args.force_unzip else False
stored = process_image(args.path, args.bands, verbose, args.pansharpen, args.ndvi, force_unzip, args.ndvi1)
if args.upload:
u = Uploader(args.key, args.secret, args.region)
u.run(args.bucket, get_file(stored), stored)
return ["The output is stored at %s" % stored]
elif args.subs == 'search':
try:
if args.start:
args.start = reformat_date(parse(args.start))
if args.end:
args.end = reformat_date(parse(args.end))
except (TypeError, ValueError):
return ["You date format is incorrect. Please try again!", 1]
s = Search()
try:
lat = float(args.lat) if args.lat else None
lon = float(args.lon) if args.lon else None
except ValueError:
return ["The latitude and longitude values must be valid numbers", 1]
result = s.search(paths_rows=args.pathrow,
lat=lat,
lon=lon,
limit=args.limit,
start_date=args.start,
end_date=args.end,
cloud_max=args.cloud)
if result['status'] == 'SUCCESS':
v.output('%s items were found' % result['total'], normal=True, arrow=True)
if result['total'] > 100:
return ['Over 100 results. Please narrow your search', 1]
else:
v.output(json.dumps(result, sort_keys=True, indent=4), normal=True, color='green')
return ['Search completed!']
elif result['status'] == 'error':
return [result['message'], 1]
elif args.subs == 'download':
d = Downloader(download_dir=args.dest)
try:
bands = convert_to_integer_list(args.bands)
if args.pansharpen:
bands.append(8)
if args.ndvi:
bands = [4, 5]
downloaded = d.download(args.scenes, bands)
if args.process:
force_unzip = True if args.force_unzip else False
for scene, src in downloaded.iteritems():
if args.dest:
path = join(args.dest, scene)
else:
path = join(settings.DOWNLOAD_DIR, scene)
# Keep using Google if the image is before 2015
if src == 'google':
path = path + '.tar.bz'
stored = process_image(path, args.bands, False, args.pansharpen, args.ndvi, force_unzip)
if args.upload:
try:
u = Uploader(args.key, args.secret, args.region)
except NoAuthHandlerFound:
return ["Could not authenticate with AWS", 1]
except URLError:
return ["Connection timeout. Probably the region parameter is incorrect", 1]
u.run(args.bucket, get_file(stored), stored)
v.output("The output is stored at %s" % stored, normal=True, arrow=True)
return ['Image Processing Completed', 0]
else:
return ['Download Completed', 0]
except IncorrectSceneId:
return ['The SceneID provided was incorrect', 1]
import sys from downloader import Downloader from parser import Parser import time from random import random if __name__=="__main__": buffer = [sys.argv[1]] while len(buffer) > 0: url = buffer.pop() d = Downloader() html = d.download(url) if html is not None: p = Parser() urls = p.parse(html) p.save() if len(buffer) < 100000: buffer = buffer + urls time.sleep(3*random())
class BlogCrawler(object):
def __init__(self):
self.charset = 'utf-8'
self.parser = CommonBlogParser()
self.downloader = Downloader()
# 设置页面url加载的参数
self.uid = ''
# ========= 完成解析用户微博数据前的准备工作 ========#
def _init_(self, uid, url):
"""
解析用户微博数据前的准备工作,包括:
1. 获取当前用户的page_id
2. 获取当前用户的微博总页数
"""
self.http_params = {
'_t':'0',
'count':'50',
'page':1,
'pagebar':'',
'pre_page':'0'
}
content = self.downloader.download(url)
# 判断用户是否存在
if not self.exist(content):
raise UserNotFoundError(url)
# 获取用户ID
'''
btag = "$CONFIG['oid']='"
etag = "';"
bpos = content.find(btag) + len(btag)
epos = content.find(etag, bpos)
uid = content[bpos:epos]
'''
self.uid = uid
# 获取 page_id
self.page_id = self._parse_pageid(content)
# 获取微博总页数
self.pagenum = self._caculate_pagenum(content)
# 获取pid,抓取微博所需的domain参数
self.pid = self._parse_pid(content)
# 获取用户头像地址和昵称
img_url, nick_name = self._parse_userinfo(content)
self.parser.init_user(self.uid, img_url, nick_name)
self.url = self.get_url(uid)
def exist(self, content):
"""
判断当前用户是否存在
------------------------------
return: 用户存在返回True,否则返回False
"""
if content.find('<title>错误提示') != -1:
return False
return True
def _parse_pageid(self, content):
"""
解析页面的page_id
----------------------------------
content: 待解析的网页内容
----------------------------------
return: page_id, 或空
"""
btag = "$CONFIG['page_id']='"
etag = "'"
page_id = ''
if content:
bpos = content.find(btag)
if bpos:
bpos += len(btag)
epos = content.find(etag, bpos)
page_id = content[bpos:epos]
return page_id
def _caculate_pagenum(self, content):
"""
计算微博的总页数
------------------------------
return: 微博页数
"""
msgcount = self._parse_msgcount(content)
per_pagenum = 45
total_pagenum = msgcount / per_pagenum
if msgcount % per_pagenum:
total_pagenum += 1
print "blog_page #: ",total_pagenum
return total_pagenum
def _parse_msgcount(self, content):
"""
解析微博条数
---------------------------
content: 网页文本
---------------------------
return: 微博条数
"""
if not content:
raise PreprocessError(self.uid)
if(content.find('您当前访问的帐号异常') != -1):
return 0
etag1 = '>微博<\/span>'
etag2 = '<\/strong>'
btag = '>'
epos = content.find(etag1)
epos = content[:epos].rfind(etag2)
bpos = content[:epos].rfind(btag) + len(btag)
num = content[bpos:epos]
if(len(num) == 0):
return 0
else:
return int(content[bpos:epos])
def _parse_userinfo(self, content):
"""
解析用户的头像地址/用户昵称
-----------------------------
content: 网页文本
------------------------------
return: (img_url, nick_name)
"""
btag = '<div class="pf_photo"'
etag = '<\/div>'
bpos = content.find(btag)
epos = content.find(etag, bpos)
soup = BeautifulSoup(content[bpos:epos].replace('\\/', '/') + '</div>')
img_url = soup.img['src']
nick_name = soup.img['alt']
return img_url, nick_name
def get_filepath(self,uid):
filepath = "C:/Users/hp1/Desktop/weibo_crawler/"+str(uid)+"_blog.txt"
return filepath
#======== 解析用户的微博数据,并保存结果 =======#
def scratch(self, uid, start_pageindex=1):
"""
获取给定用户的所有满足条件的微博,并写入文件
----------------------------------------
uid: 待抓取微博数据的用户ID
start_pageindex: 从第几页开始抓取用户的微博数据
"""
self._init_(uid, self.get_url(uid))
'''
if os.path.isfile(self.get_filepath(uid)): # 用户微博已下载
print self.uid, u'用户的微博已下载！'
return None
'''
# modified on 8/9. Don't need to check, as checked in the main.py whether it has been recorded
# also, overwriting is acceptable
if start_pageindex > self.pagenum:
# print self.pagenum
return []
scratch_result = self._sequence_scratch(self.uid, start_pageindex, self.pagenum)
return scratch_result
def _sequence_scratch(self, uid, start_pageindex, end_pageindex, direction=1):
"""
执行顺序抓取策略,按照顺序从前往后或者从后往前抓取
---------------------------------------------------
uid: 待抓取的用户ID
start_pageindex: 起始页码
end_pageindex: 结束页面
direction: 抓取的方向, 1->从前往后,pageindex递增;-1->从后往前,pageindex递减
---------------------------------------------------
return: blogs
"""
blogs = []
for pageindex in range(start_pageindex, end_pageindex + direction, direction):
temp_blogs = self._parse_blogs(pageindex)
if(pageindex>10):
break
# limit the number of blog pages, modified on 9/11
print uid + ':获取第' + str(pageindex) + '页微博成功.'
blogs.extend(temp_blogs)
time.sleep(1)
if not self._continue(temp_blogs, direction):
break
return blogs
def _parse_blogs(self, pageindex):
"""
获取指定微博页面的三个子页的微博内容
-----------------------------------
return: 该页的微博列表
"""
blogs = []
self.http_params['page'] = pageindex
self.http_params['count'] = '50'
self.http_params['pagebar'] = ''
# 下载第一页
self.http_params['pre_page'] = self.http_params['page'] - 1
content = self.downloader.download(self.url, self.http_params)
if content:
sub_blogs = self.parser.parse(content)
blogs.extend(sub_blogs)
if not self._continue(blogs):
#print "not continue"
return blogs
# 下载第二页
self.http_params['count'] = '15'
self.http_params['pagebar'] = '0'
self.http_params['pre_page'] = self.http_params['page']
content = self.downloader.download(self.url, self.http_params)
if content:
sub_blogs = self.parser.parse(content)
blogs.extend(sub_blogs)
if not self._continue(sub_blogs):
return blogs
# 下载第三页
self.http_params['count'] = '15'
self.http_params['pagebar'] = '1'
self.http_params['pre_page'] = self.http_params['page']
content = self.downloader.download(self.url, self.http_params)
if content:
sub_blogs = self.parser.parse(content)
blogs.extend(sub_blogs)
return blogs
def _continue(self, blogs, direction=1):
"""
判断是否需要继续进行下载任务
-----------------------------
blogs: 待判断的博客列表,按照时间先后顺序排列
direction: 判别的方向,1->判定最后一条微博是否比起始时间早;
-1->判定第一条微博是否比结束时间晚;
------------------------------
return: 继续返回True,否则返回False
"""
is_continue = True
if blogs:
if (direction == -1 and dt.compare(blogs[0]['pt'], config.end_time) > 0) or \
(direction == 1 and dt.compare(blogs[-1]['pt'], config.begin_time) < 0):
is_continue = False
return is_continue
def get_url(self,uid):
"""
获取下载的网页地址
"""
url = 'http://weibo.com/' + str(uid) + '?from=otherprofile'
return url
def _parse_pid(self, content):
btag = "$CONFIG['pid']='"
etag = "'"
pid = ''
if content:
bpos = content.find(btag)
if bpos:
bpos += len(btag)
epos = content.find(etag, bpos)
pid = content[bpos:epos]
return pid
class TileLoader(object):
TILE_WIDTH = 256 # tile is square
TILE_FORMAT = 'png'
def __init__(self, min_lat, min_lon, max_lat, max_lon, width, max_zoom = 18):
self.tiles = []
self.min_lat = min_lat
self.min_lon = min_lon
self.max_lat = max_lat
self.max_lon = max_lon
self.mercator = GlobalMercator()
self.downloader = Downloader()
# count how many horizontal tiles we need
self.x_tiles_needed = math.ceil(width / self.TILE_WIDTH)
self.max_zoom = max_zoom
def download(self, cache_dir, url, http_headers):
"""Downloads tiles and returns list of downloaded tiles."""
tile_files = {}
tiles = self._get_tile_list()
for (tx, ty, tz) in tiles:
cx, cy, cz = self._convert_tile(tx, ty, tz)
tile_url = url.replace('{x}', str(cx)).replace('{y}', str(cy)).replace('{z}', str(cz))
tile_file = self._gen_tile_file(tx, ty, tz, cache_dir)
self.downloader.download(tile_file, tile_url, http_headers)
tile_files[tile_url] = tile_file
# wait downloads to be finished
self.downloader.wait()
# validate all tiles
valid = True
for tile_url, tile_file in tile_files.iteritems():
if self.TILE_FORMAT == 'png' and imghdr.what(tile_file) != 'png':
sys.stderr.write("%s is not PNG image\n" % tile_url)
valid = False
if not valid:
return None
return tile_files.values()
def _get_tile_list(self):
"""Returns list of tiles needed to cover bounding box."""
tiles = []
tile_info = self._find_tiles()
if tile_info is not None:
(tminx, tminy, tmaxx, tmaxy, tz) = tile_info
for ty in range(tminy, tmaxy + 1):
for tx in range(tminx, tmaxx + 1):
tiles.append((tx, ty, tz))
return tiles
def _find_tiles(self):
"""Returns optimal zoom level based on given width."""
for zoom_level in range(1, self.max_zoom + 1):
tminx, tminy = self._lat_lon_to_tile(self.min_lat, self.min_lon, zoom_level)
tmaxx, tmaxy = self._lat_lon_to_tile(self.max_lat, self.max_lon, zoom_level)
x_tiles = tmaxx + 1 - tminx
if x_tiles > self.x_tiles_needed or zoom_level == self.max_zoom:
# optimal zoom level found
return (tminx, tminy, tmaxx, tmaxy, zoom_level)
return None
def _lat_lon_to_tile(self, lat, lon, zoom_level):
"""Converts given latLon to tile XY"""
mx, my = self.mercator.LatLonToMeters(lat, lon)
tx, ty = self.mercator.MetersToTile(mx, my, zoom_level)
return (tx, ty)
def _gen_tile_file(self, tx, ty, tz, cache_dir):
"""Returns filename where tile will be saved as."""
filename = "%d_%d_%d.%s" % (tx, ty, tz, self.TILE_FORMAT)
return os.path.join(cache_dir, filename)
from downloader import Downloader from accountlib import AccountManager import urllib2 #cookie = "SINAGLOBAL = 1791602201041.3557.1455610750935 ;ULV = 1455611177148:2:2:2:7716728692915.958.1455611177145:1455610750940 ;SUBP = 0033WrSXqPxfM725Ws9jqgMF55529P9D9WWNbPECqqxh2rppyWaDQBvZ5JpX5KMt ;SUHB = 0jBJqq9P-KpPgN ;un = [email protected] ;wvr = 6 ;SUS = SID-1340714021-1455611173-GZ-b2ey8-468e97b8ca4455bc4ba3beddabec7cd6 ;SUE = es%3D8484201c133ec33b03f1ed14aa4534fa%26ev%3Dv1%26es2%3D33ba64a44d9ac86cf555cf05bc397327%26rs0%3DM3QtGCbcUToEqKLA6eAZVpMrEX7u4bQVwvi5fHwr4DhrFNaB0594dwFDsL2CZMg5fRLrIkFt3zc9Bx10kzDewhd7AbovJSdm8cKV0c4V1VEfND1YM3XwCaiwZgbhwWc6jXLCbykNpryMLWTdianTFmPUmFrF0%252BazZmYEFLfT7ww%253D%26rv%3D0 ;SUP = cv%3D1%26bt%3D1455611174%26et%3D1455697574%26d%3Dc909%26i%3D7cd6%26us%3D1%26vf%3D0%26vt%3D0%26ac%3D0%26st%3D0%26uid%3D1340714021%26name%3Dguanglingsan1988%2540sina.com%26nick%3Dschumacher%26fmp%3D%26lcp%3D2012-02-02%252019%253A20%253A09 ;SUB = _2A257xq12DeRxGedN71IW8SrMyT2IHXVYtZm-rDV8PUNbvtBeLRnGkW9LHet86m9AJ9R6RMhU07ClXHxqCx1S0A.. ;ALF = 1487147173 ;SSOLoginState = 1455611174 ;_s_tentry = login.sina.com.cn ;UOR = ,,login.sina.com.cn ;Apache = 7716728692915.958.1455611177145 " cookie="SINAGLOBAL=3670791019162.063.1432519568807; ULV=1434184446636:3:1:1:9776813265987.371.1434184446576:1432539758675; SUHB=0S7S3YyGl7ABmk; YF-Ugrow-G0=169004153682ef91866609488943c77f; SUS=SID-5513307770-1434867321-GZ-fdui1-417cce02c02cba62afb4b09ce64141b5; SUE=es%3D77e325518a1eeaab4d42c04535d022d9%26ev%3Dv1%26es2%3Dda7c170b38a64fa4d9b6668f496fa074%26rs0%3DzdWWsJgKtTVoMTjEP3CWSLj5LpFJ5UF0%252BWyN6Q8Sd35saJbSk7N2YdacjGPXamqnsYetxrZNNIwMVsz0JNGf%252FkJZ%252FIv1Bh9YQHxwFkUE3K1i7kZDBboUO0yOR%252Fz0Ucw37WwoeeAGM28l5q%252FSbHFjWwe%252F3DJSj1ZdRE59Qrdrt%252Fo%253D%26rv%3D0; SUP=cv%3D1%26bt%3D1434867321%26et%3D1434953721%26d%3Dc909%26i%3D41b5%26us%3D1%26vf%3D0%26vt%3D0%26ac%3D17%26st%3D0%26uid%3D5513307770%26name%3Dmkqtx11141ua%2540163.com%26nick%3D%25E6%25AF%2581%25E9%25A6%2599%25E5%258A%2588%25E5%25BC%25B9%26fmp%3D%26lcp%3D2015-03-20%252000%253A59%253A43; YF-V5-G0=d22a701aae075ca04c11f0ef68835839; _s_tentry=login.sina.com.cn; UOR=,,login.sina.com.cn; Apache=9776813265987.371.1434184446576; YF-Page-G0=8fee13afa53da91ff99fc89cc7829b07; WBStore=0d3077cd0cad2262|undefined; SUB=_2A254giYpDeTxGeNL6lES8CnLzDyIHXVb9hDhrDV8PUJbvNBeLRjHkW8xuQX_wA9ncQZsaP1yWBfuXyq9-w..; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9W5iPVj051o7KP9Hly_f8Jud5JpX5K-t; ALF=1466403316; SSOLoginState=1434867321; wvr=6" url='http://weibo.com/aj/v6/mblog/info/big?ajwvr=6&id=3855095853042261&max_id=3856157879315930&page=1' '''manager = AccountManager() manager.init() manager.login() ''' '''headers = { 'User-Agent': 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.65 Safari/537.36', "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en-US,en;q=0.8,zh-CN;q=0.6,zh;q=0.4", "Accept-Encoding": "gzip,deflate,sdch", "Connection": "keep-alive", "Host": "weibo.com" } req = urllib2.Request(url, headers=headers) #print req,headers result = urllib2.urlopen(req, timeout=10) text = result.read() print text''' manager = AccountManager() manager.init() manager.login() downloader = Downloader(cookie) html_data = downloader.download(url) #print html_data
def download():
downloader = Downloader()
downloader.download()
pass
def main(args):
"""
Main function - launches the program.
:param args:
The Parser arguments
:type args:
Parser object
:returns:
List
:example:
>>> ["The latitude and longitude values must be valid numbers", 1]
"""
v = VerbosityMixin()
if args:
if 'clip' in args:
bounds = convert_to_float_list(args.clip)
else:
bounds = None
if args.subs == 'process':
verbose = True if args.verbose else False
force_unzip = True if args.force_unzip else False
stored = process_image(args.path, args.bands, verbose, args.pansharpen, args.ndvi, force_unzip,
args.ndvigrey, bounds)
if args.upload:
u = Uploader(args.key, args.secret, args.region)
u.run(args.bucket, get_file(stored), stored)
return ["The output is stored at %s" % stored]
elif args.subs == 'search':
try:
if args.start:
args.start = reformat_date(parse(args.start))
if args.end:
args.end = reformat_date(parse(args.end))
if args.latest > 0:
args.limit = 25
end = datetime.now()
start = end - relativedelta(days=+365)
args.end = end.strftime("%Y-%m-%d")
args.start = start.strftime("%Y-%m-%d")
except (TypeError, ValueError):
return ["Your date format is incorrect. Please try again!", 1]
s = Search()
try:
lat = float(args.lat) if args.lat else None
lon = float(args.lon) if args.lon else None
except ValueError:
return ["The latitude and longitude values must be valid numbers", 1]
address = args.address
if address and (lat and lon):
return ["Cannot specify both address and latitude-longitude"]
result = s.search(paths_rows=args.pathrow,
lat=lat,
lon=lon,
address=address,
limit=args.limit,
start_date=args.start,
end_date=args.end,
cloud_max=args.cloud)
if result['status'] == 'SUCCESS':
if args.json:
return json.dumps(result)
if args.latest > 0:
datelist = []
for i in range(0, result['total_returned']):
datelist.append((result['results'][i]['date'], result['results'][i]))
datelist.sort(key=lambda tup: tup[0], reverse=True)
datelist = datelist[:args.latest]
result['results'] = []
for i in range(0, len(datelist)):
result['results'].append(datelist[i][1])
result['total_returned'] = len(datelist)
else:
v.output('%s items were found' % result['total'], normal=True, arrow=True)
if result['total'] > 100:
return ['Over 100 results. Please narrow your search', 1]
else:
v.output(json.dumps(result, sort_keys=True, indent=4), normal=True, color='green')
return ['Search completed!']
elif result['status'] == 'error':
return [result['message'], 1]
elif args.subs == 'download':
d = Downloader(download_dir=args.dest)
try:
bands = convert_to_integer_list(args.bands)
if args.process:
if args.pansharpen:
bands.append(8)
if args.ndvi or args.ndvigrey:
bands = [4, 5]
if not args.bands:
bands = [4, 3, 2]
downloaded = d.download(args.scenes, bands)
if args.process:
if not args.bands:
args.bands = '432'
force_unzip = True if args.force_unzip else False
for scene, src in downloaded.iteritems():
if args.dest:
path = join(args.dest, scene)
else:
path = join(settings.DOWNLOAD_DIR, scene)
# Keep using Google if the image is before 2015
if src == 'google':
path = path + '.tar.bz'
stored = process_image(path, args.bands, False, args.pansharpen, args.ndvi, force_unzip,
args.ndvigrey, bounds=bounds)
if args.upload:
try:
u = Uploader(args.key, args.secret, args.region)
except NoAuthHandlerFound:
return ["Could not authenticate with AWS", 1]
except URLError:
return ["Connection timeout. Probably the region parameter is incorrect", 1]
u.run(args.bucket, get_file(stored), stored)
return ['The output is stored at %s' % stored, 0]
else:
return ['Download Completed', 0]
except IncorrectSceneId:
return ['The SceneID provided was incorrect', 1]
class ComicsViewer(ScalableWindow):
"""A program to display image feeds."""
__name__ = "Comics!"
__version__ = "1.0"
__author__ = "Moses"
__gsignals__ = {
"removed": (gobject.SIGNAL_RUN_FIRST, None, ()),
"updated": (gobject.SIGNAL_RUN_FIRST, None, (str,)),
}
########################################################################
# Helper methods #
########################################################################
def set_visibility(self, visible):
"""Hides or unhides the window. If Compiz is running and the Widget
plugin is loaded, the window is sent to the widget layer. Makes sure
that the window is visible on screen."""
if visible:
screen = self.get_screen()
screen_width, screen_height = (screen.get_width(), screen.get_height())
w, h = self.get_size()
x = self.__settings.get_int("x", (screen_width - w) / 2)
y = self.__settings.get_int("y", (screen_height - h) / 2)
# Show the window first...
self.show()
x %= screen_width
if x < 0:
x += screen_width
y %= screen_height
if y < 0:
y += screen_height
# this must be between self.show() and self.move()
# it calls self.get_position(), without it the comic
# will not be painted the very first time
self.save_settings()
# ...and then move it
self.move(x, y)
if has_widget_layer():
compiz_widget_set(self, False)
else:
if self.get_window():
self.save_settings()
if has_widget_layer():
self.show()
compiz_widget_set(self, True)
else:
self.hide()
def close(self):
self.destroy()
def get_menu_item_name(self, item):
"""Return the menu item name of item."""
if item[DATE] > 0:
tt = time.localtime(item[DATE])
# Translators: This is a date/time format string. You can check the
# output in terminal with 'date +"%%A %d %B"'.
return time.strftime(_("%A %d %B"), tt)
else:
return item[TITLE]
def get_link_name(self, item):
"""Return the link name of item."""
return item[TITLE]
def get_link_dimensions(self):
"""Return the dimensions of the link."""
if self.__link and self.show_link:
size = self.__link.rsize
if -1 in size:
return (0, 0)
else:
return size
else:
return (0, 0)
def get_ticker_dimensions(self):
"""Return the dimensions of the ticker."""
if self.__ticker:
return self.__ticker.rsize
else:
return (0, 0)
def get_image_dimensions(self):
"""Return the dimensions of the current image."""
if self.__pixbuf:
return (self.__pixbuf.get_width(), self.__pixbuf.get_height())
elif self.__is_error:
return ERROR_IMAGE.get_dimension_data()[:2]
else:
return DEFAULT_IMAGE.get_dimension_data()[:2]
def get_window_dimensions(self):
"""Get the required size of the window."""
l_dim = self.get_link_dimensions()
t_dim = self.get_ticker_dimensions()
i_dim = self.get_image_dimensions()
c_dim = max(i_dim[0], l_dim[0] + 2.0 * (t_dim[0] + TICKER_LINK_SEPARATION)), i_dim[1]
return (c_dim[0] + self.__border[0] + self.__border[2], c_dim[1] + self.__border[1] + self.__border[3])
def rescale_window(self):
"""Change the scale of the window so that the window will fit on
screen."""
screen = self.get_screen()
screen_size = screen.get_width(), screen.get_height()
w_dim = self.canvas_size
scale = None
if w_dim[0] > screen_size[0]:
scale = 0.95 * float(screen_size[0]) / w_dim[0]
if w_dim[1] > screen_size[1]:
if not scale or scale > screen_size[1] / w_dim[1]:
scale = 0.95 * float(screen_size[1]) / w_dim[1]
if scale:
self.set_scale(scale)
def update_size(self):
"""Update the size of the window and place controls."""
l_dim = self.get_link_dimensions()
t_dim = self.get_ticker_dimensions()
i_dim = self.get_image_dimensions()
c_dim = (max(i_dim[0], l_dim[0] + 2.0 * (t_dim[0] + TICKER_LINK_SEPARATION)), i_dim[1])
w_dim = (c_dim[0] + self.__border[0] + self.__border[2], c_dim[1] + self.__border[1] + self.__border[3])
# Resize window
self.set_canvas_size(w_dim)
self.rescale_window()
# Place link
if self.__link:
self.move_child(self.__link, (w_dim[0] - l_dim[0]) / 2, w_dim[1] - (self.__border[3] + l_dim[1]) / 2 - 1)
# Place ticker
if self.__ticker:
self.move_child(self.__ticker, TICKER_DISTANCE, w_dim[1] - t_dim[1] - TICKER_DISTANCE)
def select_item(self, item):
"""Select a strip for downloading."""
self.__ticker.set_ticking(True)
if self.__download_id and self.__downloader:
self.__downloader.disconnect(self.__download_id)
self.__download_id = None
self.__downloader = Downloader(item[URL], self.__settings["cache-file"])
self.__download_id = self.__downloader.connect("completed", self.on_download_completed, item)
self.__downloader.download()
def draw_frame(self, ctx, p1, p2, rad):
"""Trace a rectangle with rounded corners."""
ctx.set_line_cap(cairo.LINE_CAP_SQUARE)
# Top-left
if rad[0] > 0:
ctx.arc(p1[0] + rad[0], p1[1] + rad[0], rad[0], pi, 3 * pi / 2)
else:
ctx.move_to(*p1)
# Top-right
if rad[1] > 0:
ctx.arc(p2[0] - rad[1], p1[1] + rad[1], rad[1], 3 * pi / 2, 0)
else:
ctx.line_to(p2[0], p1[1])
# Bottom-right
if rad[2] > 0:
ctx.arc(p2[0] - rad[2], p2[1] - rad[2], rad[2], 0, pi / 2)
else:
ctx.line_to(*p2)
# Bottom-left
if rad[3] > 0:
ctx.arc(p1[0] + rad[3], p2[1] - rad[3], rad[3], pi / 2, pi)
else:
ctx.line_to(p1[0], p2[1])
# Close the path to draw the last line
ctx.close_path()
def draw_window(self, ctx):
"""Draw the window on which the image is painted."""
ctx.save()
# Define shape
w_dim = self.get_window_dimensions()
self.draw_frame(ctx, (1, 1), (w_dim[0] - 1, w_dim[1] - 1), self.__border_radii)
ctx.set_source_rgb(*WINDOW_COLOR)
ctx.fill_preserve()
ctx.set_source_rgb(*BORDER_COLOR)
ctx.stroke()
ctx.restore()
def draw_image(self, ctx):
"""Draw the image."""
ctx.save()
w_dim = self.get_window_dimensions()
i_dim = self.get_image_dimensions()
x = (w_dim[0] - i_dim[0]) / 2
# Has an image been downloaded?
if self.__pixbuf:
dim = self.get_image_dimensions()
ctx.set_source_pixbuf(self.__pixbuf, x, self.__border[1])
ctx.rectangle(x, self.__border[1], dim[0], dim[1])
ctx.clip()
ctx.paint()
# Has an error occurred?
elif self.__is_error:
ctx.set_operator(cairo.OPERATOR_OVER)
ctx.translate(x, self.__border[1])
ERROR_IMAGE.render_cairo(ctx)
# Otherwise draw the default image
else:
ctx.set_operator(cairo.OPERATOR_OVER)
ctx.translate(x, self.__border[1])
DEFAULT_IMAGE.render_cairo(ctx)
ctx.restore()
def make_menu(self):
"""Create the context menu."""
menu = gtk.Menu()
# Generate history menu
if len(self.feeds.feeds[self.feed_name].items) > 1:
history_container = gtk.ImageMenuItem(gtk.STOCK_JUMP_TO)
history_menu = gtk.Menu()
history_menu.foreach(lambda child: history_menu.remove(child))
items = self.feeds.feeds[self.feed_name].items.items()
items.sort(reverse=True)
for date, item in items:
label = gtk.Label()
text = self.get_menu_item_name(item)
if self.__current_timestamp == date:
label.set_markup("<b>" + text + "</b>")
else:
label.set_markup(text)
align = gtk.Alignment(xalign=0.0)
align.add(label)
menu_item = gtk.MenuItem()
menu_item.data = item
menu_item.connect("activate", self.on_history_activated)
menu_item.add(align)
history_menu.append(menu_item)
history_container.set_submenu(history_menu)
menu.append(history_container)
size_container = gtk.MenuItem(_("Size"))
size_menu = gtk.Menu()
zoom_normal_item = gtk.ImageMenuItem(gtk.STOCK_ZOOM_100)
zoom_normal_item.connect("activate", self.on_normal_activated)
zoom_in_item = gtk.ImageMenuItem(gtk.STOCK_ZOOM_IN)
zoom_in_item.connect("activate", self.on_larger_activated)
zoom_out_item = gtk.ImageMenuItem(gtk.STOCK_ZOOM_OUT)
zoom_out_item.connect("activate", self.on_smaller_activated)
size_menu.append(zoom_normal_item)
size_menu.append(zoom_in_item)
size_menu.append(zoom_out_item)
size_container.set_submenu(size_menu)
menu.append(size_container)
show_link_item = gtk.CheckMenuItem(_("Show link"))
show_link_item.set_active(self.show_link)
show_link_item.connect("toggled", self.on_show_link_toggled)
menu.append(show_link_item)
save_as_item = gtk.ImageMenuItem(stock_id="gtk-save-as")
save_as_item.set_sensitive(not self.__pixbuf is None)
save_as_item.connect("activate", self.on_save_as_activated)
menu.append(save_as_item)
close_item = gtk.ImageMenuItem(stock_id="gtk-close")
close_item.connect("activate", self.on_close_activated)
menu.append(close_item)
menu.show_all()
return menu
########################################################################
# Standard python methods #
########################################################################
def __init__(self, applet, settings, visible=False):
"""Create a new ComicsView instance."""
super(ComicsViewer, self).__init__()
self.applet = applet
self.__settings = settings
# Initialize fields
self.feeds = applet.feeds
self.__update_id = None
self.__download_id = None
self.__downloader = None
try:
self.__pixbuf = gdk.pixbuf_new_from_file(settings["cache-file"])
except Exception:
self.__pixbuf = None
self.__is_error = False
self.__link = WWWLink("", "", LINK_FONTSIZE)
self.__link.connect("size-allocate", self.on_link_size_allocate)
self.__ticker = Ticker((20.0, 20.0))
self.__current_timestamp = 0.0
self.__border = BORDER
self.__border_radii = BORDER_RADII
# Connect events
self.connect("destroy", self.on_destroy)
self.feeds.connect("feed-changed", self.on_feed_changed)
# Build UI
self.__link.connect("button-press-event", self.on_link_clicked)
self.put_child(self.__link, 0, 0)
self.put_child(self.__ticker, 0, 0)
self.__ticker.show()
self.set_skip_taskbar_hint(True)
self.set_skip_pager_hint(True)
self.load_settings()
if visible:
self.set_visibility(visible)
########################################################################
# Property updating methods #
########################################################################
def load_settings(self):
"""Load the settings."""
self.set_show_link(self.__settings.get_bool("show_link", False))
self.set_feed_name(self.__settings.get_string("feed_name", ""))
def save_settings(self):
"""Save the settings."""
x, y = self.get_position()
self.__settings["x"] = x
self.__settings["y"] = y
self.__settings.save()
def set_feed_name(self, new_feed_name):
"""Set the name of the feed to use."""
if self.__update_id:
self.feeds.feeds[self.feed_name].disconnect(self.__update_id)
self.__update_id = None
if new_feed_name in self.feeds.feeds:
self.feed_name = new_feed_name
self.__update_id = self.feeds.feeds[self.feed_name].connect("updated", self.on_feed_updated)
if self.feeds.feeds[self.feed_name].status == Feed.DOWNLOAD_OK:
self.on_feed_updated(self.feeds.feeds[self.feed_name], Feed.DOWNLOAD_OK)
self.__settings["feed_name"] = str(new_feed_name)
elif len(self.feeds.feeds) > 0:
self.set_feed_name(self.feeds.feeds.keys()[0])
def set_show_link(self, new_show_link):
"""Show or hide the link label."""
self.show_link = new_show_link
if self.show_link:
self.__link.show()
self.__border = LINK_BORDER
self.__border_radii = LINK_BORDER_RADII
else:
self.__link.hide()
self.__border = BORDER
self.__border_radii = BORDER_RADII
self.__settings["show_link"] = str(new_show_link)
self.update_size()
########################################################################
# Event hooks #
########################################################################
def on_destroy(self, widget):
self.emit("removed")
self.__settings.delete()
if self.__update_id:
self.feeds.feeds[self.feed_name].disconnect(self.__update_id)
self.__update_id = None
if self.__download_id and self.__downloader:
self.__downloader.disconnect(self.__download_id)
self.__download_id = None
del self.__pixbuf
def on_link_clicked(self, widget, e):
# Start the web browser in another process
os.system("%s %s &" % (BROWSER_COMMAND, widget.url))
def on_history_activated(self, widget):
# The widget is a GtkMenuItem, where data is a str
self.select_item(widget.data)
def on_normal_activated(self, widget):
self.set_scale(1.0)
def on_larger_activated(self, widget):
self.set_scale(self.scale * 1.2)
def on_smaller_activated(self, widget):
self.set_scale(self.scale / 1.2)
def on_show_link_toggled(self, widget):
self.set_show_link(widget.get_active())
def on_close_activated(self, widget):
self.close()
def on_save_as_activated(self, widget):
"""Run FileChooserDialog and save file."""
self.dialog = gtk.FileChooserDialog(
_("Save comic as…"),
buttons=(gtk.STOCK_CANCEL, gtk.RESPONSE_CANCEL, gtk.STOCK_SAVE, gtk.RESPONSE_OK),
action=gtk.FILE_CHOOSER_ACTION_SAVE,
)
self.dialog.set_icon_from_file(os.path.join(UI_DIR, "comics.svg"))
self.dialog.set_do_overwrite_confirmation(True)
self.dialog.set_current_name(self.__link.text + ".jpg")
# Set filters, default jpg, without ico
for format in gdk.pixbuf_get_formats():
if format["is_writable"] and format["name"] != "ico":
ff = gtk.FileFilter()
ff.set_name(format["description"])
for i in format["mime_types"]:
ff.add_mime_type(i)
self.dialog.add_filter(ff)
if format["name"] == "jpeg":
self.dialog.set_filter(ff)
self.dialog.connect("notify::filter", self.on_filter_changed)
if self.dialog.run() == gtk.RESPONSE_OK:
ff = gtk.FileFilter()
ff = self.dialog.get_filter()
name = ff.get_name()
for format in gdk.pixbuf_get_formats():
if format["description"] == name:
name = format["name"]
break
try:
self.__pixbuf.save(self.dialog.get_filename(), name)
except Exception:
self.applet.show_message(
_("Failed to save <i>%s</i>.") % self.dialog.get_filename(), gtk.STOCK_DIALOG_ERROR
)
self.dialog.destroy()
def on_filter_changed(self, pspec, data):
"""Change filename extension."""
current_name = self.dialog.get_filename().rsplit(".", 1)
if len(current_name) == 2:
ff = gtk.FileFilter()
ff = self.dialog.get_filter()
for i in gdk.pixbuf_get_formats():
if i["description"] == ff.get_name():
for ext in i["extensions"]:
if current_name[1] == ext:
return
self.dialog.set_current_name("%s.%s" % (os.path.basename(current_name[0]), i["extensions"][0]))
def on_link_size_allocate(self, widget, e):
i_dim = self.get_image_dimensions()
l_dim = self.get_link_dimensions()
t_dim = self.get_ticker_dimensions()
c_dim = (max(i_dim[0], l_dim[0] + 2.0 * (t_dim[0] + TICKER_LINK_SEPARATION)), i_dim[1])
w_dim = (c_dim[0] + self.__border[0] + self.__border[2], c_dim[1] + self.__border[1] + self.__border[3])
# Place link
if self.show_link:
self.move_child(self.__link, (w_dim[0] - l_dim[0]) / 2, w_dim[1] - (self.__border[3] + l_dim[1]) / 2 - 1)
# Does the window need to be resized?
if l_dim[0] > i_dim[0]:
self.set_canvas_size(w_dim)
def on_widget_show(self, widget):
"""Set the compiz widget property on all top-level windows."""
compiz_widget_set(widget, compiz_widget_get(self))
def on_draw_background(self, ctx):
"""Draw the window."""
self.draw_window(ctx)
self.draw_image(ctx)
def on_feed_updated(self, feed, result):
"""The feed has been updated."""
if result == Feed.DOWNLOAD_OK:
# Only emit the updated signal when there actually is an update
if self.__current_timestamp != 0.0:
self.emit("updated", feed.items[feed.newest][TITLE])
if feed.newest in feed.items:
self.select_item(feed.items[feed.newest])
def on_download_completed(self, o, code, item):
"""A new image has been downloaded."""
self.__ticker.set_ticking(False)
self.__is_error = code != Downloader.OK
self.__current_timestamp = item[DATE]
self.__link.set_text(self.get_link_name(item))
self.__link.set_url(item[LINK])
self.__downloader = None
self.__download_id = None
if not self.__is_error:
try:
del self.__pixbuf
except AttributeError:
# Received destroy signal after download was initiated
return
try:
self.__pixbuf = gdk.pixbuf_new_from_file(o.filename)
except gobject.GError:
self.__pixbuf = None
self.__is_error = True
self.update_size()
def on_feed_changed(self, feeds, feed_name, action):
"""A feed has been changed."""
if action == FeedContainer.FEED_REMOVED:
if self.feed_name == feed_name:
self.__settings.delete()
self.destroy()
class Controller:
def __init__(self):
self._conversion_tag = None
self._converter = Converter()
self._converter.complete_callback = self._on_convert_complete
self._converter.progress_callback = self._on_convert_progress_change
self._download_tag = None
self._downloader = Downloader()
self._downloader.complete_callback = self._on_download_complete
self._downloader.progress_callback = self._on_download_progress_change
self.view = mainform.MainForm(
parent=None,
title=metadata.title,
icon_path=_app_icon_path(),
conversions=Converter.get_conversions(),
)
self.view.url_textbox.Bind(wx.EVT_TEXT_ENTER, self._on_url_enter_press)
self.view.option_button.Bind(wx.EVT_BUTTON, self._on_option_click)
self.view.download_button.Bind(wx.EVT_BUTTON, self._on_download_click)
self.view.cancel_button.Bind(wx.EVT_BUTTON, self._on_cancel_click)
self.view.ok_button.Bind(wx.EVT_BUTTON, self._on_ok_click)
self.view.Bind(wx.EVT_ACTIVATE, self._on_set_focus)
self.view.Bind(wx.EVT_CLOSE, self._on_form_close)
self._auto_fill_url()
self.view.Show()
_initialize_log()
# self.view handlers
#======================================================================
def _on_cancel_click(self, event):
# TODO: cancel _parse_url()
self._downloader.cancel()
self._converter.cancel()
self.view.show_input_layout()
def _on_download_click(self, event):
self.view.show_loading_layout()
conversion_combobox = self.view.conversion_combobox
selected_index = conversion_combobox.GetSelection()
self._conversion_tag = conversion_combobox.GetClientData(
selected_index)
url = self.view.url_textbox.GetValue()
if not url:
self.view.show_input_layout()
return
if os.path.isfile(url):
if self._conversion_tag.is_valid():
self._converter.convert_file(url, self._conversion_tag)
return
self.view.show_input_layout()
return
_thread.start_new_thread(
_parse_url, (url, self._on_parse_url_complete))
def _on_form_close(self, event):
# TODO: cancel _parse_url()
self._downloader.cancel()
self.view.Destroy()
def _on_ok_click(self, event):
self.view.show_input_layout()
self._auto_fill_url()
def _on_option_click(self, event):
pass
def _on_set_focus(self, e):
if not e.GetActive():
return
if not self.view.url_textbox.IsShownOnScreen():
return
self._auto_fill_url()
def _on_url_enter_press(self, e):
self._on_download_click(e)
# thread handlers
#======================================================================
def _on_convert_complete(self, event):
self._notify_user(event.message)
def _on_convert_progress_change(self, event):
self.view.set_progress(event.percentage, event.message)
def _on_download_complete(self, event):
if event.error:
self.view.error_dialog(event.message)
self.view.show_input_layout()
return
if self._conversion_tag.is_valid():
source = event.destination
self._converter.convert_file(source, self._conversion_tag)
return
self._notify_user(event.message)
def _on_download_thumbnail_complete(self, info_dict):
if "data" in info_dict:
bitmap = info_dict["data"]
if bitmap:
self.view.set_thumbnail(bitmap)
def _on_download_progress_change(self, event):
self.view.set_progress(event.percentage, event.message)
def _on_parse_url_complete(self, download_tag):
self._download_tag = download_tag
if download_tag.error:
self.view.error_dialog(download_tag.error)
self.view.show_input_layout()
return
if download_tag.thumbnail_url:
_thread.start_new_thread(
_download_thumbnail,
(
download_tag.thumbnail_url,
self._on_download_thumbnail_complete
)
)
self.view.set_main_label(download_tag.title)
file_filter = "*{0}|*{0}".format(download_tag.ext)
destination = _prompt_to_save_file(self.view,
download_tag.title,
file_filter)
if not destination:
self.view.show_input_layout()
return
self._downloader.download(self._download_tag.video_url,
destination)
# private functions
#======================================================================
def _auto_fill_url(self):
url = _clipboard_url()
if not url:
return
if self.view.url_textbox.GetValue() == url:
return
self.view.set_input_text(url)
self.view.url_textbox.SetFocus()
self.view.url_textbox.SetInsertionPointEnd()
def _notify_user(self, message):
self.view.show_confirmation_layout(message)
if not self.view.IsActive():
self.view.RequestUserAttention()
class BlogParser:
def __init__(self,uid = '',nickname = '',imgurl = ''):
self.uid = uid
self.username = nickname
self.userimg = imgurl
self.downloader = Downloader()
self.blog = {
'uid': self.uid, #用户的id
'un': self.username, #用户用户名
'iu': self.userimg, #用户头像URL
'mid': '', #消息id
'mc': '', #消息内容
'nc': [], #消息中@的用户
'run': '', #转发的消息的用户名
'ruid': '', #转发的消息的用户ID
'rmc': '', #转发的消息的内容
#'pu': '', #消息中的图片
# deleted on 8/9
'rrc': '', #转发的消息的转发次数
'rcc': '', #转发的消息的评论次数
'rlc': '', #转发的消息的赞次数，14/9/2015
'rpage': '', #转发的消息的微博页面
'rpt': '', #转发的消息的发布时间
'rc': '0', #消息的转发次数
'cc': '0', #消息的评论次数
'lc': '0', #消息的赞次数，14/9/2015
'lp': [], #消息的赞的人，15/9/2015
'comment': [], #消息的评论，包括评论人，被评论人和评论内容
'repost': [], #消息的转发的人
'srn': '', #消息来源
'page': '', #消息的微博页面
'pt': '', #消息的发布时间
'feedpin': 0 #是否置顶
'location':''
}
pass
def init_user(self, uid='', imgurl='', nickname=''):
'''
用户信息的初始化
'''
self.uid = uid
self.username = nickname
self.userimg = imgurl
def init_blog(self):
self.blog = {
'uid': self.uid, #用户的id
'un': self.username, #用户用户名
'iu': self.userimg, #用户头像URL
'mid': '', #消息id
'mc': '', #消息内容
'nc': [], #消息中@的用户
'run': '', #转发的消息的用户名
'ruid': '', #转发的消息的用户ID
'rmc': '', #转发的消息的内容
# 'pu': '', #消息中的图片
'rrc': '', #转发的消息的转发次数
'rcc': '', #转发的消息的评论次数
'rlc': '', #转发的消息的赞次数，14/9/2015
'rpage': '', #转发的消息的微博页面
'rpt': '', #转发的消息的发布时间
'rc': '0', #消息的转发次数
'cc': '0', #消息的评论次数
'lc': '0', #消息的赞次数，14/9/2015
'lp': [], #消息的赞的人，15/9/2015
'comment': [], #消息的评论，包括评论人，被评论人和评论内容
'repost': [], #消息的转发的人
'srn': '', #消息来源
'page': '', #消息的微博页面
'pt': '', #消息的发布时间
'feedpin': 0 #是否置顶
'location':''
}
def parse(self, html):
'''
解析给定html字符串里面的微博数据
-----------------------------------------
html: 给定的html字符串
--------------------------------------
return: blog列表
'''
bpos = html.find('<!--feed内容-->')
epos = html.find('<!--翻页-->', bpos)
bloghtml = html[bpos:epos].replace('\\/', '/') + '</div>' # the original one was not complete
soup = BeautifulSoup(bloghtml)
blogsouplist = soup.find_all('div', class_ = 'WB_feed_type')
#class_='WB_cardwrap WB_feed_type S_bg2 ')
bloglist = []
for blogsoup in blogsouplist:
self.init_blog()
self._parse_blog(blogsoup)
bloglist.append(self.blog)
return bloglist
def _parse_blog(self, blogsoup):
'''
解析一条微博文本
----------------------------------------
blogsoup: 用BeautifulSoup包装的微博文本
'''
raise NotImplementedError()
def _get_attr(self, soup, attr):
'''
获取soup对象的attr属性值
-------------------------------
soup: 获取属性值的soup对象
attr: 待获取的属性值
-------------------------------
return: 成功则返回获取的属性值,否则返回空串
'''
attrvalue = ''
'''
if( not attr in soup):
print "soup"
print soup
'''
if soup:
attrvalue = soup[attr]
return attrvalue
def _get_text(self, soup):
'''
获取soup对象的text
--------------------------------
soup: 待获取text的soup对象
--------------------------------
return: 获取到的text值或返回空串
'''
text = ''
if soup:
text = soup.get_text().strip()
tag = '//'
pos = text.find(tag)
# modified on 8/9
if(pos != -1):
text = text[:pos]
return text
def _parse_statistics(self, statsoup):
'''
获取 转发/评论/赞 数
---------------------------------------
statsoup: 统计数据所在html片段的BSoup对象
---------------------------------------
return: 转发数,评论数
'''
forwardcount = '0'
commentcount = '0'
likecount = '0'
statstr = str(statsoup)
bpos = statstr.rfind('转发')
if bpos != -1:
epos = statstr.find('<', bpos)
forwardcount = statstr[bpos + 6:epos]
forwardcount = forwardcount.strip()
if(len(forwardcount)==0):
forwardcount = '0'
bpos = statstr.rfind('评论')
if bpos != -1:
epos = statstr.find('<', bpos)
commentcount = statstr[bpos + 6:epos]
commentcount = commentcount.strip()
try:
if(len(commentcount) != 0):
temp = int(commentcount)
except:
import pdb
pdb.set_trace()
if(len(commentcount)==0):
commentcount = '0'
like_part = statsoup.find('span',attrs = {'node-type':'like_status'})
like_num = like_part.find('em').contents
if(not len(like_num)):
likecount = 0
else:
likecount = like_num[0].strip()
#print like_num
return forwardcount, commentcount, likecount
def _parse_blog_from(self, fromsoup):
'''
获取微博的 地址/时间/来源 等信息
--------------------------------------
fromsoup: 包含以上信息片段的soup
--------------------------------------
return: 地址,时间,来源
'''
page = ''
pt = ''
srn = ''
asoup = fromsoup.find('a', attrs={'node-type': 'feed_list_item_date'})
if asoup:
pt = asoup['title']
page = "http://www.weibo.com" + asoup['href']
srnsoup = fromsoup.find('a', attrs={'action-type': 'app_source'})
if srnsoup:
srn = srnsoup.get_text().strip()
return page, pt, srn
def _parse_at_people(self,fromsoup):
'''
get the people @ in this blog
'''
content = fromsoup.find('div',attrs = {'class':'WB_text W_f14'})
if( '//' in content.contents[0]):
return []
friendlist = content.find_all('a',attrs = {'target':'_blank','render':'ext'})
atlist = []
if(friendlist):
for friend in friendlist:
text = friend.contents[0]
#if( friend['class'] == 'a_topic'):
if (friend['extra-data'] == 'type=topic'):
if('//' in text):
break
else:
continue
user_url = friend['href']
btag = 'weibo.com/n/'
etag = '?from=feed'
bpos = user_url.find(btag)+len(btag)
epos = user_url.find(etag)
username = user_url[bpos:epos]
url = 'http://weibo.com/aj/v6/user/newcard?ajwvr=6&name='+username+'&type=1'
userpage = self.downloader.download(url)
if(userpage.find('"code":"100001"') != -1):
continue
btag = 'uid="'
etag = '"'
bpos = userpage.find(btag)+len(btag)
epos = userpage[bpos:].find(etag)+bpos
oid = int(userpage[bpos:epos])
atlist.append(oid)
if('//' in text):
break
return atlist
def _parse_like(self):
pagenum = int(self.blog['lc'])/30+1
like_list = []
if(int(self.blog['lc'])%30 == 0):
pagenum = pagenum-1
for i in range(1,pagenum+1):
url = 'http://www.weibo.com/aj/v6/like/big?ajwvr=6&mid='+self.blog['mid']+'&page='+str(i)
content = self.downloader.download(url)
content = content.replace('\\/','/')
btag = '<div'
etag = '</div>'
bpos = content.find(btag)
epos = content.rfind(etag)
content = content[bpos:epos]
soup = BeautifulSoup(content)
liked = soup.find_all('li')
for item in liked:
if(item['uid']):
like_list.append(item['uid'])
return like_list
def _parse_comment(self):
pagenum = int(self.blog['cc'])/20+1
if(int(self.blog['cc'])%20 == 0):
pagenum = pagenum-1
comment_list = []
username = ''
for i in range(1,pagenum+1): # get from each page
url = 'http://www.weibo.com/aj/v6/comment/big?ajwvr=6&id='+self.blog['mid']+'&page='+str(i)
content = self.downloader.download(url)
content = content.replace('\\/','/')
btag = '<div'
etag = '</div'
bpos = content.find(btag)
epos = content.rfind(etag)
content = content[bpos:epos]
soup = BeautifulSoup(content)
comment = soup.find_all('div',attrs={'class':'list_li S_line1 clearfix'})
# get from each comment
for item in comment:
user = item.find('div', attrs ={'class':'WB_text'})
user_con = user.contents
user_at = 0
user_res = user_con[1]['usercard'][3:]
username = user_con[1].contents[0]
con_len = len(user_con)
for i in range(2,con_len):
if (user_con[i] == '//'):
break
else:
if(type(user_con[i]) != bs4.element.NavigableString):
con_dic = user_con[i].attrs
if('extra-data' in con_dic.keys()):
if(user_con[i]['extra-data'] == 'type=atname'):
user_url = user_con[i]['href']
btag = 'weibo.com/n/'
etag = '?from=feed&loc=at'
bpos = user_url.find(btag)+len(btag)
epos = user_url.find(etag)
username = user_url[bpos:epos]
url = 'http://weibo.com/aj/v6/user/newcard?ajwvr=6&name='+username+'&type=1'
username = user_con[i]['usercard'][5:]
userpage = self.downloader.download(url)
btag = 'uid="'
etag = '"'
if(userpage.find(btag) != -1):
bpos = userpage.find(btag)+len(btag)
epos = userpage[bpos:].find(etag)+bpos
user_at = userpage[bpos:epos]
else:
user_at = '0'
break
# modified because there might be some at in the comment that distort the information
# tested on Tangwei's weibo
# modified on 20/9/2015
if(user_at):
users = [user_res,user_at]
else:
users = [user_res,'0']
text = item.find('div',attrs = {'class':'WB_text'})
comment_cont = text.get_text().strip()
com_pos = comment_cont.find(username)
comment_cont = comment_cont[com_pos+len(username)+1:]
tag = '//'
pos = comment_cont.find(tag)
if(pos != -1):
comment_cont = comment_cont[:pos]
time = item.find('div',attrs = {'class':'WB_from S_txt2'})
time = time.contents[0]
locol_time = datetime.datetime.now()
if(time.find('分钟前')+time.find('今天') != -2):
str_time = locol_time.strftime("%Y-%m-%d")
elif(time.find('昨天') != -1):
comment_time = locol_time-datetime.timedelta(days = 1)
str_time = comment_time.strftime("%Y-%m-%d")
else:
comment_time = time.split()[0]
if(comment_time.find('月') != -1):
mtag = '月'
dtag = '日'
pos1 = comment_time.find(mtag)
pos2 = comment_time.find(dtag)
month = comment_time[:pos1]
day = comment_time[pos1+1:pos2]
str_time = '2015-'+month+'-'+day
else:
str_time = comment_time
comment_list.append([users,comment_cont,str_time])
return comment_list
def _parse_repost(self):
pagenum = int(self.blog['rc'])/20+1
if(int(self.blog['rc'])%20 == 0):
pagenum = pagenum-1
repost_list = []
for i in range(1,pagenum+1):
url = 'http://www.weibo.com/aj/v6/mblog/info/big?ajwvr=6&id='+self.blog['mid']+'&page='+str(i)
content = self.downloader.download(url)
content = content.replace('\\/','/')
btag = '<div'
etag = '</div'
bpos = content.find(btag)
epos = content.rfind(etag)
content = content[bpos:epos]
soup = BeautifulSoup(content)
repost = soup.find_all('div',attrs = {'class':'list_li S_line1 clearfix'})
for item in repost:
repost_data = item.find('div',attrs = {'class':'WB_face W_fl'})
repost_data = repost_data.find('img')
repost_user = repost_data['usercard'][3:]
repost_list.append(repost_user)
return repost_list
# only contain the users' ID number, the text together was omitted
def _parse_location(self,soup):
loc = soup.find('a',attrs = {'class':'W_btn_b btn_22px W_btn_cardlink'})
location = loc['title']
return location
def main(args):
"""
Main function - launches the program
"""
v = VerbosityMixin()
if args:
if args.subs == 'process':
verbose = True if args.verbose else False
try:
bands = convert_to_integer_list(args.bands)
p = Process(args.path, bands=bands, verbose=verbose)
except IOError:
exit("Zip file corrupted", 1)
except FileDoesNotExist as e:
exit(e.message, 1)
stored = p.run(args.pansharpen)
exit("The output is stored at %s" % stored)
elif args.subs == 'search':
try:
if args.start:
args.start = reformat_date(parse(args.start))
if args.end:
args.end = reformat_date(parse(args.end))
except (TypeError, ValueError):
exit("You date format is incorrect. Please try again!", 1)
s = Search()
try:
lat = float(args.lat) if args.lat else None
lon = float(args.lon) if args.lon else None
except ValueError:
exit("The latitude and longitude values must be valid numbers", 1)
result = s.search(paths_rows=args.pathrow,
lat=lat,
lon=lon,
limit=args.limit,
start_date=args.start,
end_date=args.end,
cloud_max=args.cloud)
if result['status'] == 'SUCCESS':
v.output('%s items were found' % result['total'], normal=True, arrow=True)
if result['total'] > 100:
exit('Over 100 results. Please narrow your search', 1)
else:
v.output(json.dumps(result, sort_keys=True, indent=4), normal=True, color='green')
exit('Search completed!')
elif result['status'] == 'error':
exit(result['message'], 1)
elif args.subs == 'download':
d = Downloader(download_dir=args.dest)
try:
if d.download(args.scenes, convert_to_integer_list(args.bands)):
exit('Download Completed', 0)
except IncorrectSceneId:
exit('The SceneID provided was incorrect', 1)
class IRCClient(SimpleIRCClient):
servername = 'irc.irchighway.net'
port = 6667
channel = '#lurk'
manga_dir = 'manga'
def __init__(self):
SimpleIRCClient.__init__(self)
self.connection_status = 'not_connected'
self.downloader = Downloader(self.connection)
"""Connecting to the manga server"""
def connect(self, nickname, password):
if self.connection_status == 'connected':
return
try:
server = socket.gethostbyname(self.servername)
except socket.error:
print 'connect failed: make sure that you are connected to the internet and try again'
try:
self.connection.connect(server, self.port, nickname, password)
self.connection.user(nickname, nickname)
print 'connecting'
except:
print 'connect failed: make sure that you are connected to the internet and try again'
"""Start the client"""
def start(self):
while 1:
self.downloader.check()
self.ircobj.process_once(0.2)
"""Request the client to download a manga"""
def download(self, manga, manga_directory = 'manga'):
self.downloader.download(manga)
"""Get connection_status"""
def connection_status(self):
return self.connection_status
def on_welcome(self, connection, event):
self.connection.join(self.channel)
self.connection_status = 'connected'
self.downloader.start()
print 'connected'
def on_ctcp(self, connection, event):
try:
# print event.arguments()
args = event.arguments()[1].split()
except IndexError:
return
if args[0] != "SEND":
return
# DCC SEND requested
# check file extension
file_name = os.path.basename(args[1])
file_ext = os.path.splitext(file_name)[1]
file_size = int(args[4])
if file_ext not in ['.zip', '.rar']:
return
file_name = os.path.join(self.manga_dir, file_name)
# initiate DCC connection
peeraddress = irclib.ip_numstr_to_quad(args[2])
peerport = int(args[3])
self.dcc = self.dcc_connect(peeraddress, peerport, 'raw')
self.downloader.initiate_download(file_name, file_size, self.dcc)
def on_dccmsg(self, connection, event):
data = event.arguments()[0]
self.downloader.receive_data( data )
def on_dcc_disconnect(self, connection, event):
self.downloader.finish_download()
self.downloader.download_next()
def on_disconnect(self, connection, event):
self.connection_status = 'disconnected'
self.downloader.stop()
class FollowCrawler(object):
def __init__(self,uid,origin=0):
self.downloader = Downloader()
self.list = []
self.uid = uid
self.origin = origin
if(origin):
self.parser = FollowParser2()
self.per = 30
else:
self.parser = FollowParser1()
self.per = 20
self.friendNum = self._parse_friendnum(uid,origin)
#print "Follow #: ",self.friendNum
# origin==1, means it's the user itself, the html is a little different
def _parse_friendnum(self,uid,origin=0):
'''
count the total number of blogs
only used at the original node
'''
if(origin):
url = 'http://weibo.com/p/1005052198840781/myfollow?relate=fans'
content = self.downloader.download(url)
btag = '关注<\/span><em class="num S_txt1">'
etag = '<\/em>'
bpos = content.find(btag)+len(btag)
epos = content[bpos:].find(etag)+bpos
return int(content[bpos:epos])
else:
url = 'http://weibo.com/p/100505'+str(uid)+'/follow?relate=fans&from=100505&wvr=6&mod=headfans¤t=fans'
content = self.downloader.download(url)
'''
w = open("C:/Users/hp1/Desktop/weibo_crawler/test4.txt","w")
w.write(content)
w.close()
'''
btag1 = '的关注<\/span>'
btag2 = '>'
etag = '<\/em>'
bpos = content.find(btag1)
if(bpos == -1):
return 0
epos = content[bpos+len(btag1):].find(etag)+bpos+len(btag1)
bpos = content[:epos].rfind(btag2)+len(btag2)
if(epos==0 or bpos == 0):
return 0
return int(content[bpos:epos])
def get_url(self,uid,page,origin=0):
if(origin):
url = 'http://weibo.com/p/1005052198840781/myfollow?t=1&pids=Pl_Official_RelationMyfollow__108&cfs=&Pl_Official_RelationMyfollow__108_page='+str(page)
else:
url = 'http://weibo.com/p/100505'+str(uid)+'/follow?page='+str(page)
return url
# get the friends url, could only get 10 pages now
def _process_html(self, content):
"""
对下载的网页进行预处理,主要是替换\/为/
"""
if content:
return content.replace('\\/', '/')
return ''
def scratch(self):
'''
filepath = self.get_filepath(self.uid)
if os.path.isfile(filepath): # the friends of this user has been downloaded
print self.uid, u'用户的好友已下载！'
return None
'''
pageNum = min(10,int((self.friendNum-1)/self.per)+1)
print "Follows #:",self.friendNum
# should be careful here
# 10 is the maximun number of pages we could get
for i in range(1,pageNum+1):
html = self.downloader.download(self.get_url(self.uid,i,self.origin))
html = html.decode('utf-8','ignore')
new_friend = self.parse_friend(html,self.origin)
if(new_friend):
self.list.append(new_friend)
return self.list
def parse_friend(self,content,origin=0):
if(not origin):
if(self.friendNum >self.per):
etag = '<div class="WB_cardpage S_line1" node-type="pageList">'
btag = '<ul class="follow_list" node-type="userListBox">'
bpos = content.find(btag)
epos = content.find(etag)
else:
etag = '\/ul'
btag = '<div class="follow_inner">'
bpos = content.find(btag)+len(btag)
epos = content[bpos:].find(etag)+bpos
else:
btag = '<ul class="member_ul clearfix" node-type="relation_user_list">'
etag = '<div class="WB_cardpage S_line1"'
bpos = content.find(btag)
epos = content.find(etag)
content = content[bpos:epos]
content = self._process_html(content)
if(len(content) == 0):
print "Follows matching failed."
return None
soup = BeautifulSoup(content)
return self.parser.parse(soup,self.origin)
class TileLoader(object):
TILE_WIDTH = 256 # tile is square
MAX_ZOOM = 18
TILE_FORMAT = 'png'
def __init__(self, min_lat, min_lon, max_lat, max_lon, width):
self.tiles = []
self.min_lat = min_lat
self.min_lon = min_lon
self.max_lat = max_lat
self.max_lon = max_lon
self.mercator = GlobalMercator()
self.downloader = Downloader()
# count how many horizontal tiles we need
self.x_tiles_needed = math.ceil(width / self.TILE_WIDTH)
def download(self, cache_dir, url, http_headers):
"""Downloads tiles and returns list of downloaded tiles."""
tile_files = []
tiles = self._get_tile_list()
for (tx, ty, tz) in tiles:
cx, cy, cz = self._convert_tile(tx, ty, tz)
tile_url = url.replace('{x}', str(cx)).replace('{y}', str(cy)).replace('{z}', str(cz))
tile_file = self._gen_tile_file(tx, ty, tz, cache_dir)
self.downloader.download(tile_file, tile_url, http_headers)
tile_files.append(tile_file)
# wait downloads to be finished
self.downloader.wait()
return tile_files
def _get_tile_list(self):
"""Returns list of tiles needed to cover bounding box."""
tiles = []
tile_info = self._find_tiles()
if tile_info is not None:
(tminx, tminy, tmaxx, tmaxy, tz) = tile_info
for ty in range(tminy, tmaxy + 1):
for tx in range(tminx, tmaxx + 1):
tiles.append((tx, ty, tz))
return tiles
def _find_tiles(self):
"""Returns maximum zoom level based on given width."""
for zoom_level in range(1, self.MAX_ZOOM + 1):
tminx, tminy = self._lat_lon_to_tile(self.min_lat, self.min_lon, zoom_level)
tmaxx, tmaxy = self._lat_lon_to_tile(self.max_lat, self.max_lon, zoom_level)
x_tiles = tmaxx + 1 - tminx
if x_tiles > self.x_tiles_needed:
# minimum zoom level found
return (tminx, tminy, tmaxx, tmaxy, zoom_level)
# no suitable zoom level found, maybe too many tiles needed
return None
def _lat_lon_to_tile(self, lat, lon, zoom_level):
"""Converts given latLon to tile XY"""
mx, my = self.mercator.LatLonToMeters(lat, lon)
tx, ty = self.mercator.MetersToTile(mx, my, zoom_level)
return (tx, ty)
def _gen_tile_file(self, tx, ty, tz, cache_dir):
"""Returns filename where tile will be saved as."""
filename = "%d_%d_%d.%s" % (tx, ty, tz, self.TILE_FORMAT)
return os.path.join(cache_dir, filename)
