def main():
"""pass"""
# load weibonames
path = os.path.abspath(
os.path.join(os.path.dirname(__file__), 'conf/weibonames.conf'))
with open(path) as fp:
weibonames = [
line.strip().decode('utf_8') for line in fp
if not line.startswith('#')
]
# create database
dbpath = os.path.abspath(os.path.join(os.path.dirname(__file__), DBNAME))
if not os.path.exists(dbpath):
create_database(dbpath)
# start crawler thread
for name in weibonames:
crawler = CrawlerThread(weiboname=name)
crawler.start()
# start download thread
downloader = DownloadManager()
downloader.start()
def __init__(self, webconf):
self.webconf = webconf
config = ConfigParser.ConfigParser()
config.readfp(open('/etc/raspadmin/downloader.conf'))
self.path = config.get("PATH", "downloadrep")
self.alldebrid = 0
try:
if config.get("ALLDEBRID", "usealldebrid").upper() == "Y":
self.alldebriduser = config.get("ALLDEBRID", "alldebriduser")
self.alldebridpass = config.get("ALLDEBRID", "alldebridpass")
self.alldebrid = AllDebrid(self.alldebriduser,
self.alldebridpass)
except:
self.alldebrid = None
self.downloadManager = DownloadManager(self.path)
rarfile.UNRAR_TOOL = "unrar-nonfree"
def main(ctx, url, name, num_threads):
num_downloads = num_threads
url = url
file_name = name
if not file_name:
file_name = parse_filename(url)
with OneLinePrinter("Initializing..."):
manager = DownloadManager(url)
manager.create_downloads(num_downloads)
file_size = manager.size
formatter = ProgressBarFormatter('Downloading')
with OneLinePrinter(formatter):
with DownloadFile(file_name, file_size) as f:
for download in manager.iter_downloads():
b = formatter.create_bar(download.length)
formatter.add_bar(b)
download.bind(on_chunk, b, f)
download.start()
for download in manager.iter_downloads():
download.join()
def handleRolloutRequest(cmd):
print("Rollouts correlationId is: " + str(cmd.getRolloutsCorrelationId()))
# print('Parsing software module information')
for swMod in cmd.getSoftwareModules():
execResult = ""
featureId = swMod.name.replace(":", "-") + "-" + swMod.version
swCache = SoftwareFeatureCache.loadOrCreate(featureId)
# print(swMod.toJson())
for art in swMod.artifacts:
updateLastOperation(cmd, "DOWNLOADING", "Downloading " + art.name,
swMod)
filePath = DownloadManager().download(art)
swCache.addFile(filePath)
updateLastOperation(cmd, "DOWNLOADED", "Downloaded " + art.name,
swMod)
# # https://vorto.eclipseprojects.io/#/details/vorto.private.test:Executor:1.0.0
updateLastOperation(cmd, "INSTALLING",
"Executing script: " + filePath, swMod)
res = "Installed a script to the location {}.\n".format(filePath)
updateLastOperation(cmd, "INSTALLED", execResult, swMod)
execResult += res
swCache.save()
swCache.updateDittoFeature(client, deviceInfo, execResult)
updateLastOperation(cmd, "FINISHED_SUCCESS", execResult, swMod)
# form
def get_form(self, index):
form = self.doc.forms[index]
form.action = urlparse.urljoin(self.url, form.action)
return form.action, form.fields
#
def get_html(self):
return self.html
if __name__ == "__main__":
import time
from downloader import DownloadManager
downloader = DownloadManager()
url = "http://www.cs.colorado.edu/"
error_msg, url, redirected_url, html = downloader.download(url)
print error_msg, url, redirected_url, len(html)
time.sleep(2)
page = WebPage(url, html)
page.parse_links()
links = page.filter_links(tags=['a'],patterns=['^(http://www\.cs\.colorado\.edu)(/info.+)$'])
elements = page.doc.findall('./body//div')
for e in elements:
print "ELEMETNS =========================================="
print lxml.html.tostring(e,pretty_print=True)
print "ITEMS------------------------------------------"
def __init__(self):
super(Crawler, self).__init__()
self.downloader = DownloadManager()
self.webpage = None
self.init_database()
self.rules = {}
def __init__(self, webconf):
self.webconf = webconf
config = ConfigParser.ConfigParser()
config.readfp(open('/etc/raspadmin/downloader.conf'))
self.path = config.get("PATH", "downloadrep")
self.downloadManager = DownloadManager(self.path)
def create_downloader(self):
# Instantiate class that takes care of downloading videos
dl = DownloadManager(self.app.save_dir)
return dl
def __init__(self):
self.downloader = DownloadManager()
self.webpage = None
self.rules = {}
self.dbop = OperatorDB()
