def __init__(self, handle):
        ''' Initialize the toolbars and the reading board '''
        super(ICanReadActivity, self).__init__(handle)
        self.reading = False
        self.testing = False
        self.recording = False
        self.grecord = None
        self.datapath = get_path(activity, 'instance')

        if 'LANG' in os.environ:
            language = os.environ['LANG'][0:2]
        elif 'LANGUAGE' in os.environ:
            language = os.environ['LANGUAGE'][0:2]
        else:
            language = 'es'  # default to Spanish

        # FIXME: find some reasonable default situation
        language = 'es'

        if os.path.exists(os.path.join('~', 'Activities',
                                       'ICanRead.activity')):
            self._lessons_path = os.path.join('~', 'Activities',
                                              'ICanRead.activity', 'lessons',
                                              language)
        else:
            self._lessons_path = os.path.join('.', 'lessons', language)

        self._images_path = self._lessons_path.replace('lessons', 'images')
        self._sounds_path = self._lessons_path.replace('lessons', 'sounds')
        self._setup_toolbars()

        # Create a canvas
        self.scrolled_window = gtk.ScrolledWindow()
        self.set_canvas(self.scrolled_window)
        self.scrolled_window.set_policy(gtk.POLICY_NEVER, gtk.POLICY_AUTOMATIC)
        self.scrolled_window.show()
        canvas = gtk.DrawingArea()
        width = gtk.gdk.screen_width()
        height = int(gtk.gdk.screen_height() * 2.75)
        canvas.set_size_request(width, height)
        self.scrolled_window.add_with_viewport(canvas)
        canvas.show()

        self._level = self._levels_combo.get_active()
        self._page = Page(canvas,
                          self._lessons_path,
                          self._images_path,
                          self._sounds_path,
                          self._levels[self._level],
                          parent=self)

        # Restore state from Journal or start new session
        if 'page' in self.metadata:
            self._restore()
        else:
            self._page.new_page()

        # Set up sound combo box
        self._reload_sound_combo()
        self._selected_sound = self.sounds_combo.get_active()
Пример #2
0
def crawler(name, query):
    '''Main Crawler for Job'''
    start = datetime.now()
    print name
    db = Database(name)
    db.create_colls()
    #get from source
    for n in db.sources.find():
        if n["url"] not in db.queue.distinct("url"):
            db.queue.insert(n)

    while db.queue.count > 0:

        print "Beginning crawl"
        # print "Number of seeds urls in sources databases:", db.sources.count()
        # print "Number of pending url to inspect:", len(db.queue.distinct("url"))
        for url in db.queue.distinct("url"):

            if url not in db.results.find({"url": url}):
                print url
                p = Page(url, query)
                if p.create():
                    a = Article()
                else:
                    print p.error_type

                #print "Links", p.outlinks
                #db.results.update(p.info, {'$push': {"date": datetime.today()}}, upsert=True)
                #db.results.insert(p.info)
                # if p.outlinks is not None:
                # 	try:
                # 		for n_url in p.outlinks:
                # 			if n_url is not None or  n_url not in db.queue.find({"url":n_url}) or n_url not in db.results.find({"url":n_url}) or n_url not in db.log.find({"url":n_url}):
                # 				# Checking correct url before is problematic
                # 				# next_p = Page(n_url, query)
                # 				# if next_p.clean_url(p.url) is not None:
                # 				print n_url
                # 				db.queue.insert({"url":n_url})
                # 	except mongo_err:
                # 		db.log.udpate({"url":url, "error_type": "pymongo error inserting outlinks", "query": self.query, "status":False},{'$push': {"date": datetime.today()}}, upsert=True)
                # elif p.error_type != 0:
                # 	''' if the page is not relevant do not store in db'''
                # 	db.log.update(p.bad_status(),{'$push':{"date": datetime.today()}}, upsert=True)
                # else:
                # 	continue

            db.queue.remove({"url": url})
            if db.queue.count() == 0:
                print db.stats()
                break

        if db.queue.count() == 0:
            print db.stats()
            break

    end = datetime.now()
    elapsed = end - start
    print "crawl finished, %i results and %i sources are stored in Mongo Database: %s in %s" % (
        db.results.count(), db.sources.count(), name, elapsed)
    return True
Пример #3
0
    def on_init(self):
        Page.on_init(self)
        self.set_background_color(255, 255, 255)
        self.insert_vertical_image("%s-vertical.bmp" % self.info.cd_distro.name)

        # navigation
        self.insert_navigation(_("< Back"), _("Finish"), _("Cancel"), default=2)
        self.navigation.button1.on_click = self.on_back
        self.navigation.button2.on_click = self.on_finish
        self.navigation.button3.on_click = self.on_cancel

        # main container
        self.insert_main()
        self.main.set_background_color(255, 255, 255)
        self.main.title = ui.Label(self.main, 40, 20, self.main.width - 80, 60, _("Reboot required"))
        self.main.title.set_font(size=20, bold=True, family="Arial")
        txt = _(
            "To start the Live CD you need to reboot your machine leaving the CD in the tray. If your machine cannot boot from the CD, the last option should work in most cases."
        )
        self.main.label = ui.Label(self.main, 40, 90, self.main.width - 80, 40, txt)
        self.main.reboot_now = ui.RadioButton(self.main, 60, 150, self.main.width - 100, 20, _("Reboot now"))
        self.main.reboot_later = ui.RadioButton(
            self.main, 60, 180, self.main.width - 100, 20, _("I want to manually reboot Later")
        )
        self.main.cd_boot = ui.RadioButton(self.main, 60, 210, self.main.width - 100, 20, _("Help me to boot from CD"))
        self.main.reboot_later.set_check(True)
Пример #4
0
def generate_tree(directory):
    """Take a directory and generate a tree of ``Page`` objects."""

    # Find the index file and create the root
    fn = os.path.join(directory, 'index.html')
    name = os.path.split(directory)[-1]
    try:
        root = Page(name, fn)
    except IOError:
        raise IOError('Could not find `%s`.' % fn)

    # Iterate through the directory and create all the other pages.
    for item in os.listdir(directory):
        if os.path.basename(item)[0] == '.':  # Ignore hidden files
            continue
        fn = os.path.join(directory, item)
        # If it's a directory, put on my recursion shoes.
        if os.path.isdir(fn):
            root[item] = generate_tree(fn)
        else:
            name = '.'.join(os.path.basename(fn).split('.')[:-1])
            if name == 'index':  # Ignore index file, we've already done that.
                continue
            root[item] = Page(name, fn)

    return root
    def on_init(self):
        Page.on_init(self)
        self.set_background_color(255, 255, 255)
        self.insert_vertical_image("%s-vertical.bmp" % self.info.distro.name)

        #navigation
        self.insert_navigation(_("Finish"), default=1)
        self.navigation.button1.on_click = self.on_finish

        #main container
        self.insert_main()
        self.main.set_background_color(255, 255, 255)
        self.main.title = ui.Label(
            self.main, 40, 20, self.main.width - 80, 60,
            _("Completing the %s Setup Wizard") % self.info.distro.name)
        self.main.title.set_font(size=20, bold=True, family="Arial")
        self.main.label = ui.Label(
            self.main, 40, 90, self.main.width - 80, 40,
            _("You need to reboot to complete the installation"))
        self.main.reboot_now = ui.RadioButton(self.main, 60, 150,
                                              self.main.width - 100, 20,
                                              _("Reboot now"))
        self.main.reboot_later = ui.RadioButton(
            self.main, 60, 180, self.main.width - 100, 20,
            _("I want to manually reboot later"))
        self.main.reboot_later.set_check(True)
Пример #6
0
def create_templates(arg, dirname, names):

    output_dir = dirname.replace(arg["repo_dir"], "")
    e = output_dir.split("/")
    output_dirname = e[-1]

    template_path = os.path.join(
            arg['output_dir'], "templates/") + output_dirname

    # Check if template dir exists, if not create

    if not os.path.exists(template_path):
        os.makedirs(template_path)

    print "Searching For Markdown Templates in %s/" % output_dirname

    files = find_markdown_templates(dirname)

    for f in files:

        file_path = os.path.join(arg["repo_dir"], 'md', output_dirname, f)
        """
        created = time.ctime(os.path.getctime(file_path))
        modified = time.ctime(os.path.getmtime(file_path))
        """
        created = os.path.getctime(file_path)
        modified = os.path.getmtime(file_path)


        page = Page(output_dirname, f, created, modified)
        page.write_html_file(arg["output_dir"], dirname)
        arg["page_list"].append(page)
        print "Generated Page At %s" % page.handler
Пример #7
0
    def crawl(self):
        """ The crawling is done in two phases after the URL pops from the queue
        1. Add URL with some info to notes (if source is more than 40k chars incl. html)
        2. Push children URLs to queue if they pass all crawling validations

        """
        link = self.queue.pop(0)
        #for link in nea:
        #one_level
        f = open("crawler.log", "a")
        f.write("+++++++++++" + link)
        # todo implement timeout!!!
        request = urllib2.Request(link)
        opener = urllib2.build_opener()

        # source = urllib.urlopen(link).read()
        parser = Parser()
        source = opener.open(request, timeout=None).read()
        parser.feed(source) # parses all the html source
        parser.close()
        opener.close()
        if source.__len__() > 1000:  # Change page character limit
            f.write(" [" + str(source.__len__()) + "]  * \n")
            self.notes[link]="working"
            webPage = Page(link, source)
            webPage.flash()
            #webPage.flashTkn()
        for url in parser.urls:
            if 'http' not in url:
               url = link + url
            if url not in self.history:
                self.queue.append(url)
                self.history.append(url)
                f.write(url + "\n")
        f.close()
Пример #8
0
    def get_content_page(self, new_view):
        page = Page()
        content = ContentPage()
        if new_view == '0':
            # Grab sovereign page concatenation
            current_view = content.print_sovereign_page()
        #if the response yields galaxy
        elif new_view == '1':
            # Grab sovereign page concatenation
            current_view = content.print_galaxy_page()

        #if the response yields excelsior
        elif new_view == '2':
            # Grab sovereign page concatenation
            current_view = content.print_excelsior_page()

        #if the response yields constitution
        elif new_view == '3':
            # Grab sovereign page concatenation
            current_view = content.print_constitution_page()

        #if the response yields intrepid
        elif new_view == '4':
            # Grab sovereign page concatenation
            current_view = content.print_intrepid_page()

        elif new_view == '5':
            # Grab main page concatenation
            current_view = page.print_main_page()
            # change data back to main page

        return current_view
Пример #9
0
	def data_initialize(self, file):
		
		data_list = []

		if file != None:
			assert file[-4:] == ".csv", "only accept csv file"
			with open(file, "r") as fr:
				content = fr.read()
				for seq, data in enumerate(content.split(",")):
					data_list.append(Page(sequence = seq, num = int(data)))

		else:
			print("Input the Number, if you finish your inputting, please input -1 ")
			counter = 0 
			while(1):
				num = input()
				if num == "-1":
					break
				else:
					data_list.append(Page(sequence = counter, num = int(num)))
				counter += 1


		for i in data_list:
			print(i.num, end = " ")
		return data_list
Пример #10
0
def main():

    runs = int(sys.argv[1])
    hop_list = []
    tests = 0
    for i in range(0, runs):

        try:

            test = Page("http://en.wikipedia.org/wiki/Special:Random")
            print("\n\033[1m Test No: \033[0m", str(i + 1))
            print("===========================")
            print("Crawling...")
            test.crawl()
            print("\n=======branches/hops======")
            print(test.branches)
            hop_list.append(test.branches)
            print("===========================\n")
            i += 1
            tests += 1
        except:
            print("\n")
            continue

    print("\033[1m Results: \033[0m")
    print("Number of tests completed without errors:", tests)
    print("Number of tests that failed:", runs - tests)

    if len(hop_list) == 0:
        print("Mean number of hops cannot be calculated/n")
    else:
        print("Mean number of hops:", str(sum(hop_list) / len(hop_list)), "\n")
Пример #11
0
def main():
    """"""
    mainUrl = "http://events.cnbloggercon.org/event/cnbloggercon2007/"
    content = Page(mainUrl).Content
    soup = BeautifulSoup(content)

    links = soup("a")
    peopleList = []
    for item in links:
        href = item["href"]
        if href.lower()[:8] == "/people/":
            peopleList.append("http://events.cnbloggercon.org" + href + "profile/")

    pageLink = []
    for link in peopleList:
        print link
        ct = Page(str(link.encode("utf-8")).lower()).Content
        pos = ct.find("个人网站")
        if pos == -1:
            continue
        ct = ct[pos:]
        so = BeautifulSoup(ct)
        fLink = so("a")[0]["href"]
        pageLink.append(fLink)

    f = file("abcde.txt", "w")
    for i in pageLink:
        f.write(i)
        f.write("\r\n")
Пример #12
0
    def on_init(self):
        Page.on_init(self)
        self.frontend.set_title(_("%s Uninstaller") % self.info.previous_distro_name)

        #header
        if self.info.uninstall_before_install:
            msg = _("Uninstallation required")
        else:
            msg = _("You are about to uninstall %s") % self.info.previous_distro_name
        self.insert_header(
            msg,
            "",
            "%s-header.bmp" % self.info.previous_distro_name)

        #navigation
        self.insert_navigation(_("Uninstall"), _("Cancel"), default=2)
        self.navigation.button2.on_click = self.on_cancel
        self.navigation.button1.on_click = self.on_uninstall

        #Main control container
        self.insert_main()
        if self.info.uninstall_before_install:
            msg = _("A previous installation was detected, it needs to be uninstalled before continuing")
        else:
            msg = _("Are you sure you want to uninstall?")

        self.uninstall_label = ui.Label(
            self.main,
            40, 40, self.main.width - 80, 30,
            msg)
Пример #13
0
 def get(self, url):
     p = Page.all().filter('url =', url).get()
     if p:
         h = Page.all().filter('url =', url).order('-created').fetch(15)
         self.render('history.html', history = h, user=self.user)
     else:
         self.redirect('/_edit'+url) 
Пример #14
0
class Game():   
    
    def __init__(self, gru_file=None):
        
        self.compiler = Compiler()
        if gru_file:
            self.stream = self.compiler.decompile(gru_file) 
        else:            
            self.stream = self.compiler.compile(None)   
        self.metadata = self.stream.metadata   
        self.flags = Flags(self.stream)
        self.wheel = Wheel(config)
        self.title = Title(config)
        self.inventory = Inventory(self.stream, self.flags, config)
        self.combiner = Combiner(self.stream, self.flags, self.inventory)
        self.page = Page(config, self.flags, self.combiner, self.inventory)
        self.state = State(self.stream, self.flags, self.inventory, self.wheel, self.combiner, self.title, self.page)               
        if self.metadata.has_key("start"):
            start = self.metadata["start"]
            self.state.update(start)
        else:
            self.state.update("start")

    def draw(self, tick):
        self.inventory.draw()
        self.wheel.draw()
        self.title.draw()
        self.page.draw(tick)

    
        
        
        
Пример #15
0
    def on_init(self):
        Page.on_init(self)
        self.set_background_color(255,255,255)
        self.frontend.set_title(_("%s CD Boot Helper") % self.info.cd_distro.name)
        self.insert_vertical_image("%s-vertical.bmp" % self.info.cd_distro.name)

        #sanity checks
        self.info.distro = self.info.cd_distro
        self.info.target_drive = None
        for drive in [self.info.system_drive] + self.info.drives:
            if drive.free_space_mb > self.info.distro.max_iso_size/1000000:
                self.info.target_drive = drive
                break
        if not self.info.target_drive:
            self.frontend.show_error_message(_("Not enough disk space to proceed"))

        #navigation
        self.insert_navigation(_("Accessibility"), _("Install"), _("Cancel"), default=2)
        self.navigation.button3.on_click = self.on_cancel
        self.navigation.button2.on_click = self.on_install
        self.navigation.button1.on_click = self.on_accessibility

        #main container
        self.insert_main()
        self.main.set_background_color(255,255,255)
        self.main.title = ui.Label(self.main, 40, 20, self.main.width - 80, 60, _("Install CD boot helper"))
        self.main.title.set_font(size=20, bold=True, family="Arial")
        txt = _("If your machine cannot boot off the CD, this program will install a new boot menu entry to help you boot from CD. In most cases this program is not needed, and it is sufficient to reboot with the CD-Rom in the tray.\n\nDo you want to proceed and install the CD booter?")
        self.main.label = ui.Label(self.main, 40, 90, self.main.width - 80, 80, txt)
 def __init__(self, testsetup, lookup):
     Page.__init__(self, testsetup)
     self.lookup = lookup
     if type(lookup) is int:
         self._root_locator = (self._base_locator[0], "%s[%i]" % (self._base_locator[1], lookup))
     elif type(lookup) is unicode:
         self._root_locator = (self._base_locator[0], "%s[h3[normalize-space(text()) = '%s']]" % (self._base_locator[1], lookup))
Пример #17
0
def gen_single_page(
    state,
    page_id,
    *,
    inner_only=False,
):
    single_page = page_id
    config = state.config

    remote = state.get_mongo()
    from bson.objectid import ObjectId
    page_id = ObjectId(single_page)
    remote_pages = remote.ndtest.pages.find({'_id': page_id})
    try:
        raw_page = next(remote_pages)
    except StopIteration:
        print("page not found")
        return {'data': 'page not found'}

    series_id = ObjectId(raw_page['series'])
    raw_series = remote.ndtest.series.find({'_id': series_id})
    raw_series = next(raw_series)
    series = Series(raw_series, config)

    root = Page(series=series)
    path = []
    root.add_page(path, raw_page)

    return {'data': root.render(inner_only=inner_only)}
Пример #18
0
def page(path):
    app.logger.info("Checking if page :" + path)
    page_file = Page(path)

    if page_file is not None:
        app.logger.info("parsing page")

        nav_items = []
        for page in os.listdir("pages"):
            if page.endswith('.md'):
                page_dict = {
                    'type': 'file',
                    'name': Page(page).title,
                    'path': '/' + page
                }
                nav_items.append(page_dict)
            elif os.path.isdir("pages/" + page):
                page_dict = {
                    'type': 'subfolder',
                    'name': page,
                    'path': '/' + page
                }
                nav_items.append(page_dict)

        return render_template("base.html",
                               content=page_file.markup,
                               nav=nav_items,
                               title=page_file.title,
                               header=site_info.header,
                               subtitle=site_info.subtitle)
    else:
        app.logger.info("page not found")
        return "404"
Пример #19
0
class Skastic:
    def __init__(self, filename, args):
        self.function_pages = {}
        self.main_page = None
        self.args = args
        self.analyze_only = bool(args.draw)
        self.load(filename, args)
        if not self.analyze_only:
            self.exec()

    def load(self, filename, args):
        self.main_page = Page(filename, self.analyze_only)
        if self.analyze_only:
            self.main_page.draw(args.draw)
        else:
            self.load_includes(args)

    def load_includes(self, args):
        for filename in args.include:
            self.function_pages[filename.split('.')[0]] = Page(filename, False)

    def exec(self):
        function_asts = [x.python_ast for x in self.function_pages.values()]
        module_ast = make_module(function_asts, self.main_page.python_ast)

        if self.args.dump_ast:
            print(ast.dump(module_ast))

        exec(compile(module_ast, filename="<ast>", mode="exec"))
Пример #20
0
    def __init__(self):
        self.duration = 0.0       # Druation of current song.
        self.posBarPressed = False  # Set song position from backend in slider.
        self.posBarDownVal = -1

        Page.__init__(self);
        self.cfg.get_backend().connect(Signals.VOLUME_CHANGED,self.volume_changed)
        self.cfg.get_backend().connect(Signals.RANDOM_CHANGED,self.random_changed)
        self.cfg.get_backend().connect(Signals.REPEAT_CHANGED,self.repeat_changed)
        self.cfg.get_backend().connect(Signals.SONG_CHANGED,self.song_changed)
        self.cfg.get_backend().connect(Signals.POS_CHANGED,self.pos_changed)
        self.cfg.get_backend().connect(Signals.STATE_CHANGED,self.state_changed)

        self.btn_prev = ImageButton(gtk.STOCK_MEDIA_PREVIOUS)
        self.btnbox.add(self.btn_prev)
        self.btn_stop = ImageButton(gtk.STOCK_MEDIA_STOP)
        self.btnbox.add(self.btn_stop)
        self.btn_pause = ImageButton(gtk.STOCK_MEDIA_PAUSE)
        self.btnbox.add(self.btn_pause)
        self.btn_play = ImageButton(gtk.STOCK_MEDIA_PLAY)
        self.btnbox.add(self.btn_play)
        self.btn_next = ImageButton(gtk.STOCK_MEDIA_NEXT)
        self.btnbox.add(self.btn_next)


        self.btn_play.connect("clicked", self.btn_clicked)
        self.btn_stop.connect("clicked", self.btn_clicked)
        self.btn_prev.connect("clicked", self.btn_clicked)
        self.btn_next.connect("clicked", self.btn_clicked)
        self.btn_pause.connect("clicked", self.btn_clicked)

        self.volevent = False
Пример #21
0
 def get(self, url):
     p = Page.all().filter('url =', url).get()
     if p:
         h = Page.all().filter('url =', url).order('-created').fetch(15)
         self.render('history.html', history=h, user=self.user)
     else:
         self.redirect('/_edit' + url)
Пример #22
0
    def test_get_content(self):

        page = Page(
            "https://www.epocacosmeticos.com.br/mascara-facial-ricca-bubble-help/p",
            "https://www.epocacosmeticos.com.br/", set())

        product_name = "Máscara Facial Ricca - Bubble Help! - 1 Un"
        title = "Máscara Facial Ricca - Bubble Help! - Época Cosméticos"

        self.assertEqual(product_name, page.get_content()['product_name'])
        self.assertEqual(title, page.get_content()['title'])

        page = Page(
            "https://www.epocacosmeticos.com.br/artliner-lancome-delineador/p",
            "https://www.epocacosmeticos.com.br/", set())

        product_name = "Artliner Lancôme - Delineador"
        title = "Artliner Lancôme - Delineador - Época Cosméticos"

        self.assertEqual(product_name, page.get_content()['product_name'])
        self.assertEqual(title, page.get_content()['title'])

        page = Page("https://google.com", "https://google.com", set())

        self.assertRaises(ValueError, page.get_content)
Пример #23
0
def navigate(origin_title, destination_title):
    """Given title of start page and title of end page, navigates to
    end page and returns a path object"""
    startP = Page(origin_title)
    goalP = Page(destination_title)
    path = nav_help(startP, goalP)
    return Path(path)
Пример #24
0
    def create(cls, key, label, user, description=None, locale=None):
        from group import Group
        from membership import Membership
        from page import Page

        instance = Instance(unicode(key).lower(), label, user)
        instance.description = description
        instance.default_group = Group.by_code(Group.INSTANCE_DEFAULT)
        if locale is not None:
            instance.locale = locale
        meta.Session.add(instance)
        supervisor_group = Group.by_code(Group.CODE_SUPERVISOR)
        membership = Membership(user, instance, supervisor_group,
                                approved=True)
        meta.Session.add(membership)
        if config.get_bool('adhocracy.create_initial_instance_page'):
            Page.create(instance, label, u"", user)

        # Autojoin the user in instances
        config_autojoin = config.get('adhocracy.instances.autojoin')
        if (config_autojoin and
                (config_autojoin == 'ALL' or
                 key in (k.strip() for k in config_autojoin.split(',')))):
            users = adhocracy.model.User.all()
            for u in users:
                autojoin_membership = Membership(u, instance,
                                                 instance.default_group)
                meta.Session.add(autojoin_membership)

        meta.Session.flush()
        return instance
Пример #25
0
def jquery_ready():
    jQuery(document).ajaxError(_on_error)

    def _on_hide(e):
        mount_html(
            jQuery(this).find("div.dialog-data"),
            "<div class='alert alert-info' role='alert'>Sending data - please wait</div>",
            False,
            False,
        )

    jQuery("div.dialog-form").on("hide.bs.modal", _on_hide)

    def _local_fun():
        console.log("collapsed")

    jQuery(".navbar-ex1-collapse").on("hidden.bs.collapse", _local_fun)

    if window.APPLICATION_TEMPLATE == "traditional":
        window.ACTIVE_PAGE = Page(0, jQuery("#body_body"))
        # __new__(Vue({'el': '#body_body'}))
    else:
        if window.APPLICATION_TEMPLATE == "modern":
            txt = jQuery(".page").html()
            txt2 = jQuery.trim(txt)
            if txt2:
                txt = jQuery.trim(jQuery(".page")[0].outerHTML)
                jQuery(".page").remove()
                menu = get_menu()
                menu.new_page(jQuery("title").text(), txt, window.location.href)
        else:
            window.ACTIVE_PAGE = Page(0, jQuery("#body_body"))
            if window.APPLICATION_TEMPLATE == "to_print":
                __new__(Vue({"el": "#body_body"}))
Пример #26
0
    def on_init(self):
        Page.on_init(self)

        #header
        if self.info.distro:
            distro_name = self.info.distro.name
        elif self.info.previous_distro_name:
            distro_name = self.info.previous_distro_name
        self.insert_header(
            _("Installing %(distro)s-%(version)s") %
            dict(distro=distro_name, version=self.info.version),
            _("Please wait"), "%s-header.bmp" % distro_name)

        #navigation
        self.insert_navigation(_("Cancel"))
        self.navigation.button1.on_click = self.on_cancel

        #main container
        self.insert_main()
        self.main.task_label = ui.Label(self.main, 20, 20, self.width - 40, 20)
        self.main.progressbar = ui.ProgressBar(self.main, 20, 50,
                                               self.width - 40, 20)
        self.main.subtask_label = ui.Label(self.main, 20, 80, self.width - 40,
                                           20)
        self.main.subprogressbar = ui.ProgressBar(self.main, 20, 110,
                                                  self.width - 40, 20)
        self.main.localiso_button = ui.Button(
            self.main, 20, 150, 200, 20,
            _("Do not download, use a local file"))

        self.main.localiso_button.hide()
        self.main.subtask_label.hide()
        self.main.subprogressbar.hide()
Пример #27
0
    def __init__(self):
        self.model = gtk.ListStore(object,str,str,str)
        Page.__init__(self)

        self.cfg = PythmConfig()

        self.btn_connect = ImageButton(gtk.STOCK_CONNECT)
        self.btnbox.add(self.btn_connect)
        self.btn_connect.connect("clicked",self.btn_connect_clicked)

        self.btn_start = ImageButton(gtk.STOCK_EXECUTE)
        self.btnbox.add(self.btn_start)
        self.btn_start.connect("clicked",self.btn_start_clicked)

        self.btn_stop = ImageButton(gtk.STOCK_STOP)
        self.btnbox.add(self.btn_stop)
        self.btn_stop.connect("clicked",self.btn_stop_clicked)

        self.btn_refresh = ImageButton(gtk.STOCK_REFRESH)
        self.btnbox.add(self.btn_refresh)
        self.btn_refresh.connect("clicked",self.btn_refresh_clicked)

        self.mgr = BackendManager()

        if self.cfg.get_backend() != None:
            self.start_backend(self.cfg.get_backend())

        self.refresh()
        self.row_selected(None)
        self.set_sensitive(True)
Пример #28
0
    def import_page(self,
                    page: Page,
                    page_key,
                    table,
                    table_folder,
                    num_base_pages=None):
        inner_idx, pr_idx = page_key

        try:
            binary_file = open(
                self.database_folder + "/" + table_folder + "/" +
                "pagerange_" + str(pr_idx), 'rb')
        except FileNotFoundError:
            return False

        if num_base_pages == None:
            num_base_pages = table.page_ranges[pr_idx].base_page_count

        offset = self.get_page_offset_in_pr(page_key, num_base_pages)

        # Read num_records
        binary_file.seek(offset)
        num_records = ifb(binary_file.read(8))

        # Read data
        data = bytearray(binary_file.read(PAGE_SIZE))

        page.num_records = num_records
        page.load(data, num_records)
        # todo: is_dirty, pinning

        binary_file.close()
Пример #29
0
    def _load_from_cache(self, urls):
        """
        left_urls = []
        pages = []
        for url in urls:
            obj = self._contains(url)
            if obj:
                page = Page(url)
                page.load_from_json(obj)
                pages.append(page)
            else:
                left_urls.append(url)

        return pages, left_urls
        """
        urls = set(urls)
        loaded_urls = set()
        pages = []
        for cache in self.caches:
            for url in cache.keys():
                if (url in urls) and (url not in loaded_urls):
                    page = Page(url)
                    loaded_urls.add(url)
                    page.load_from_json(cache.get(url))
                    if page.body:  # skip the page whose text extraction was failed
                        pages.append(page)
        left_urls = [url for url in urls if url not in loaded_urls]
        return pages, left_urls
Пример #30
0
    def request_stocks(self):
        if not self.__remove_old_data():
            return

        self.__request_count = 2
        self.__get_stocks(constant.SH_URL, Page())
        self.__get_stocks(constant.SZ_URL, Page())
Пример #31
0
    def process(self, queue, pbar, outputDir, killEvent, failedPages):
        """
        The process function of each thread.

        Parameters:
            queue (queue of int): The download queue.
            pbar (tqdm): The progress bar.
            outputDir (str): The output directory.
            killEvent (threading.Event): The event object that can terminate the process.
            failedPages (queue of Pages): The collection of Pages that failed to be downloaded.
        """
        while not queue.empty() and not killEvent.is_set():
            pageNum = queue.get()
            pageNumStr = f'{pageNum:04}'

            if pageNum == 404:
                # Skip the 404 error page, haha
                logger.debug('Skipping the intentional 404 Error page...')
            elif pageNumStr in self.pages and self.pages[
                    pageNumStr].isDownloaded:
                logger.debug('Skipping page %d...', pageNum)
            else:
                page = Page.fetch(pageNum)
                self.pages[pageNumStr] = page
                err = page.downloadImage(outputDir)
                if err is not None:
                    failedPages.put(Page.getPageUrl(pageNum))

            pbar.update()
            queue.task_done()
Пример #32
0
    def on_init(self):
        Page.on_init(self)

        #header
        if self.info.distro:
            distro_name = self.info.distro.name
        elif self.info.previous_distro_name:
            distro_name = self.info.previous_distro_name
        self.insert_header(
            _("Installing %(distro)s-%(version)s") % dict(distro=distro_name, version=self.info.version),
            _("Please wait"),
            "%s-header.bmp" % distro_name)

        #navigation
        self.insert_navigation(_("Cancel"))
        self.navigation.button1.on_click = self.on_cancel

        #main container
        self.insert_main()
        self.main.task_label = ui.Label(self.main, 20, 20, self.width - 40, 20)
        self.main.progressbar = ui.ProgressBar(self.main, 20, 50, self.width - 40, 20)
        self.main.subtask_label = ui.Label(self.main, 20, 80, self.width - 40, 20)
        self.main.subprogressbar = ui.ProgressBar(self.main, 20, 110, self.width - 40, 20)
        self.main.localiso_button = ui.Button(self.main, 20, 150, 200, 20, _("Do not download, use a local file"))

        self.main.localiso_button.hide()
        self.main.subtask_label.hide()
        self.main.subprogressbar.hide()
Пример #33
0
def _get_page_phantomjs(url, proxy=''):
    '''
    Get the page using phantomjs.

    Note: only http schema is currently supported.
    '''
    LOG.info('Trying to get page %s via PhantomJS, proxy %s.' %
             (url, proxy or 'N/A'))
    start_ts = time()
    import os
    jspath = os.path.join(os.path.dirname(__file__), 'get.js')
    content = check_output(['phantomjs', '--proxy=' + proxy, jspath, url])
    elapsed = time() - start_ts

    if content:
        LOG.info('Successfully got page %s in %gs.' % (url, elapsed))
        if proxy:
            report_proxy_status(proxy, True)
    else:
        LOG.warning(
            'Failed to get page %s in %gs, yet we can\'t know what happened.' %
            (url, elapsed))
        if proxy:
            report_proxy_status(proxy, False)
        return None

    enc = guess_content_encoding(content)
    p = Page(url, content, enc)
    p.elapsed = elapsed
    p.proxy = proxy
    p.req_url = url
    return p
Пример #34
0
    def __init__(self, testsetup):
        Page.__init__(self, testsetup)

        handles = self.selenium.window_handles
        for i in handles:
            self.selenium.switch_to_window(i)
            if self.selenium.title == "BrowserID": break
    def test(self):
        mark_value = 5
        marks = [mark_value]

        photos = self.upload_photo(USERNAME_SECOND)
        self.photos = photos
        name = self.get_name(USERNAME_FIRST)
        self.set_marks(None, photos, marks)

        self.login(USERNAME_SECOND)

        page = Page(self.driver)
        events_modal = page.top_menu.events_modal
        events_modal.open()

        photo = events_modal.get_photo()

        marks_modal = events_modal.marks_modal
        marks_modal.open(True)

        marks_modal.remove(name)

        page.open()
        events_modal.open()

        self.assertNotEqual(photo, events_modal.get_photo())
Пример #36
0
    def add_base_value(self, page_range, value):
        """
        Add a record to a the base pages

        :param value: int                   # the value to be added to the base pages

        :return: int                        # the page number of the page being written too
        :return: int                        # the offset of the write operation
        """
        page = self.pages[self.last_base_page]
        offset = 0

        # if page has capacity
        if page.has_capacity():

            # write to it
            offset = page.write(value)

        # otherwise create a new one and add it
        else:
            page = Page(page_range, page_range.get_page_number())
            offset = page.write(value)
            self.last_base_page += 1
            self.pages.insert(self.last_base_page, page)
            self.last_page += 1

        return self.last_base_page, offset
Пример #37
0
def generatePages(title, options, kind):
    pages = []

    # Group options into bunches of 5
    num_per_page = 5
    grouped_options = [
        options[n:n + num_per_page]
        for n in range(0, len(options), num_per_page)
    ]

    for page_index, items in enumerate(grouped_options):
        msg = ''

        for opt_index, option in enumerate(items):
            msg += f'`{opt_index + 1}` {option.name} '

            if kind == 'Home College' or kind == 'Target College':
                msg += f'**{option.code}**'

            msg += '\n\n'

        page = Page(title,
                    msg,
                    count=opt_index + 1,
                    page_num=page_index + 1,
                    total_pages=len(grouped_options))

        if pages:
            page.prev = pages[-1]
            pages[-1].next = page

        pages.append(page)

    return pages
Пример #38
0
    def create(cls, key, label, user, description=None, locale=None):
        from group import Group
        from membership import Membership
        from page import Page

        instance = Instance(unicode(key).lower(), label, user)
        instance.description = description
        instance.default_group = Group.by_code(Group.INSTANCE_DEFAULT)
        if locale is not None:
            instance.locale = locale
        meta.Session.add(instance)
        supervisor_group = Group.by_code(Group.CODE_SUPERVISOR)
        membership = Membership(user, instance, supervisor_group,
                                approved=True)
        meta.Session.add(membership)
        if config.get_bool('adhocracy.create_initial_instance_page'):
            Page.create(instance, label, u"", user)

        # Autojoin the user in instances
        config_autojoin = config.get('adhocracy.instances.autojoin')
        if (config_autojoin and
                (config_autojoin == 'ALL' or
                 key in (k.strip() for k in config_autojoin.split(',')))):
            users = adhocracy.model.User.all()
            for u in users:
                autojoin_membership = Membership(u, instance,
                                                 instance.default_group)
                meta.Session.add(autojoin_membership)

        meta.Session.flush()
        return instance
Пример #39
0
    def on_init(self):
        Page.on_init(self)
        self.frontend.set_title(
            _("%s Uninstaller") % self.info.previous_distro_name)

        #header
        if self.info.uninstall_before_install:
            msg = _("Uninstallation required")
        else:
            msg = _("You are about to uninstall %s"
                    ) % self.info.previous_distro_name
        self.insert_header(msg, "",
                           "%s-header.bmp" % self.info.previous_distro_name)

        #navigation
        self.insert_navigation(_("Uninstall"), _("Cancel"), default=2)
        self.navigation.button2.on_click = self.on_cancel
        self.navigation.button1.on_click = self.on_uninstall

        #Main control container
        self.insert_main()
        if self.info.uninstall_before_install:
            msg = _(
                "A previous installation was detected, it needs to be uninstalled before continuing"
            )
        else:
            msg = _("Are you sure you want to uninstall?")

        self.uninstall_label = ui.Label(self.main, 40, 40,
                                        self.main.width - 80, 30, msg)
Пример #40
0
 def process_pages(self):
     skipped = []
     pbar = ProgressBar(widgets=['Processing pages: ', SimpleProgress()], maxval=len(self.urls)).start()
     i = 0
     
     for (num, url) in self.urls:
         pbar.update(int(num))
         if (num and url):
             html = helpers.get_html(num, url)
             if html is not None:
                 self.urls_with_nums[url] = num
                 soup = BeautifulSoup(html.encode('utf-8', 'ignore'), 'lxml')
                 page = Page(title=soup.title.string, num=num, html=soup.prettify(), url=url, text=soup.body.get_text())
                 page.index = i
                 self.indices_with_pages[i] = page
                 if page.ID not in self.pages_with_ids.keys():
                     self.pages_with_ids[page.ID] = page
                 else:
                     raise RuntimeError('COLLISION: %s collides with %s with hash %s.' % (page.num, self.pages_with_ids[page.ID].num, page.ID))
                 for link in soup.find_all('a'):
                     if link.get('href') and 'mailto:' != link.get('href').strip()[0:7]:
                         page.a.append(link)
                 self.pages.append(page)
                 i += 1
             else:
                 skipped.append(num)
         else:
             skipped.append(num)
     pbar.finish()
     print "Skipped page(s) %s because of an error." % (', '.join(skipped))
Пример #41
0
class Main(object):
	def __init__(self, delegate, input):
		self.delegate = delegate
		self.input = input
		self.work = []
		self.page = Page()

	def run(self):
		try:
			while not self.input.closed:
				line = self.input.readline()
				if line is None or line.strip() == EOF:
					break
				line = line.strip()
				if line:
					self.process_line(line)
		except KeyboardInterrupt:
			pass
		except StandardError:
			print "UI process input loop received exception:"
			import traceback
			traceback.print_exc()
		finally:
			self.delegate.exit()
	
	def process_line(self, line):
		node = Data.decode(line)
		self.page.process_node(node)
		self.delegate.update(self.page)
Пример #42
0
    def on_init(self):
        Page.on_init(self)
        self.frontend.set_title(_("%s Uninstaller") % self.info.previous_distro_name)

        #header
        msg = _("You Sure ?")
        self.insert_header(
            msg,
            "",
            "StartOS-header.bmp")#% self.info.previous_distro_name[0:7])

        #navigation
        self.insert_navigation(_("Uninstall"), _("Cancel"), default=2)
        self.navigation.button2.on_click = self.on_cancel
        self.navigation.button1.on_click = self.on_continue

        msg = _("Uninstaller will remove below directroys, if them contain your\
        usefull things ,please click <cancel> and backup them")

        msg = msg + "\r\n\r\n" + self.info.previous_target_dir
        if self.info.pre_install_path and os.path.isdir(self.info.pre_install_path)\
            and self.info.previous_target_dir.lower() != self.info.pre_install_path.lower():
                msg = msg + "\r\n" + self.info.pre_install_path # 这里确定要不要增加X:\startos-loop
        if self.info.pre_install_path2 and os.path.isdir(self.info.pre_install_path2)\
            and self.info.previous_target_dir.lower() != self.info.pre_install_path2.lower():
                msg = msg + "\r\n" + self.info.pre_install_path2 # 这里确定要不要增加X:\startos-livecd
        
        #Main control container
        self.insert_main()
        
        self.uninstall_label = ui.Label(
            self.main,
            40, 40, self.main.width - 80, 80,# x,y, width, height
            "")
        self.uninstall_label.set_text(msg)
Пример #43
0
    def get(self):
        '''
        if form has been entered try the urgent_info form field, if it has been entered, change the value to a string
        if it does not exist, create a different string
         create a concatenated string to pass into a new Form instance print method
         otherwise call the default print method of the Form class for default results
        '''
        if self.request.GET:

            try:
                urgent_updates = self.request.GET["urgent_info"]
                if urgent_updates:
                    urgent_updates = "- You will also receive updates."
            except StandardError:
                urgent_updates = "- You <em>will not</em> receive any updates."

            form_fields = ("<div id='confirm'><p>" +self.request.GET["first_name"] + " " + self.request.GET["last_name"] + "<br>"
                + self.request.GET["address"] + "<br>"
                + self.request.GET["city"] + ", " + self.request.GET["state"] + " " + self.request.GET["zipcode"] + "</p><p>"
                + "<a href='mailto:" + self.request.GET["email"] + "'>" + self.request.GET["email"] + "</a><br>"
                + urgent_updates + "</p></div>")
            page = Page(self)
            self.response.write(page.print_form_results(form_fields))

        else:
            page = Page(self)
            base = page.print_form_results()
            self.response.write(base)
Пример #44
0
    def fetch(self, url, inlinks):
        if self.allowed(url):
            try:
                f = self.__http.request('Get', url, timeout=2)
            except socket.timeout:
                return Page()
            except:
                return Page()
            else:
                try:
                    header = f.headers
                except:
                    header = None

                try:
                    ctype = header['Content-Type']
                except:
                    ctype = None

                try:
                    lang = header['Content-Language']
                except:
                    lang = None
                if ctype and lang:
                    if 'text/html' in ctype and 'en' in lang:
                        try:
                            html = f.data
                        except:
                            print "Cannot read"
                            return Page()
                        else:
                            rhead = {}
                            for item in header:
                                rhead[item] = header[item]
                            return Page(url, rhead, html, inlinks, True)
                    else:
                        print "Not english or html"
                        return Page()
                elif ctype:
                    if 'text/html' in ctype:
                        try:
                            html = f.data
                        except:
                            print "Cannot read"
                            return Page()
                        else:
                            rhead = {}
                            for item in header:
                                rhead[item] = header[item]
                            return Page(url, rhead, html, inlinks, True)
                    else:
                        print "Not html"
                        return Page()
                else:
                    print "No header"
                    return Page()
        else:
            print "Robots not allowed"
            return Page()
Пример #45
0
    def get(self):


        p = Page()

        bodark = Character()
        bodark.name="Bodark Bjorn"
        bodark.str = 15
        bodark.dex = 7
        bodark.chr = 8
        bodark.int = 3
        bodark.con = 10


        nock = Character()
        nock.name= "Nock Fletching"
        nock.str = 7
        nock.dex = 15
        nock.chr = 12
        nock.int = 6
        nock.con = 3

        magnar = Character()
        magnar.name="Magnar Magnil"
        magnar.str = 16
        magnar.dex = 17
        magnar.chr = 10
        magnar.int = 2
        magnar.con = 8


        lina = Character()
        lina.name= "Lina Medina"
        lina.str = 5
        lina.dex = 5
        lina.chr = 3
        lina.int = 15
        lina.con = 15

        players = {
            "bodark":bodark,
            "nock":nock,
            "magnar":magnar,
            "lina":lina
        }

        p.character = players["bodark"]
        render =  p.print_info()

        if self.request.GET:
            if self.request.GET.has_key('char'):

                character = self.request.GET['char']

                if players.has_key(character):
                    p.character= players[character]
                    render = p.print_info()

        self.response.write(render)
Пример #46
0
 def googleSearch(self, term, numPages = 1):
     term = self.convertToUrlReady(term, 'google')
     links = []
     for start in range(0,numPages):
         url = "http://www.google.com/search?q=" + term +"&start=" + str(start*10)
         page = Page(url)
         links.append(page.gSoupToLinks())
     return links
Пример #47
0
    def load_url(self, url):
        # pass
        # var page = new Page(index, url, chrome, options.fetchContent);
        index = 0
        self.page = Page(index, url, self.ws, fetch_content=True)

        # print "## HERE", url
        self.navigate_to_page(url)
Пример #48
0
	def __init__(self, path):
		Page.__init__(self)

		module,name=path.rsplit('.', 1)

		app=__import__(module, fromlist=[ name ])

		self.app=getattr(app, name)()
Пример #49
0
 def amazonSearch(self, term, numPages = 1):
     term = self.convertToUrlReady(term, 'amazon')
     links = []
     for start in range(0, numPages):
         url = 'http://www.amazon.com/s/ref=nb_sb_noss?url=search-alias%3Daps&field-keywords=' + term +'&page=' + str(start) + '&rh=i%3Aaps%2Ck%3A' + term
         page = Page(url, domain='amazon')
         links.append(page.aSoupToLinks())
     return links
Пример #50
0
def decode_page(BYTES_page) -> Page:
    page = Page(True)  # type: Page
    # data, num_records=None, is_dirty=None, force=False):
    # is_loaded = True
    num_records = int_from_bytes(BYTES_page[0:8])
    data = bytearray(BYTES_page[8:])
    page.load(data, num_records)
    return page
Пример #51
0
 def __init__(self):
     Page.__init__(self)
     self._current_line = 0
     self._tail = False
     self._content = ""
     with open("/tmp/clazy.json", 'r') as f:
         self._parse_lines(f.read())
         f.close()
Пример #52
0
 def test_sphere(self, setup):
     sphere = data.get_data()
     page = Page(self.driver)
     page.set_sphere(sphere[4][1])
     page_results = page.get_page_results()
     correct_results = results.get_sphere_results(sphere[4][0])
     assert page_results[0] == correct_results[0], values[0]
     assert page_results[1] == correct_results[1], values[0]
Пример #53
0
 def test_cone(self, setup):
     cone = data.get_data()
     page = Page(self.driver)
     page.set_cone(cone[3][1], cone[3][3], cone[3][5])
     page_results = page.get_page_results()
     correct_results = results.get_cone_results(cone[3][0], cone[3][2], cone[3][4])
     assert page_results[0] == correct_results[0], values[0]
     assert page_results[1] == correct_results[1], values[0]
Пример #54
0
    def on_init(self):
        Page.on_init(self)

        if self.info.needfix != 0:
            wintitle = _("%s Uninstaller/Fixer") % self.info.previous_distro_name
            if self.info.uninstall_before_install:
                msg = _("Uninstallation/fixed required")
            else:
                msg = _("You are about to uninstall/Fix %s") % self.info.previous_distro_name
        else:
            wintitle = _("%s Uninstaller") % self.info.previous_distro_name
        #header
            if self.info.uninstall_before_install:
                 msg = _("Uninstallation required")
            else:
                 msg = _("You are about to uninstall %s") % self.info.previous_distro_name

        self.frontend.set_title(wintitle)
        self.insert_header(
            msg,
            "",
            "StartOS-header.bmp")#% self.info.previous_distro_name[0:7])
            
        #navigation
        if self.info.needfix != 0:# and self.info.registry:
            self.insert_navigation(_("Fix"), _("Uninstall"), _("Cancel"), default=3)
            self.navigation.button3.on_click = self.on_cancel
            self.navigation.button2.on_click = self.on_uninstall
            self.navigation.button1.on_click = self.run_fixboot
        else:
            self.insert_navigation(_("Uninstall"), _("Cancel"), default=2)
            self.navigation.button2.on_click = self.on_cancel
            self.navigation.button1.on_click = self.on_uninstall
        
        #Main control container
        self.insert_main()
        if self.info.uninstall_before_install:
            if self.info.inst == "uninstall.exe":
                msg = _("Do you want to uninstall %s?") % self.info.previous_distro_name
            else:
                msg = _("A previous installation was detected, it needs to be uninstalled before continuing")
        else:
            msg = _("Are you sure you want to uninstall?")

        FIXLABEL_HEIGHT = 60
        self.uninstall_label = ui.Label(
            self.main,
            40, 40, self.main.width - 80, 30 + FIXLABEL_HEIGHT,# x,y, width, height
            "")
        if self.info.needfix:
            msg = msg + _("\r\n\r\nNote: Installer found that your previous installation was broken, but the loop device is still exists, and advises you to <fix> first !")
        self.uninstall_label.set_text(msg)

        self.backup_iso = ui.CheckButton(
            self.main, 80, 70, self.main.width - 120, 12,
            _("Backup the downloaded files (ISO)"))
        self.backup_iso.set_check(False)
        self.backup_iso.hide()
Пример #55
0
	def run_job(self):
		self.status = {}
		self.status["scope"] = "running crawl job"
		if self.query is None:
			self.status["msg"] = "Unable to start crawl: no query has been set."
			self.status["code"] = 600.1
			self.status["status"] = False
			return False 
		else:
			query = Query(self.query)
			
		seeds = self.collect_sources()
		if self.db.sources.count() == 0:
			self.status["msg"] = "Unable to start crawl: no seeds have been set."
			self.status["code"] = 600.1
			self.status["status"] = False
			return False
		else:
			self.send_seeds_to_queue()
		
		start = datetime.now()
		if self.db.queue.count == 0:
			self.status["msg"] = "Error while sending urls into queue: queue is empty"
			self.status["code"] = 600.1
			self.status["status"] = False
			return False
			
		else:
			self.status["msg"] = "running crawl on %i sources with query '%s'" %(len(self.db.sources.distinct("url")), self.query)
				
			while self.db.queue.count > 0:	
				for url in self.db.queue.distinct("url"):
					if url != "":
						page = Page(url)
						if page.check() and page.request() and page.control():
							article = page.extract("article")
							if article.status is True:
								if article.is_relevant(query):			
									self.db.results.insert(article.repr())
									if article.outlinks is not None and len(article.outlinks) > 0:
										self.db.queue.insert(article.outlinks)
							else:	
								self.db.logs.insert(article.logs)
						else:
							self.db.logs.insert(page.status)	
					self.db.queue.remove({"url": url})
					
					if self.db.queue.count() == 0:		
						break
				if self.db.queue.count() == 0:		
						break
			end = datetime.now()
			elapsed = end - start
			delta = end-start

			self.status["msg"] = "%s. Crawl done sucessfully in %s s" %(self.status["msg"],str(elapsed))
			self.status["status"] = True
			return True
Пример #56
0
 def __init__(self):
     Page.__init__(self)
     self._current_line = 0
     self._code = [
         "NumberWrapper* wrap3 = nullptr;",
         "",
         "//Check if the URI is complete or short",
         "const bool hasAtSign = uri.hasHostname();",
         "",
         "// Append the account hostname, this should always reach the same destination",
         "// as long as the server implementation isn't buggy.",
         "const auto extendedUri = (hasAtSign || !account) ? uri : URI(QStringLiteral(\"%1@%2\")",
         "    .arg(uri)",
         "    .arg(account->hostname()));",
         "",
         "//Try to see if there is a better candidate with a suffix (LAN only)",
         "if ( !hasAtSign && account ) {",
         "",
         "    // Attempt an early candidate using the extended URI. The later",
         "    // `confirmedCandidate2` will attempt a similar, but less likely case.",
         "    if ((wrap2 = d_ptr->m_hDirectory.value(extendedUri))) {",
         "        if (auto cm = d_ptr->fillDetails(wrap2, extendedUri, account, contact, type))",
         "        return cm;",
         "    }",
         "}",
         "",
         "//Check",
         "ContactMethod* confirmedCandidate = d_ptr->fillDetails(wrap,uri,account,contact,type);",
         "",
         "//URIs can be represented in multiple way, check if a more verbose version",
         "//already exist",
         "ContactMethod* confirmedCandidate2 = nullptr;",
         "",
         "//Try to use a ContactMethod with a contact when possible, work only after the",
         "//contact are loaded",
         "if (confirmedCandidate && confirmedCandidate->contact())",
         "    confirmedCandidate2 = d_ptr->fillDetails(wrap2,uri,account,contact,type);",
         "",
         "ContactMethod* confirmedCandidate3 = nullptr;",
         "",
         "//Else, try to see if the hostname correspond to the account and flush it",
         "//This have to be done after the parent if as the above give \"better\"",
         "//results. It cannot be merged with wrap2 as this check only work if the",
         "//candidate has an account.",
         "if (hasAtSign && account && uri.hostname() == account->hostname()) {",
         "    if ((wrap3 = d_ptr->m_hDirectory.value(uri.userinfo()))) {",
         "        foreach(ContactMethod* number, wrap3->numbers) {",
         "        if (number->account() == account) {",
         "            if (contact && ((!number->contact()) || (contact->uid() == number->contact()->uid())))",
         "                number->setPerson(contact); //TODO Check all cases from fillDetails()",
         "            //TODO add alternate URI",
         "            confirmedCandidate3 = number;",
         "            break;",
         "        }",
         "        }",
         "    }",
         "}",
     ]
Пример #57
0
    def get(self):
		page = Page()

		# redPanda class that gets all the attributes from the AbstractAnimals class
		red_panda = Sound()
		red_panda.name = "Red Panda"
		red_panda.phylum = "Chordata"
		red_panda.cclass = "Mammalia"
		red_panda.order = "Carnivora"
		red_panda.family = "Ailuridae"
		red_panda.genus = "Ailurus"
		red_panda.url = "http://animaldiversity.ummz.umich.edu/collections/contributors/Kjeldsen_Per/Red_Panda7208/large.jpg"
		red_panda.alifespan = "The maximum lifespan of the red panda is 14 years, but the average is eight to ten."
		red_panda.habitat = "Red pandas live in temperate climates in deciduous and coniferous forests. There is usually an understory of bamboo and hollow trees. The average temperature is 10 to 25 degrees Celsius, and the average annual rainfall is 350 centimeters."
		red_panda.geolocation = "Red pandas are found throughout the Himalayan mountains between 2,200 and 4,800 meters in elevation in northern Burma, Nepal, Sikkim region of India, and the districts of western Sichuan and Yunnan in China. Their geographic range is bounded in the north by the Namlung Valley in the Mugo District and the Lake Rara region of northern Nepal, in the south by the Liakiang Range of western Yunnan, and the northern and eastern boundary is the upper Min Valley of western Sichuan."
		red_panda.sound = "Wieuw Wieuw"
		red_panda.update()

		# great_auk class that gets all the attributes from the AbstractAnimals class
		great_auk = Sound();
		great_auk.name = "Great Auk"
		great_auk.phylum = "Chordata"
		great_auk.cclass = "Aves"
		great_auk.order = "Charadriiformes"
		great_auk.family = "Alcidae"
		great_auk.genus = "Pinguinus"
		great_auk.url = "http://animaldiversity.ummz.umich.edu/collections/contributors/grzimek_birds/Alcidae/pinguinus_impennis/medium.jpg"
		great_auk.alifespan = "As in other large seabirds, annual survival is thought to have been relatively high. Estimated lifespan was from 20 to 25 years."
		great_auk.habitat = "In recorded history, great auks were found breeding only on isolated, rocky islands in the North Atlantic. They foraged in the open ocean when not breeding."
		great_auk.geolocation = "The last great auk was seen in 1852 off the Newfoundland Banks. They were extinct in the western Atlantic before 1800 and persisted a bit longer in some parts of the eastern Atlantic and off the coast of Iceland. They lived and bred on scattered, offshore islands in the northern Atlantic in recorded history, although it is possible they once occurred on continental coastlines but were extirpated from those areas earlier. They were found from Canada, Greenland, and Iceland to the British Isles and Scandinavia. Colonies in the western Atlantic may have been less numerous than in the eastern Atlantic. Archeological records indicate they once occurred as far south as southern Spain and New England in the United States, Pleistocene records indicate great auks occurred as far south as Italy and other parts of the Mediterranean. Great auks became extinct before any natural history studies were conducted, so little is known about their lives except for a few studies of lone, captive birds and the casual records of mariners."
		great_auk.sound = "Wrah Wrah"
		great_auk.update()

		# Leopard Cat class that gets all the attributes from the AbstractAnimals class
		leopard_cat = Sound()
		leopard_cat.name = "Leopard Cat"
		leopard_cat.phylum = "Chordata"
		leopard_cat.cclass = "Mammalia"
		leopard_cat.order = "Carnivora"
		leopard_cat.family = "Felidae"
		leopard_cat.genus = "Prionailurus"
		leopard_cat.url = "http://animaldiversity.ummz.umich.edu/collections/contributors/tanya_dewey/leopardcat/large.jpg"
		leopard_cat.alifespan = "In the wild, leopard cats have an average lifespan of about 4 years, and have been known to live up to 20 years in captivity. The lifespan of captive individuals varies greatly as individuals may die from the stress of transport. When leopard cats are released into non-native environments by breeders, they usually die not long after."
		leopard_cat.habitat = "Prionailurus bengalensis is found in tropical and temperate forests, coniferous forests, shrub land habitat, and grasslands. Its distribution is limited to areas with less than 10 cm of snow annually, and it is not found in steppe or arid climates. Prionailurus bengalensis has a fairly diverse diet and is able to find food in most habitats. It seems relatively impervious to human disturbance as populations in secondary growth and disturbed areas are stable and it is often found near agricultural fields and rural settlements. Prionailurus bengalensis is an exceptional swimmer, possibly explaining its distribution on islands, and is intolerant of temperatures above 35 C, possibly explaining its absence from central India. It is capable of living at higher elevations (i.e., 3000 m) with minimal snow fall."
		leopard_cat.geolocation = "Prionailurus bengalensis is one of the most widespread carnivore species in Asia, and can be found throughout most of southern Asia. Prionailurus bengalensis occupies eastern Afghanistan and northern Pakistan, northern and coastal India, Myanmar, Laos, Thailand, Indonesia, Malaysia, Vietnam, Taiwan, Sumatra, Java, Bali, Borneo, Nepal, Korea, Cambodia, parts of the Philippines, and Eastern China. Prionailurus bengalensis has been divided into a number of subspecies over its range that differ in coloration, pelage, body length, and reproductive cycles."	
		leopard_cat.sound = "Whish Whish"
		leopard_cat	.update()

		#Array with all the animals in it
		animals = [red_panda,great_auk,leopard_cat]

		self.response.write(page.head())
		self.response.write(page.nav())
		if self.request.GET:
			indexAnimal = int(self.request.GET['animal']) #Grabs the index number out of the URL on the top of the browser
			self.response.write(page.content(animals[indexAnimal])) #fills the templating in with the index number 
		 #Buttons on the page
		self.response.write(page.foot()) # Footer of the page. Currently blank		
Пример #58
0
    def get(self):
        #import page
        page = Page()

        #airJordan Shoe Instance

        airj = Shoe()
        airj.name = 'Air Jordans'
        airj.price = 180 #$
        airj.limit = 2 #pairs
        airj.img ="http://upload.wikimedia.org/wikipedia/en/3/37/Jumpman_logo.svg"

        #nikeFoampositePro Shoe Instance

        nfp = Shoe()
        nfp.name = 'Nike Foamposite Pro'
        nfp.price = 220 #$
        nfp.limit = 1 #pairs
        nfp.img = "http://upload.wikimedia.org/wikipedia/en/3/37/Jumpman_logo.svg"

        #nikeFoampositeOne Shoe Instance

        nfo = Shoe()
        nfo.name = 'Nike Foamposite One'
        nfo.price = 220 #$
        nfo.limit = 1 #pairs
        nfo.img ="http://upload.wikimedia.org/wikipedia/en/3/37/Jumpman_logo.svg"

        #lebron Shoe Instance

        leb = Shoe()
        leb.name = 'LeBrons'
        leb.price = 200 #$
        leb.limit = 1 #pairs
        leb.img ="http://upload.wikimedia.org/wikipedia/en/3/37/Jumpman_logo.svg"


        #kobe Shoe Instance

        kobe = Shoe()
        kobe.name = 'Kobes'
        kobe.price = 120 #$
        kobe.limit = 4 #pairs
        kobe.img ="http://upload.wikimedia.org/wikipedia/en/3/37/Jumpman_logo.svg"

        shoes = [
            airj,nfp,nfo,leb,kobe
        ]

        #HTML
        self.response.write(page.header())
        self.response.write(page.form())
        if self.request.GET:
            heat = (int(self.request.GET['heat']))-1
            print heat
            self.response.write(self.html(shoes[heat]))
        self.response.write(page.footer())
Пример #59
0
 def update(self):
     logging.info("Checking sources for updates")
     for id, values in PageImporter.pages.iteritems():
         logging.info("Checking page: " + id + " (" + values['country'] + ")")
         page = Page(id, values['country'], values['url'])
         if page.import_tables() == 0:
             logging.info(id + " not modified")
         else:
             logging.info("Finished importing page: " + id + " (" + values['country'] + ")")
Пример #60
0
 def __init__(self, driver):
     Page.__init__(self, driver)
     self.altom_logo = Element(self.driver, (By.XPATH, ".//*[@id='site-title']/a/img"))
     self.sidebar = Element(self.driver, (By.CLASS_NAME, "sidebar"))
     self.main_content = Element(self.driver, (By.ID, "mainContent"))
     self.main_menu = Element(self.driver, (By.ID, "mainMenu"))
     self.video = Element(self.driver, (By.ID, "presentation"))
     self.phone_number = Element(self.driver, (By.CLASS_NAME, "mainPhone"))
     self.blocks = Element(self.driver, (By.CLASS_NAME, "blocks"))