Example #1
0
def list_contacts(selected_addressbooks, vcard_list):
    if selected_addressbooks.__len__() == 1:
        print "Address book: %s" % selected_addressbooks[0]
        table = [["Id", "Name", "Phone", "E-Mail"]]
    else:
        print "Address books: %s" % ', '.join(selected_addressbooks)
        table = [["Id", "Name", "Phone", "E-Mail", "Address book"]]
    for index, vcard in enumerate(vcard_list):
        row = []
        row.append(index + 1)
        row.append(vcard.get_full_name())
        if vcard.get_phone_numbers().__len__() > 0:
            phone1 = vcard.get_phone_numbers()[0]
            row.append("%s: %s" % (phone1['type'], phone1['value']))
        else:
            row.append("")
        if vcard.get_email_addresses().__len__() > 0:
            email1 = vcard.get_email_addresses()[0]
            row.append("%s: %s" % (email1['type'], email1['value']))
        else:
            row.append("")
        if selected_addressbooks.__len__() > 1:
            row.append(vcard.get_addressbook_name())
        table.append(row)
    print helpers.pretty_print(table)
Example #2
0
def list_contacts(selected_addressbooks, vcard_list):
    if selected_addressbooks.__len__() == 1:
        print "Address book: %s" % selected_addressbooks[0]
        table = [["Id", "Name", "Phone", "E-Mail"]]
    else:
        print "Address books: %s" % ', '.join(selected_addressbooks)
        table = [["Id", "Name", "Phone", "E-Mail", "Address book"]]
    for index, vcard in enumerate(vcard_list):
        row = []
        row.append(index+1)
        row.append(vcard.get_full_name())
        if vcard.get_phone_numbers().__len__() > 0:
            phone1 = vcard.get_phone_numbers()[0]
            row.append("%s: %s" % (phone1['type'], phone1['value']))
        else:
            row.append("")
        if vcard.get_email_addresses().__len__() > 0:
            email1 = vcard.get_email_addresses()[0]
            row.append("%s: %s" % (email1['type'], email1['value']))
        else:
            row.append("")
        if selected_addressbooks.__len__() > 1:
            row.append(vcard.get_addressbook_name())
        table.append(row)
    print helpers.pretty_print(table)
Example #3
0
async def on_message(ctx, category: str):
    helpers.command_logger(ctx.author, f'STANDINGS {category}')
    if category == 'u90':
        response = 'Tommy correctly predicted Alex Trebek would be the first under 90 pick to pass and has won 1 beer from all the bois.'
    elif category == 'u40':
        response = helpers.pretty_print(constants.picks['under40'])
    elif category == 'pick3':
        response = helpers.pretty_print(constants.picks['pick3'])
    await ctx.send(response)
    def prompt(self):
        from helpers import confirms, fetch_user_input, pretty_print

        pretty_print('\nWe hate to see you go. Are you sure you want to quit?')
        while not self.interrupt:
            self.input = fetch_user_input(
                'Press [y] to quit or any other key to [cancel].')

            if confirms(self.input):
                self.interrupt = True
                pretty_print('\nSee you soon!')
                sys.exit(0)
            else:
                self.callback()
Example #5
0
	def value_iteration(self, grid_size, epsilon, discount_factor):
		""" Calculate value iteration. Returns optimal policy grid """
		# Initialize values
		x_length = grid_size[0]
		y_length = grid_size[1]
		converged = False
		v_grid = np.zeros((x_length, y_length, x_length, y_length))
		temp_grid = np.zeros((x_length, y_length, x_length, y_length))
		delta_grid = np.zeros((x_length, y_length, x_length, y_length))
		actions = self.predator.get_policy([0,0,0,0]).keys()
		loop = 0
		# Continue until converged
		while(not converged):
			loop +=1
			# loop over possible states of game(predator(i,j) and prey (k,l))
			for i in range(0, x_length):
				for j in range(0, y_length):
					for k in range(0, x_length):
						for l in range(0, y_length):
							highest_q = 0
							best_action = ""
							# For every action calculate q-value
							for action in actions:
								q_value = self.q_value([i,j,k,l], action, temp_grid, discount_factor, grid_size)
								# Update highest q_value
								if q_value > highest_q:
									highest_q = q_value
							# Update grid
							v_grid[i][j][k][l] = highest_q

			# Calculate difference between grids
			delta_grid = abs(np.array(v_grid) - np.array(temp_grid))

			# Reset temp grid to current value grid for next run
			temp_grid = v_grid
			# Reset value grid to zeros to save all new values in
			v_grid = np.zeros((x_length, y_length, x_length, y_length))
			
			# Get highest difference in delta grid
			delta = np.amax(delta_grid)
			
			# Check if converged
			if(delta < epsilon* (1-discount_factor)/discount_factor):
				converged = True
		# Print last value grid for prey state 2,2
		helpers.pretty_print(temp_grid[:][:][2][2], label=['v'])
		# Calculate optimal policy using value grid
		optimal_policy = self.policy_from_grid(temp_grid, discount_factor, grid_size, False)
		return optimal_policy
Example #6
0
def list_contacts(vcard_list):
    selected_address_books = []
    for contact in vcard_list:
        if contact.get_address_book() not in selected_address_books:
            selected_address_books.append(contact.get_address_book())
    if len(selected_address_books) == 1:
        print("Address book: %s" % str(selected_address_books[0]))
        table = [["Id", "Name", "Phone", "E-Mail"]]
    else:
        print("Address books: %s" % ', '.join([str(book) for book in selected_address_books]))
        table = [["Id", "Name", "Phone", "E-Mail", "Address book"]]
    for index, vcard in enumerate(vcard_list):
        row = []
        row.append(index+1)
        if vcard.get_nickname() != "" \
                and Config().show_nicknames():
            row.append("%s (Nickname: %s)" % (vcard.get_full_name(), vcard.get_nickname()))
        else:
            row.append(vcard.get_full_name())
        if vcard.get_phone_numbers().__len__() > 0:
            phone1 = vcard.get_phone_numbers()[0]
            row.append("%s: %s" % (phone1['type'], phone1['value']))
        else:
            row.append("")
        if vcard.get_email_addresses().__len__() > 0:
            email1 = vcard.get_email_addresses()[0]
            row.append("%s: %s" % (email1['type'], email1['value']))
        else:
            row.append("")
        if selected_address_books.__len__() > 1:
            row.append(vcard.get_address_book().get_name())
        table.append(row)
    print(helpers.pretty_print(table))
Example #7
0
def list_contacts(vcard_list):
    selected_address_books = []
    for contact in vcard_list:
        if contact.get_address_book() not in selected_address_books:
            selected_address_books.append(contact.get_address_book())
    if len(selected_address_books) == 1:
        print("Address book: %s" % str(selected_address_books[0]))
        table = [["Id", "Name", "Phone", "E-Mail"]]
    else:
        print("Address books: %s" %
              ', '.join([str(book) for book in selected_address_books]))
        table = [["Id", "Name", "Phone", "E-Mail", "Address book"]]
    for index, vcard in enumerate(vcard_list):
        row = []
        row.append(index + 1)
        if vcard.get_nickname() != "" \
                and Config().show_nicknames():
            row.append("%s (Nickname: %s)" %
                       (vcard.get_full_name(), vcard.get_nickname()))
        else:
            row.append(vcard.get_full_name())
        if vcard.get_phone_numbers().__len__() > 0:
            phone1 = vcard.get_phone_numbers()[0]
            row.append("%s: %s" % (phone1['type'], phone1['value']))
        else:
            row.append("")
        if vcard.get_email_addresses().__len__() > 0:
            email1 = vcard.get_email_addresses()[0]
            row.append("%s: %s" % (email1['type'], email1['value']))
        else:
            row.append("")
        if selected_address_books.__len__() > 1:
            row.append(vcard.get_address_book().get_name())
        table.append(row)
    print(helpers.pretty_print(table))
Example #8
0
def main(parsed_args):

    pp_args(parsed_args)

    if 'key' in parsed_args.keys():
        set_key_globals(int(parsed_args['key']))
    else:
        set_key_globals(26)

    if 'old' in parsed_args.keys():
        set_global_command(parsed_args['old'])
    else:
        set_global_command('ax64')

    runs, actions, passalong = genruns(parsed_args)
    comment(total_runs=len(runs), actions=actions, passalong=passalong)

    fname = 'runs.txt'
    comment('W {} runs to {}'.format(len(runs), fname))
    pretty_print('W {} runs to {}'.format(len(runs), fname))

    write_runs_to_file(runs, actions, passalong, fname)
    comment('=' * 16, 'running...')
    runs_output, vectors = do_runs(runs, actions, passalong)
    comment('=' * 16, 'done!')
    write_csv('times.csv', 'NO.,TIME', runs_output)
    write_csv('vectors.csv', 'IP,OP,KEY,ROUNDS,IV,LZS', vectors)

    comment('W {} runtimes to times.csv'.format(len(runs_output)))
    pretty_print('W {} runtimes to times.csv'.format(len(runs_output)))

    comment('W {} test vectors written vectors.csv'.format(len(vectors)))
    pretty_print('W {} test vectors written vectors.csv'.format(len(vectors)))
Example #9
0
def list_contacts(vcard_list):
    selected_address_books = []
    for contact in vcard_list:
        if contact.get_address_book() not in selected_address_books:
            selected_address_books.append(contact.get_address_book())
    if len(selected_address_books) == 1:
        print("Address book: %s" % str(selected_address_books[0]))
        table = [["Index", "Name", "Phone", "E-Mail", "uid"]]
    else:
        print("Address books: %s" % ', '.join([str(book) for book in selected_address_books]))
        table = [["Index", "Name", "Phone", "E-Mail", "uid", "Address book"]]
    for index, vcard in enumerate(vcard_list):
        row = []
        row.append(index+1)
        if len(vcard.get_nicknames()) > 0 \
                and Config().show_nicknames():
            row.append("%s (Nickname: %s)" % (vcard.get_full_name(), vcard.get_nicknames()[0]))
        else:
            row.append(vcard.get_full_name())
        if len(vcard.get_phone_numbers().keys()) > 0:
            phone_dict = vcard.get_phone_numbers()
            first_type = sorted(phone_dict.keys(), key=lambda k: k[0].lower())[0]
            row.append("%s: %s" % (first_type, sorted(phone_dict.get(first_type))[0]))
        else:
            row.append("")
        if len(vcard.get_email_addresses().keys()) > 0:
            email_dict = vcard.get_email_addresses()
            first_type = sorted(email_dict.keys(), key=lambda k: k[0].lower())[0]
            row.append("%s: %s" % (first_type, sorted(email_dict.get(first_type))[0]))
        else:
            row.append("")
        if vcard.get_uid():
            row.append(vcard.get_uid()[:Config().get_length_of_uid()])
        else:
            row.append("")
        if selected_address_books.__len__() > 1:
            row.append(vcard.get_address_book().get_name())
        table.append(row)
    print(helpers.pretty_print(table))
Example #10
0
    def grab_all(self):
        self._local_setup()
        self.next_url = 'http://portal.ruc.edu.cn/cas/login?service=http%3A%2F%2Fportal.ruc.edu.cn%2Fidc%2Feducation%2Fselectcourses%2Fresultquery%2FResultQueryAction.do%3Fmethod%3DforwardAllQueryXkjg'
        self._login()

        r_cookies = requests.post(self.next_url, cookies=self.cookies, verify=False)
        content = r_cookies.content.decode(self.charset)
        self.cookies = r_cookies.cookies

        '''parser, start.'''

        ''' - get colleges'''
        strainer_colleges = SoupStrainer("select", id="condition_yx")
        soup_colleges = BeautifulSoup(r_cookies.content.decode('gbk'), parse_only=strainer_colleges)
        colleges = [option['value'] for option in soup_colleges.select("option") if option['value']]
        colleges_name = [option.get_text() for option in soup_colleges.select("option") if option['value']]
        pretty_print(colleges_name)
        print "{0} colleges.".format(len(colleges))

        ''' - iter colleges'''
        total_courses = 0
        for i, college in enumerate(colleges):
            courses = []
            url_courses = 'http://portal.ruc.edu.cn/idc/education/selectcourses/resultquery/ResultQueryAction.do'
            '''get courses'''
            for j in range(1, 15):
                data = {
                    'method': "allJxb",
                    'condition_xnd': "2012-2013",
                    'condition_xq': "1",
                    'condition_yx': college.encode('gbk'),
                    'isNeedInitSQL': "true",
                    'ksj1': j,
                    'ksj2': j,
                }
                r_courses = requests.post(url_courses, data=data, cookies=self.cookies)
                content = r_courses.content.decode('gbk')

                soup_courses = BeautifulSoup(content)
                rows = soup_courses.find_all("row")

                if len(rows) == 1:
                    continue

                for r in rows:
                    teacher = r.select("xm")[0].get_text(strip=True).replace('/', ',')
                    time_and_location_texts = r.select("sksj > tagbr")

                    lessons = self.get_lessons(time_and_location_texts)

                    course = {
                        'original_id': r.select("jxbh")[0].get_text(strip=True),
                        'name': r.select("kcmc")[0].get_text(strip=True),
                        'credit': str(float(r.select("xf")[0].get_text(strip=True))),
                        'teacher': teacher,
                        'lessons': lessons,
                    }
                    courses.append(course)

            print "#{0} {1}: {2} courses.".format(i, colleges_name[i].encode('utf8'), len(courses))
            if len(courses) == 0:
                continue
            total_courses += len(courses)
            output_dir = os.path.join(os.path.dirname(__file__), 'ruc')
            if not os.path.exists(output_dir):
                os.makedirs(output_dir)
            if courses != []:
                with open(os.path.join(output_dir, colleges_name[i] + '.yaml'), 'w') as yaml_file:
                    yaml_file.write(pretty_format(courses))
        print "Done! Totally exported {0} courses.".format(total_courses)
Example #11
0
def do_runs(runs, actions, passalong):
    global current_instance, current_round
    import sys

    runs_output = []
    vectors = []

    run_id = 0
    realtime = open('realtime.txt', 'w')

    comment('I check realtime.txt for live updates!')
    pretty_print('I check realtime.txt for live updates!')

    for run in runs:
        equations = []
        test_vectors = []
        to_fix = []
        round_vectors = []

        run_id += 1
        comment('=' * 16, '=' * 16, 'run %s' % str(run))

        fix = run['fix']
        rounds = run['rounds']
        ins = run['ins']

        cmd = get_command(sys.argv[0], run, actions)
        filename = get_filename(run, actions)

        x1 = get_key_as_eqs()
        equations += x1
        comment(num_of_key_eqns=len(x1))

        if fix is 0:
            comment("No bits fixed")
        else:
            random.seed()
            all_bits = range(1, 241)
            # WE DONT REALLY WANT ALL BITS!
            all_bits = []
            for bb in range(1, rounds + 1):
                all_bits += [bb, bb + 120]

            comment("sampling fixes from:" + str(all_bits))
            pretty_print("I sampling fixes from:" + str(all_bits))

            partiy_bits = [
                24, 48, 72, 96, 120, 24 + 120, 48 + 120, 72 + 120, 96 + 120,
                120 + 120
            ]
            for p in partiy_bits:
                if p in all_bits:
                    all_bits.remove(p)

            if int(fix) > len(all_bits):
                fix = len(all_bits)

            to_fix = random.sample(all_bits, int(fix))
            to_fix = sorted(to_fix)
            kfixes_eq = get_k_fix_eqns(to_fix)
            comment('{} bits fixed'.format(len(to_fix)))

            test_vectors += kfixes_eq
            equations += kfixes_eq

        key_to_gen_default = 'ABCD1ABCD2ABCD3ABCD4ABCD5ABCD6ABCD7ABCD8ABCD9ABCDAABCDBABCDC'

        key_to_gen_default_bits = format(int(key_to_gen_default, 16), 'b')

        key_generated = [int(bii) for bii in key_to_gen_default_bits]

        for bit in to_fix:
            key_generated[bit - 1] = 1

        key_generated = set_partity(key_generated)

        x88 = []
        for mn in range(len(key_generated)):
            x88.append('{}={}'.format(ret_str('k', '%03d' % (mn + 1)),
                                      key_generated[mn]))

        test_vectors += x88

        for instance in xrange(1, ins + 1):
            current_instance = instance
            current_round = 0

            comment('Instance {}'.format(instance))

            F = get_deterministic_iv()  # len 61
            iv = ''.join([str(i) for i in F])
            x3 = get_iv_eqns(F)
            equations += x3

            comment(iv_eqns=len(x3))

            x4 = get_input_as_str('69C7C85A3')
            x4 = get_input_as_eq(x4)
            equations += x4

            if rounds > 0:
                for r in range(1, rounds + 1):
                    x5 = get_round_eqns(r)
                    equations += x5
                comment(no_rounds_to_gen=r)
            else:
                pass

            x45 = []

            long_term_key_for_engine = {
                "alpha": 23,
                "D": D,
                "P": P,
            }

            engine = ARGON(long_term_key_for_engine, F, key_generated[:120],
                           key_generated[120:])
            engine.set_current_instance(current_instance)

            x456 = engine.u(rounds)
            x456 = x456[1:]
            x456 = [str(i) for i in x456]
            op = ''.join(x456)

            x4 = get_input_as_eq(op)
            x45 += x4

            equations += x45
            comment(output_equations=len(x45))

            kkkey = ''.join([str(i) for i in key_generated])

            test_vectors += engine.get_test_vectors()

            curr_vector = ('69C7C85A3', bin2hex_str9(op), bin2hex_str60(kkkey),
                           rounds, bin2hex_str16(iv), keynum)
            round_vectors.append(curr_vector)

        comment('=' * 16,
                instances=instance,
                total_no_equations=len(equations))

        with open(filename + '.txt', 'w') as f:

            def fcomment(*args, **kwargs):
                "Prints args & kwgars to stdout, uses a delimiter ( typically // )."
                delimiter = '//'
                for arg in args:
                    f.write('{delimiter} {com}\n'.format(com=arg,
                                                         delimiter=delimiter))
                for key, value in kwargs.iteritems():
                    f.write('{delimiter} {key} = {value}\n'.format(
                        key=key, value=value, delimiter=delimiter))

            def wreq(equations):
                for eq in equations:
                    f.write('{}\n'.format(eq))

            # Write comments
            fcomment(command=cmd,
                     run=run,
                     actions=actions,
                     passalong=passalong)
            fcomment(num_of_key_eqns=len(x1),
                     num_of_parity_eqns=10,
                     num_of_iv_eqns=len(x3))
            fcomment(generated_instances=instance, generated_rounds=rounds)
            fcomment(total_number_of_equations=len(equations))
            fcomment('vectors (in hex, also written to vectors.csv)')
            vectors += round_vectors
            for vector in round_vectors:
                fcomment('+++',
                         ip=vector[0],
                         op=vector[1],
                         key=vector[2],
                         rounds=vector[3],
                         iv=vector[4],
                         lzs=vector[5])

            if len(vectors) == 0: fcomment('+++')

            # Write equations
            wreq(equations)
            comment('W {} equations written to {}.txt'.format(
                len(equations), filename))
            pretty_print('W {} equations written to {}.txt'.format(
                len(equations), filename))
            f.close()

        with open(filename + '_solution.txt', 'w') as f:

            def fcomment(*args, **kwargs):
                "Prints args & kwgars to stdout, uses a delimiter ( typically // )."
                delimiter = '//'
                for arg in args:
                    f.write('{delimiter} {com}\n'.format(com=arg,
                                                         delimiter=delimiter))
                for key, value in kwargs.iteritems():
                    f.write('{delimiter} {key} = {value}\n'.format(
                        key=key, value=value, delimiter=delimiter))

            def wreq(eqs):
                for eq in eqs:
                    f.write('{}\n'.format(eq))

            fcomment(generated_instances=instance, generated_rounds=rounds)
            fcomment(command=cmd,
                     run=run,
                     actions=actions,
                     passalong=passalong)

            for vector in round_vectors:
                fcomment('+++',
                         ip=vector[0],
                         op=vector[1],
                         key=vector[2],
                         rounds=vector[3],
                         iv=vector[4],
                         lzs=vector[5])

            wreq(test_vectors)

            comment('W {} solutions written to {}_solution.txt'.format(
                len(test_vectors), filename))
            pretty_print('W {} solutions written to {}_solution.txt'.format(
                len(test_vectors), filename))

            f.close()

        #if 'printequationsprettilyplease' in passalong:
        #    pp_eqn(equations)

        mode_ = None

        if platform == "linux" or platform == "linux2":
            comm_ = comm[0]
        elif platform == "darwin":
            comm_ = comm[0]
        elif platform == "win32":
            comm_ = comm[1]

        if actions['sat'] and actions['xl']:
            mode_ = '4444 /deg4'
        elif actions['xl']:
            mode_ = '4000 /deg4'

        if mode_ is not None and comm is not None:
            proc = ' '.join([comm_, mode_, filename + '.txt'])  #+passalong)
            try:
                comment('C {}'.format(proc))
                pretty_print('C {}'.format(proc))

                time_it_took = timed_process(proc)
                comment('C ax64 returned')

                run_str = 'rounds={},ins={},fix={}'.format(rounds, ins, fix)
                run_str_2 = '{}, {}\n'.format(run_id, time_it_took)

                runs_output.append((run_id, run_str, time_it_took))
                realtime.write(run_str_2)
            except Exception as e:
                raise e

    realtime.close()
    return runs_output, vectors
Example #12
0
    def grab_all(self):
        # self._local_setup()
        # self._login()
        self._fake_login()

        url_courses = self.url_prefix + "jxrw_zd.aspx?xh=" + self.username
        '''get viewstate'''
        r_viewstate = requests.get(url_courses, cookies=self.cookies)
        result = re.search(
            '<input type="hidden" name="__VIEWSTATE" value="(.+)" />',
            r_viewstate.content)
        viewstate = result.group(1)

        print "Get viewstate: done."
        '''parser, start.'''
        ''' - get colleges'''
        strainer_colleges = SoupStrainer("select", id="ddlXY")
        soup_colleges = BeautifulSoup(r_viewstate.content.decode(self.charset),
                                      parse_only=strainer_colleges)
        colleges = [
            option['value'] for option in soup_colleges.select("option")
            if option['value']
        ]
        pretty_print(colleges)
        print "{} colleges.".format(len(colleges))
        ''' - iter colleges'''
        total_courses = 0
        for i, college in enumerate(colleges):
            '''get courses'''
            data = {
                '__EVENTTARGET': "",
                '__EVENTARGUMENT': "",
                '__VIEWSTATE': viewstate,
                'ddlXN': "2012-2013",
                'ddlXQ': "1",
                'ddlXY': college.encode(self.charset),
                'ddlZY': "",
                'ddlKC': "",
                'btnFilter': u' 查 询 '.encode(self.charset),
            }
            r_courses = requests.post(url_courses,
                                      data=data,
                                      cookies=self.cookies)
            content = r_courses.content.decode(self.charset)

            strainer_courses = SoupStrainer("table", id="DBGrid")
            soup_courses = BeautifulSoup(content, parse_only=strainer_courses)
            rows = soup_courses.select("tr")

            courses = []
            for r in rows:
                if r.has_key('class') and r['class'] == ["datagridhead"]:
                    continue

                cols = r.select("td")
                semester_text = cols[0].get_text(strip=True)
                teacher = cols[7].get_text(strip=True).replace('/', ',')
                time_texts = map(string.strip, cols[8].get_text().split(';'))
                locations = map(string.strip, cols[9].get_text().split(';'))

                lessons = self.get_lessons(time_texts, locations,
                                           semester_text)

                course = {
                    'original_id': cols[3].get_text(strip=True),
                    'name': cols[4].get_text(strip=True),
                    'credit': float(cols[6].get_text(strip=True)),
                    'teacher': teacher,
                    'lessons': lessons,
                }
                courses.append(course)

            print "#{} {}: {} courses.".format(i, college.encode("utf8"),
                                               len(courses))
            total_courses += len(courses)
            output_dir = os.path.join(os.path.dirname(__file__), 'zju')
            if not os.path.exists(output_dir):
                os.makedirs(output_dir)
            with open(os.path.join(output_dir,
                                   str(i) + '.yaml'), 'w') as yaml_file:
                yaml_file.write(pretty_format(courses))
            # with open(os.path.join(output_dir, str(i) + '.html'), 'w') as html_file:
            #     html_file.write(soup_courses.prettify().encode("utf8"))
        print "Done! Totally exported {} courses.".format(total_courses)
Example #13
0
    def grab_all(self):
        self._local_setup()
        self.next_url = 'http://portal.ruc.edu.cn/cas/login?service=http%3A%2F%2Fportal.ruc.edu.cn%2Fidc%2Feducation%2Fselectcourses%2Fresultquery%2FResultQueryAction.do%3Fmethod%3DforwardAllQueryXkjg'
        self._login()

        r_cookies = requests.post(self.next_url,
                                  cookies=self.cookies,
                                  verify=False)
        content = r_cookies.content.decode(self.charset)
        self.cookies = r_cookies.cookies
        '''parser, start.'''
        ''' - get colleges'''
        strainer_colleges = SoupStrainer("select", id="condition_yx")
        soup_colleges = BeautifulSoup(r_cookies.content.decode('gbk'),
                                      parse_only=strainer_colleges)
        colleges = [
            option['value'] for option in soup_colleges.select("option")
            if option['value']
        ]
        colleges_name = [
            option.get_text() for option in soup_colleges.select("option")
            if option['value']
        ]
        pretty_print(colleges_name)
        print "{0} colleges.".format(len(colleges))
        ''' - iter colleges'''
        total_courses = 0
        for i, college in enumerate(colleges):
            courses = []
            url_courses = 'http://portal.ruc.edu.cn/idc/education/selectcourses/resultquery/ResultQueryAction.do'
            '''get courses'''
            for j in range(1, 15):
                data = {
                    'method': "allJxb",
                    'condition_xnd': "2012-2013",
                    'condition_xq': "1",
                    'condition_yx': college.encode('gbk'),
                    'isNeedInitSQL': "true",
                    'ksj1': j,
                    'ksj2': j,
                }
                r_courses = requests.post(url_courses,
                                          data=data,
                                          cookies=self.cookies)
                content = r_courses.content.decode('gbk')

                soup_courses = BeautifulSoup(content)
                rows = soup_courses.find_all("row")

                if len(rows) == 1:
                    continue

                for r in rows:
                    teacher = r.select("xm")[0].get_text(strip=True).replace(
                        '/', ',')
                    time_and_location_texts = r.select("sksj > tagbr")

                    lessons = self.get_lessons(time_and_location_texts)

                    course = {
                        'original_id':
                        r.select("jxbh")[0].get_text(strip=True),
                        'name':
                        r.select("kcmc")[0].get_text(strip=True),
                        'credit':
                        str(float(r.select("xf")[0].get_text(strip=True))),
                        'teacher':
                        teacher,
                        'lessons':
                        lessons,
                    }
                    courses.append(course)

            print "#{0} {1}: {2} courses.".format(
                i, colleges_name[i].encode('utf8'), len(courses))
            if len(courses) == 0:
                continue
            total_courses += len(courses)
            output_dir = os.path.join(os.path.dirname(__file__), 'ruc')
            if not os.path.exists(output_dir):
                os.makedirs(output_dir)
            if courses != []:
                with open(os.path.join(output_dir, colleges_name[i] + '.yaml'),
                          'w') as yaml_file:
                    yaml_file.write(pretty_format(courses))
        print "Done! Totally exported {0} courses.".format(total_courses)
Example #14
0
    def grab_all(self):
        self._local_setup()
        self._login()

        url_courses = self.url_prefix + 'courseSearchAction.do?temp=1'

        '''get TOKEN'''
        r_viewstate = requests.get(url_courses, cookies=self.cookies)
        result = re.search('<input type="hidden" name="org.apache.struts.taglib.html.TOKEN" value="(.+)">', r_viewstate.content)
        TOKEN = result.group(1)

        print "Get TOKEN: done."

        '''parser, start.'''

        ''' - get colleges'''
        strainer_colleges = SoupStrainer('select', id="xsjc")
        soup_colleges = BeautifulSoup(r_viewstate.content.decode('gbk').replace('name', 'id'), parse_only=strainer_colleges)
        colleges = [option['value'] for option in soup_colleges.select("option") if option['value']]
        pretty_print(colleges)
        print "{0} colleges.".format(len(colleges))

        ''' - iter colleges'''
        url_courses = self.url_prefix + 'courseSearchAction.do'
        total_courses = 0
        for i, college in enumerate(colleges):
            '''get courses'''
            showColumn = [u'kch#课程号'.encode('gbk'), u'kcm#课程名'.encode('gbk'), u'xf#学分'.encode('gbk'), u'skjs#教师'.encode('gbk'), u'zcsm#周次'.encode('gbk'), u'skxq#星期'.encode('gbk'), u'skjc#节次'.encode('gbk'), u'xqm#校区'.encode('gbk'), u'jxlm#教学楼'.encode('gbk'), u'jasm#教室'.encode('gbk'), u'kxh#课序号'.encode('gbk')]
            data = {
                'org.apache.struts.taglib.html.TOKEN': TOKEN.encode('gbk'),
                'pageNumber': "0".encode('gbk'),
                'actionType': "1".encode('gbk'),
                'xsjc': college.encode('gbk'),
                'pageSize': '1000'.encode('gbk'),
                'showColumn': showColumn,
            }
            r_courses = requests.post(url_courses, data=data, cookies=self.cookies)
            content = r_courses.content.decode('gbk')

            strainer_courses = SoupStrainer("table", id="titleTop2")
            soup_courses = BeautifulSoup(content.replace('class', 'id'), parse_only=strainer_courses)
            rows = soup_courses.select("tr")
            prev_code_name = '-1'

            courses = []
            for r in rows:
                if not r.has_key('id'):
                    continue

                cols = r.select("td")
                try:
                    test_text = cols[0].get_text(strip=True)
                except:
                    break
                teacher = self.get_teachers(cols[3].get_text(strip=True))
                weeks_text = cols[4].get_text(strip=True)
                day_text = cols[5].get_text(strip=True)
                start_end_text = cols[6].get_text(strip=True)
                location = cols[7].get_text(strip=True) + ' ' + cols[8].get_text(strip=True) + ' ' + cols[9].get_text(strip=True)
                lessons = self.get_lessons(weeks_text, day_text, start_end_text, location)
                code_name = cols[10].get_text(strip=True)

                course = {
                    'original_id': cols[0].get_text(strip=True),
                    'name': cols[1].get_text(strip=True),
                    'credit': str(float(cols[2].get_text(strip=True).replace('&nbsp;', ''))),
                    'teacher': teacher,
                    'lessons': lessons,
                }

                try:
                    last_course = courses.pop()
                except:
                    pass
                else:
                    if course['original_id'] == last_course['original_id'] and course['teacher'] == last_course['teacher'] and prev_code_name == code_name:
                        course['lessons'] = course['lessons'] + last_course['lessons']
                    else:
                        courses.append(last_course)

                prev_code_name = code_name
                courses.append(course)

            print "#{0} {1}: {2} courses.".format(i, college.encode("utf8"), len(courses))
            total_courses += len(courses)
            output_dir = os.path.join(os.path.dirname(__file__), 'bupt')
            if not os.path.exists(output_dir):
                os.makedirs(output_dir)
            if courses != []:
                with open(os.path.join(output_dir, str(i) + '.yaml'), 'w') as yaml_file:
                    yaml_file.write(pretty_format(courses))
        print "Done! Totally exported {0} courses.".format(total_courses)
Example #15
0
    def grab_all(self):
        # self._local_setup()
        # self._login()
        self._fake_login()

        url_courses = self.url_prefix + "jxrw_zd.aspx?xh=" + self.username

        '''get viewstate'''
        r_viewstate = requests.get(url_courses, cookies=self.cookies)
        result = re.search('<input type="hidden" name="__VIEWSTATE" value="(.+)" />', r_viewstate.content)
        viewstate = result.group(1)

        print "Get viewstate: done."

        '''parser, start.'''

        ''' - get colleges'''
        strainer_colleges = SoupStrainer("select", id="ddlXY")
        soup_colleges = BeautifulSoup(r_viewstate.content.decode(self.charset), parse_only=strainer_colleges)
        colleges = [option['value'] for option in soup_colleges.select("option") if option['value']]
        pretty_print(colleges)
        print "{} colleges.".format(len(colleges))

        ''' - iter colleges'''
        total_courses = 0
        for i, college in enumerate(colleges):
            '''get courses'''
            data = {
                '__EVENTTARGET': "",
                '__EVENTARGUMENT': "",
                '__VIEWSTATE': viewstate,
                'ddlXN': "2012-2013",
                'ddlXQ': "1",
                'ddlXY': college.encode(self.charset),
                'ddlZY': "",
                'ddlKC': "",
                'btnFilter': u' 查 询 '.encode(self.charset),
            }
            r_courses = requests.post(url_courses, data=data, cookies=self.cookies)
            content = r_courses.content.decode(self.charset)

            strainer_courses = SoupStrainer("table", id="DBGrid")
            soup_courses = BeautifulSoup(content, parse_only=strainer_courses)
            rows = soup_courses.select("tr")

            courses = []
            for r in rows:
                if r.has_key('class') and r['class'] == ["datagridhead"]:
                    continue

                cols = r.select("td")
                semester_text = cols[0].get_text(strip=True)
                teacher = cols[7].get_text(strip=True).replace('/', ',')
                time_texts = map(string.strip, cols[8].get_text().split(';'))
                locations = map(string.strip, cols[9].get_text().split(';'))

                lessons = self.get_lessons(time_texts, locations, semester_text)

                course = {
                    'original_id': cols[3].get_text(strip=True),
                    'name': cols[4].get_text(strip=True),
                    'credit': float(cols[6].get_text(strip=True)),
                    'teacher': teacher,
                    'lessons': lessons,
                }
                courses.append(course)

            print "#{} {}: {} courses.".format(i, college.encode("utf8"), len(courses))
            total_courses += len(courses)
            output_dir = os.path.join(os.path.dirname(__file__), 'zju')
            if not os.path.exists(output_dir):
                os.makedirs(output_dir)
            with open(os.path.join(output_dir, str(i) + '.yaml'), 'w') as yaml_file:
                yaml_file.write(pretty_format(courses))
            # with open(os.path.join(output_dir, str(i) + '.html'), 'w') as html_file:
            #     html_file.write(soup_courses.prettify().encode("utf8"))
        print "Done! Totally exported {} courses.".format(total_courses)