def list_contacts(selected_addressbooks, vcard_list):
if selected_addressbooks.__len__() == 1:
print "Address book: %s" % selected_addressbooks[0]
table = [["Id", "Name", "Phone", "E-Mail"]]
else:
print "Address books: %s" % ', '.join(selected_addressbooks)
table = [["Id", "Name", "Phone", "E-Mail", "Address book"]]
for index, vcard in enumerate(vcard_list):
row = []
row.append(index + 1)
row.append(vcard.get_full_name())
if vcard.get_phone_numbers().__len__() > 0:
phone1 = vcard.get_phone_numbers()[0]
row.append("%s: %s" % (phone1['type'], phone1['value']))
else:
row.append("")
if vcard.get_email_addresses().__len__() > 0:
email1 = vcard.get_email_addresses()[0]
row.append("%s: %s" % (email1['type'], email1['value']))
else:
row.append("")
if selected_addressbooks.__len__() > 1:
row.append(vcard.get_addressbook_name())
table.append(row)
print helpers.pretty_print(table)
def list_contacts(selected_addressbooks, vcard_list):
if selected_addressbooks.__len__() == 1:
print "Address book: %s" % selected_addressbooks[0]
table = [["Id", "Name", "Phone", "E-Mail"]]
else:
print "Address books: %s" % ', '.join(selected_addressbooks)
table = [["Id", "Name", "Phone", "E-Mail", "Address book"]]
for index, vcard in enumerate(vcard_list):
row = []
row.append(index+1)
row.append(vcard.get_full_name())
if vcard.get_phone_numbers().__len__() > 0:
phone1 = vcard.get_phone_numbers()[0]
row.append("%s: %s" % (phone1['type'], phone1['value']))
else:
row.append("")
if vcard.get_email_addresses().__len__() > 0:
email1 = vcard.get_email_addresses()[0]
row.append("%s: %s" % (email1['type'], email1['value']))
else:
row.append("")
if selected_addressbooks.__len__() > 1:
row.append(vcard.get_addressbook_name())
table.append(row)
print helpers.pretty_print(table)
async def on_message(ctx, category: str):
helpers.command_logger(ctx.author, f'STANDINGS {category}')
if category == 'u90':
response = 'Tommy correctly predicted Alex Trebek would be the first under 90 pick to pass and has won 1 beer from all the bois.'
elif category == 'u40':
response = helpers.pretty_print(constants.picks['under40'])
elif category == 'pick3':
response = helpers.pretty_print(constants.picks['pick3'])
await ctx.send(response)
def prompt(self):
from helpers import confirms, fetch_user_input, pretty_print
pretty_print('\nWe hate to see you go. Are you sure you want to quit?')
while not self.interrupt:
self.input = fetch_user_input(
'Press [y] to quit or any other key to [cancel].')
if confirms(self.input):
self.interrupt = True
pretty_print('\nSee you soon!')
sys.exit(0)
else:
self.callback()
def value_iteration(self, grid_size, epsilon, discount_factor): """ Calculate value iteration. Returns optimal policy grid """ # Initialize values x_length = grid_size[0] y_length = grid_size[1] converged = False v_grid = np.zeros((x_length, y_length, x_length, y_length)) temp_grid = np.zeros((x_length, y_length, x_length, y_length)) delta_grid = np.zeros((x_length, y_length, x_length, y_length)) actions = self.predator.get_policy([0,0,0,0]).keys() loop = 0 # Continue until converged while(not converged): loop +=1 # loop over possible states of game(predator(i,j) and prey (k,l)) for i in range(0, x_length): for j in range(0, y_length): for k in range(0, x_length): for l in range(0, y_length): highest_q = 0 best_action = "" # For every action calculate q-value for action in actions: q_value = self.q_value([i,j,k,l], action, temp_grid, discount_factor, grid_size) # Update highest q_value if q_value > highest_q: highest_q = q_value # Update grid v_grid[i][j][k][l] = highest_q # Calculate difference between grids delta_grid = abs(np.array(v_grid) - np.array(temp_grid)) # Reset temp grid to current value grid for next run temp_grid = v_grid # Reset value grid to zeros to save all new values in v_grid = np.zeros((x_length, y_length, x_length, y_length)) # Get highest difference in delta grid delta = np.amax(delta_grid) # Check if converged if(delta < epsilon* (1-discount_factor)/discount_factor): converged = True # Print last value grid for prey state 2,2 helpers.pretty_print(temp_grid[:][:][2][2], label=['v']) # Calculate optimal policy using value grid optimal_policy = self.policy_from_grid(temp_grid, discount_factor, grid_size, False) return optimal_policy
def list_contacts(vcard_list):
selected_address_books = []
for contact in vcard_list:
if contact.get_address_book() not in selected_address_books:
selected_address_books.append(contact.get_address_book())
if len(selected_address_books) == 1:
print("Address book: %s" % str(selected_address_books[0]))
table = [["Id", "Name", "Phone", "E-Mail"]]
else:
print("Address books: %s" % ', '.join([str(book) for book in selected_address_books]))
table = [["Id", "Name", "Phone", "E-Mail", "Address book"]]
for index, vcard in enumerate(vcard_list):
row = []
row.append(index+1)
if vcard.get_nickname() != "" \
and Config().show_nicknames():
row.append("%s (Nickname: %s)" % (vcard.get_full_name(), vcard.get_nickname()))
else:
row.append(vcard.get_full_name())
if vcard.get_phone_numbers().__len__() > 0:
phone1 = vcard.get_phone_numbers()[0]
row.append("%s: %s" % (phone1['type'], phone1['value']))
else:
row.append("")
if vcard.get_email_addresses().__len__() > 0:
email1 = vcard.get_email_addresses()[0]
row.append("%s: %s" % (email1['type'], email1['value']))
else:
row.append("")
if selected_address_books.__len__() > 1:
row.append(vcard.get_address_book().get_name())
table.append(row)
print(helpers.pretty_print(table))
def list_contacts(vcard_list):
selected_address_books = []
for contact in vcard_list:
if contact.get_address_book() not in selected_address_books:
selected_address_books.append(contact.get_address_book())
if len(selected_address_books) == 1:
print("Address book: %s" % str(selected_address_books[0]))
table = [["Id", "Name", "Phone", "E-Mail"]]
else:
print("Address books: %s" %
', '.join([str(book) for book in selected_address_books]))
table = [["Id", "Name", "Phone", "E-Mail", "Address book"]]
for index, vcard in enumerate(vcard_list):
row = []
row.append(index + 1)
if vcard.get_nickname() != "" \
and Config().show_nicknames():
row.append("%s (Nickname: %s)" %
(vcard.get_full_name(), vcard.get_nickname()))
else:
row.append(vcard.get_full_name())
if vcard.get_phone_numbers().__len__() > 0:
phone1 = vcard.get_phone_numbers()[0]
row.append("%s: %s" % (phone1['type'], phone1['value']))
else:
row.append("")
if vcard.get_email_addresses().__len__() > 0:
email1 = vcard.get_email_addresses()[0]
row.append("%s: %s" % (email1['type'], email1['value']))
else:
row.append("")
if selected_address_books.__len__() > 1:
row.append(vcard.get_address_book().get_name())
table.append(row)
print(helpers.pretty_print(table))
def main(parsed_args):
pp_args(parsed_args)
if 'key' in parsed_args.keys():
set_key_globals(int(parsed_args['key']))
else:
set_key_globals(26)
if 'old' in parsed_args.keys():
set_global_command(parsed_args['old'])
else:
set_global_command('ax64')
runs, actions, passalong = genruns(parsed_args)
comment(total_runs=len(runs), actions=actions, passalong=passalong)
fname = 'runs.txt'
comment('W {} runs to {}'.format(len(runs), fname))
pretty_print('W {} runs to {}'.format(len(runs), fname))
write_runs_to_file(runs, actions, passalong, fname)
comment('=' * 16, 'running...')
runs_output, vectors = do_runs(runs, actions, passalong)
comment('=' * 16, 'done!')
write_csv('times.csv', 'NO.,TIME', runs_output)
write_csv('vectors.csv', 'IP,OP,KEY,ROUNDS,IV,LZS', vectors)
comment('W {} runtimes to times.csv'.format(len(runs_output)))
pretty_print('W {} runtimes to times.csv'.format(len(runs_output)))
comment('W {} test vectors written vectors.csv'.format(len(vectors)))
pretty_print('W {} test vectors written vectors.csv'.format(len(vectors)))
def list_contacts(vcard_list):
selected_address_books = []
for contact in vcard_list:
if contact.get_address_book() not in selected_address_books:
selected_address_books.append(contact.get_address_book())
if len(selected_address_books) == 1:
print("Address book: %s" % str(selected_address_books[0]))
table = [["Index", "Name", "Phone", "E-Mail", "uid"]]
else:
print("Address books: %s" % ', '.join([str(book) for book in selected_address_books]))
table = [["Index", "Name", "Phone", "E-Mail", "uid", "Address book"]]
for index, vcard in enumerate(vcard_list):
row = []
row.append(index+1)
if len(vcard.get_nicknames()) > 0 \
and Config().show_nicknames():
row.append("%s (Nickname: %s)" % (vcard.get_full_name(), vcard.get_nicknames()[0]))
else:
row.append(vcard.get_full_name())
if len(vcard.get_phone_numbers().keys()) > 0:
phone_dict = vcard.get_phone_numbers()
first_type = sorted(phone_dict.keys(), key=lambda k: k[0].lower())[0]
row.append("%s: %s" % (first_type, sorted(phone_dict.get(first_type))[0]))
else:
row.append("")
if len(vcard.get_email_addresses().keys()) > 0:
email_dict = vcard.get_email_addresses()
first_type = sorted(email_dict.keys(), key=lambda k: k[0].lower())[0]
row.append("%s: %s" % (first_type, sorted(email_dict.get(first_type))[0]))
else:
row.append("")
if vcard.get_uid():
row.append(vcard.get_uid()[:Config().get_length_of_uid()])
else:
row.append("")
if selected_address_books.__len__() > 1:
row.append(vcard.get_address_book().get_name())
table.append(row)
print(helpers.pretty_print(table))
def grab_all(self):
self._local_setup()
self.next_url = 'http://portal.ruc.edu.cn/cas/login?service=http%3A%2F%2Fportal.ruc.edu.cn%2Fidc%2Feducation%2Fselectcourses%2Fresultquery%2FResultQueryAction.do%3Fmethod%3DforwardAllQueryXkjg'
self._login()
r_cookies = requests.post(self.next_url, cookies=self.cookies, verify=False)
content = r_cookies.content.decode(self.charset)
self.cookies = r_cookies.cookies
'''parser, start.'''
''' - get colleges'''
strainer_colleges = SoupStrainer("select", id="condition_yx")
soup_colleges = BeautifulSoup(r_cookies.content.decode('gbk'), parse_only=strainer_colleges)
colleges = [option['value'] for option in soup_colleges.select("option") if option['value']]
colleges_name = [option.get_text() for option in soup_colleges.select("option") if option['value']]
pretty_print(colleges_name)
print "{0} colleges.".format(len(colleges))
''' - iter colleges'''
total_courses = 0
for i, college in enumerate(colleges):
courses = []
url_courses = 'http://portal.ruc.edu.cn/idc/education/selectcourses/resultquery/ResultQueryAction.do'
'''get courses'''
for j in range(1, 15):
data = {
'method': "allJxb",
'condition_xnd': "2012-2013",
'condition_xq': "1",
'condition_yx': college.encode('gbk'),
'isNeedInitSQL': "true",
'ksj1': j,
'ksj2': j,
}
r_courses = requests.post(url_courses, data=data, cookies=self.cookies)
content = r_courses.content.decode('gbk')
soup_courses = BeautifulSoup(content)
rows = soup_courses.find_all("row")
if len(rows) == 1:
continue
for r in rows:
teacher = r.select("xm")[0].get_text(strip=True).replace('/', ',')
time_and_location_texts = r.select("sksj > tagbr")
lessons = self.get_lessons(time_and_location_texts)
course = {
'original_id': r.select("jxbh")[0].get_text(strip=True),
'name': r.select("kcmc")[0].get_text(strip=True),
'credit': str(float(r.select("xf")[0].get_text(strip=True))),
'teacher': teacher,
'lessons': lessons,
}
courses.append(course)
print "#{0} {1}: {2} courses.".format(i, colleges_name[i].encode('utf8'), len(courses))
if len(courses) == 0:
continue
total_courses += len(courses)
output_dir = os.path.join(os.path.dirname(__file__), 'ruc')
if not os.path.exists(output_dir):
os.makedirs(output_dir)
if courses != []:
with open(os.path.join(output_dir, colleges_name[i] + '.yaml'), 'w') as yaml_file:
yaml_file.write(pretty_format(courses))
print "Done! Totally exported {0} courses.".format(total_courses)
def do_runs(runs, actions, passalong):
global current_instance, current_round
import sys
runs_output = []
vectors = []
run_id = 0
realtime = open('realtime.txt', 'w')
comment('I check realtime.txt for live updates!')
pretty_print('I check realtime.txt for live updates!')
for run in runs:
equations = []
test_vectors = []
to_fix = []
round_vectors = []
run_id += 1
comment('=' * 16, '=' * 16, 'run %s' % str(run))
fix = run['fix']
rounds = run['rounds']
ins = run['ins']
cmd = get_command(sys.argv[0], run, actions)
filename = get_filename(run, actions)
x1 = get_key_as_eqs()
equations += x1
comment(num_of_key_eqns=len(x1))
if fix is 0:
comment("No bits fixed")
else:
random.seed()
all_bits = range(1, 241)
# WE DONT REALLY WANT ALL BITS!
all_bits = []
for bb in range(1, rounds + 1):
all_bits += [bb, bb + 120]
comment("sampling fixes from:" + str(all_bits))
pretty_print("I sampling fixes from:" + str(all_bits))
partiy_bits = [
24, 48, 72, 96, 120, 24 + 120, 48 + 120, 72 + 120, 96 + 120,
120 + 120
]
for p in partiy_bits:
if p in all_bits:
all_bits.remove(p)
if int(fix) > len(all_bits):
fix = len(all_bits)
to_fix = random.sample(all_bits, int(fix))
to_fix = sorted(to_fix)
kfixes_eq = get_k_fix_eqns(to_fix)
comment('{} bits fixed'.format(len(to_fix)))
test_vectors += kfixes_eq
equations += kfixes_eq
key_to_gen_default = 'ABCD1ABCD2ABCD3ABCD4ABCD5ABCD6ABCD7ABCD8ABCD9ABCDAABCDBABCDC'
key_to_gen_default_bits = format(int(key_to_gen_default, 16), 'b')
key_generated = [int(bii) for bii in key_to_gen_default_bits]
for bit in to_fix:
key_generated[bit - 1] = 1
key_generated = set_partity(key_generated)
x88 = []
for mn in range(len(key_generated)):
x88.append('{}={}'.format(ret_str('k', '%03d' % (mn + 1)),
key_generated[mn]))
test_vectors += x88
for instance in xrange(1, ins + 1):
current_instance = instance
current_round = 0
comment('Instance {}'.format(instance))
F = get_deterministic_iv() # len 61
iv = ''.join([str(i) for i in F])
x3 = get_iv_eqns(F)
equations += x3
comment(iv_eqns=len(x3))
x4 = get_input_as_str('69C7C85A3')
x4 = get_input_as_eq(x4)
equations += x4
if rounds > 0:
for r in range(1, rounds + 1):
x5 = get_round_eqns(r)
equations += x5
comment(no_rounds_to_gen=r)
else:
pass
x45 = []
long_term_key_for_engine = {
"alpha": 23,
"D": D,
"P": P,
}
engine = ARGON(long_term_key_for_engine, F, key_generated[:120],
key_generated[120:])
engine.set_current_instance(current_instance)
x456 = engine.u(rounds)
x456 = x456[1:]
x456 = [str(i) for i in x456]
op = ''.join(x456)
x4 = get_input_as_eq(op)
x45 += x4
equations += x45
comment(output_equations=len(x45))
kkkey = ''.join([str(i) for i in key_generated])
test_vectors += engine.get_test_vectors()
curr_vector = ('69C7C85A3', bin2hex_str9(op), bin2hex_str60(kkkey),
rounds, bin2hex_str16(iv), keynum)
round_vectors.append(curr_vector)
comment('=' * 16,
instances=instance,
total_no_equations=len(equations))
with open(filename + '.txt', 'w') as f:
def fcomment(*args, **kwargs):
"Prints args & kwgars to stdout, uses a delimiter ( typically // )."
delimiter = '//'
for arg in args:
f.write('{delimiter} {com}\n'.format(com=arg,
delimiter=delimiter))
for key, value in kwargs.iteritems():
f.write('{delimiter} {key} = {value}\n'.format(
key=key, value=value, delimiter=delimiter))
def wreq(equations):
for eq in equations:
f.write('{}\n'.format(eq))
# Write comments
fcomment(command=cmd,
run=run,
actions=actions,
passalong=passalong)
fcomment(num_of_key_eqns=len(x1),
num_of_parity_eqns=10,
num_of_iv_eqns=len(x3))
fcomment(generated_instances=instance, generated_rounds=rounds)
fcomment(total_number_of_equations=len(equations))
fcomment('vectors (in hex, also written to vectors.csv)')
vectors += round_vectors
for vector in round_vectors:
fcomment('+++',
ip=vector[0],
op=vector[1],
key=vector[2],
rounds=vector[3],
iv=vector[4],
lzs=vector[5])
if len(vectors) == 0: fcomment('+++')
# Write equations
wreq(equations)
comment('W {} equations written to {}.txt'.format(
len(equations), filename))
pretty_print('W {} equations written to {}.txt'.format(
len(equations), filename))
f.close()
with open(filename + '_solution.txt', 'w') as f:
def fcomment(*args, **kwargs):
"Prints args & kwgars to stdout, uses a delimiter ( typically // )."
delimiter = '//'
for arg in args:
f.write('{delimiter} {com}\n'.format(com=arg,
delimiter=delimiter))
for key, value in kwargs.iteritems():
f.write('{delimiter} {key} = {value}\n'.format(
key=key, value=value, delimiter=delimiter))
def wreq(eqs):
for eq in eqs:
f.write('{}\n'.format(eq))
fcomment(generated_instances=instance, generated_rounds=rounds)
fcomment(command=cmd,
run=run,
actions=actions,
passalong=passalong)
for vector in round_vectors:
fcomment('+++',
ip=vector[0],
op=vector[1],
key=vector[2],
rounds=vector[3],
iv=vector[4],
lzs=vector[5])
wreq(test_vectors)
comment('W {} solutions written to {}_solution.txt'.format(
len(test_vectors), filename))
pretty_print('W {} solutions written to {}_solution.txt'.format(
len(test_vectors), filename))
f.close()
#if 'printequationsprettilyplease' in passalong:
# pp_eqn(equations)
mode_ = None
if platform == "linux" or platform == "linux2":
comm_ = comm[0]
elif platform == "darwin":
comm_ = comm[0]
elif platform == "win32":
comm_ = comm[1]
if actions['sat'] and actions['xl']:
mode_ = '4444 /deg4'
elif actions['xl']:
mode_ = '4000 /deg4'
if mode_ is not None and comm is not None:
proc = ' '.join([comm_, mode_, filename + '.txt']) #+passalong)
try:
comment('C {}'.format(proc))
pretty_print('C {}'.format(proc))
time_it_took = timed_process(proc)
comment('C ax64 returned')
run_str = 'rounds={},ins={},fix={}'.format(rounds, ins, fix)
run_str_2 = '{}, {}\n'.format(run_id, time_it_took)
runs_output.append((run_id, run_str, time_it_took))
realtime.write(run_str_2)
except Exception as e:
raise e
realtime.close()
return runs_output, vectors
def grab_all(self):
# self._local_setup()
# self._login()
self._fake_login()
url_courses = self.url_prefix + "jxrw_zd.aspx?xh=" + self.username
'''get viewstate'''
r_viewstate = requests.get(url_courses, cookies=self.cookies)
result = re.search(
'<input type="hidden" name="__VIEWSTATE" value="(.+)" />',
r_viewstate.content)
viewstate = result.group(1)
print "Get viewstate: done."
'''parser, start.'''
''' - get colleges'''
strainer_colleges = SoupStrainer("select", id="ddlXY")
soup_colleges = BeautifulSoup(r_viewstate.content.decode(self.charset),
parse_only=strainer_colleges)
colleges = [
option['value'] for option in soup_colleges.select("option")
if option['value']
]
pretty_print(colleges)
print "{} colleges.".format(len(colleges))
''' - iter colleges'''
total_courses = 0
for i, college in enumerate(colleges):
'''get courses'''
data = {
'__EVENTTARGET': "",
'__EVENTARGUMENT': "",
'__VIEWSTATE': viewstate,
'ddlXN': "2012-2013",
'ddlXQ': "1",
'ddlXY': college.encode(self.charset),
'ddlZY': "",
'ddlKC': "",
'btnFilter': u' 查 询 '.encode(self.charset),
}
r_courses = requests.post(url_courses,
data=data,
cookies=self.cookies)
content = r_courses.content.decode(self.charset)
strainer_courses = SoupStrainer("table", id="DBGrid")
soup_courses = BeautifulSoup(content, parse_only=strainer_courses)
rows = soup_courses.select("tr")
courses = []
for r in rows:
if r.has_key('class') and r['class'] == ["datagridhead"]:
continue
cols = r.select("td")
semester_text = cols[0].get_text(strip=True)
teacher = cols[7].get_text(strip=True).replace('/', ',')
time_texts = map(string.strip, cols[8].get_text().split(';'))
locations = map(string.strip, cols[9].get_text().split(';'))
lessons = self.get_lessons(time_texts, locations,
semester_text)
course = {
'original_id': cols[3].get_text(strip=True),
'name': cols[4].get_text(strip=True),
'credit': float(cols[6].get_text(strip=True)),
'teacher': teacher,
'lessons': lessons,
}
courses.append(course)
print "#{} {}: {} courses.".format(i, college.encode("utf8"),
len(courses))
total_courses += len(courses)
output_dir = os.path.join(os.path.dirname(__file__), 'zju')
if not os.path.exists(output_dir):
os.makedirs(output_dir)
with open(os.path.join(output_dir,
str(i) + '.yaml'), 'w') as yaml_file:
yaml_file.write(pretty_format(courses))
# with open(os.path.join(output_dir, str(i) + '.html'), 'w') as html_file:
# html_file.write(soup_courses.prettify().encode("utf8"))
print "Done! Totally exported {} courses.".format(total_courses)
def grab_all(self):
self._local_setup()
self.next_url = 'http://portal.ruc.edu.cn/cas/login?service=http%3A%2F%2Fportal.ruc.edu.cn%2Fidc%2Feducation%2Fselectcourses%2Fresultquery%2FResultQueryAction.do%3Fmethod%3DforwardAllQueryXkjg'
self._login()
r_cookies = requests.post(self.next_url,
cookies=self.cookies,
verify=False)
content = r_cookies.content.decode(self.charset)
self.cookies = r_cookies.cookies
'''parser, start.'''
''' - get colleges'''
strainer_colleges = SoupStrainer("select", id="condition_yx")
soup_colleges = BeautifulSoup(r_cookies.content.decode('gbk'),
parse_only=strainer_colleges)
colleges = [
option['value'] for option in soup_colleges.select("option")
if option['value']
]
colleges_name = [
option.get_text() for option in soup_colleges.select("option")
if option['value']
]
pretty_print(colleges_name)
print "{0} colleges.".format(len(colleges))
''' - iter colleges'''
total_courses = 0
for i, college in enumerate(colleges):
courses = []
url_courses = 'http://portal.ruc.edu.cn/idc/education/selectcourses/resultquery/ResultQueryAction.do'
'''get courses'''
for j in range(1, 15):
data = {
'method': "allJxb",
'condition_xnd': "2012-2013",
'condition_xq': "1",
'condition_yx': college.encode('gbk'),
'isNeedInitSQL': "true",
'ksj1': j,
'ksj2': j,
}
r_courses = requests.post(url_courses,
data=data,
cookies=self.cookies)
content = r_courses.content.decode('gbk')
soup_courses = BeautifulSoup(content)
rows = soup_courses.find_all("row")
if len(rows) == 1:
continue
for r in rows:
teacher = r.select("xm")[0].get_text(strip=True).replace(
'/', ',')
time_and_location_texts = r.select("sksj > tagbr")
lessons = self.get_lessons(time_and_location_texts)
course = {
'original_id':
r.select("jxbh")[0].get_text(strip=True),
'name':
r.select("kcmc")[0].get_text(strip=True),
'credit':
str(float(r.select("xf")[0].get_text(strip=True))),
'teacher':
teacher,
'lessons':
lessons,
}
courses.append(course)
print "#{0} {1}: {2} courses.".format(
i, colleges_name[i].encode('utf8'), len(courses))
if len(courses) == 0:
continue
total_courses += len(courses)
output_dir = os.path.join(os.path.dirname(__file__), 'ruc')
if not os.path.exists(output_dir):
os.makedirs(output_dir)
if courses != []:
with open(os.path.join(output_dir, colleges_name[i] + '.yaml'),
'w') as yaml_file:
yaml_file.write(pretty_format(courses))
print "Done! Totally exported {0} courses.".format(total_courses)
def grab_all(self):
self._local_setup()
self._login()
url_courses = self.url_prefix + 'courseSearchAction.do?temp=1'
'''get TOKEN'''
r_viewstate = requests.get(url_courses, cookies=self.cookies)
result = re.search('<input type="hidden" name="org.apache.struts.taglib.html.TOKEN" value="(.+)">', r_viewstate.content)
TOKEN = result.group(1)
print "Get TOKEN: done."
'''parser, start.'''
''' - get colleges'''
strainer_colleges = SoupStrainer('select', id="xsjc")
soup_colleges = BeautifulSoup(r_viewstate.content.decode('gbk').replace('name', 'id'), parse_only=strainer_colleges)
colleges = [option['value'] for option in soup_colleges.select("option") if option['value']]
pretty_print(colleges)
print "{0} colleges.".format(len(colleges))
''' - iter colleges'''
url_courses = self.url_prefix + 'courseSearchAction.do'
total_courses = 0
for i, college in enumerate(colleges):
'''get courses'''
showColumn = [u'kch#课程号'.encode('gbk'), u'kcm#课程名'.encode('gbk'), u'xf#学分'.encode('gbk'), u'skjs#教师'.encode('gbk'), u'zcsm#周次'.encode('gbk'), u'skxq#星期'.encode('gbk'), u'skjc#节次'.encode('gbk'), u'xqm#校区'.encode('gbk'), u'jxlm#教学楼'.encode('gbk'), u'jasm#教室'.encode('gbk'), u'kxh#课序号'.encode('gbk')]
data = {
'org.apache.struts.taglib.html.TOKEN': TOKEN.encode('gbk'),
'pageNumber': "0".encode('gbk'),
'actionType': "1".encode('gbk'),
'xsjc': college.encode('gbk'),
'pageSize': '1000'.encode('gbk'),
'showColumn': showColumn,
}
r_courses = requests.post(url_courses, data=data, cookies=self.cookies)
content = r_courses.content.decode('gbk')
strainer_courses = SoupStrainer("table", id="titleTop2")
soup_courses = BeautifulSoup(content.replace('class', 'id'), parse_only=strainer_courses)
rows = soup_courses.select("tr")
prev_code_name = '-1'
courses = []
for r in rows:
if not r.has_key('id'):
continue
cols = r.select("td")
try:
test_text = cols[0].get_text(strip=True)
except:
break
teacher = self.get_teachers(cols[3].get_text(strip=True))
weeks_text = cols[4].get_text(strip=True)
day_text = cols[5].get_text(strip=True)
start_end_text = cols[6].get_text(strip=True)
location = cols[7].get_text(strip=True) + ' ' + cols[8].get_text(strip=True) + ' ' + cols[9].get_text(strip=True)
lessons = self.get_lessons(weeks_text, day_text, start_end_text, location)
code_name = cols[10].get_text(strip=True)
course = {
'original_id': cols[0].get_text(strip=True),
'name': cols[1].get_text(strip=True),
'credit': str(float(cols[2].get_text(strip=True).replace(' ', ''))),
'teacher': teacher,
'lessons': lessons,
}
try:
last_course = courses.pop()
except:
pass
else:
if course['original_id'] == last_course['original_id'] and course['teacher'] == last_course['teacher'] and prev_code_name == code_name:
course['lessons'] = course['lessons'] + last_course['lessons']
else:
courses.append(last_course)
prev_code_name = code_name
courses.append(course)
print "#{0} {1}: {2} courses.".format(i, college.encode("utf8"), len(courses))
total_courses += len(courses)
output_dir = os.path.join(os.path.dirname(__file__), 'bupt')
if not os.path.exists(output_dir):
os.makedirs(output_dir)
if courses != []:
with open(os.path.join(output_dir, str(i) + '.yaml'), 'w') as yaml_file:
yaml_file.write(pretty_format(courses))
print "Done! Totally exported {0} courses.".format(total_courses)
def grab_all(self):
# self._local_setup()
# self._login()
self._fake_login()
url_courses = self.url_prefix + "jxrw_zd.aspx?xh=" + self.username
'''get viewstate'''
r_viewstate = requests.get(url_courses, cookies=self.cookies)
result = re.search('<input type="hidden" name="__VIEWSTATE" value="(.+)" />', r_viewstate.content)
viewstate = result.group(1)
print "Get viewstate: done."
'''parser, start.'''
''' - get colleges'''
strainer_colleges = SoupStrainer("select", id="ddlXY")
soup_colleges = BeautifulSoup(r_viewstate.content.decode(self.charset), parse_only=strainer_colleges)
colleges = [option['value'] for option in soup_colleges.select("option") if option['value']]
pretty_print(colleges)
print "{} colleges.".format(len(colleges))
''' - iter colleges'''
total_courses = 0
for i, college in enumerate(colleges):
'''get courses'''
data = {
'__EVENTTARGET': "",
'__EVENTARGUMENT': "",
'__VIEWSTATE': viewstate,
'ddlXN': "2012-2013",
'ddlXQ': "1",
'ddlXY': college.encode(self.charset),
'ddlZY': "",
'ddlKC': "",
'btnFilter': u' 查 询 '.encode(self.charset),
}
r_courses = requests.post(url_courses, data=data, cookies=self.cookies)
content = r_courses.content.decode(self.charset)
strainer_courses = SoupStrainer("table", id="DBGrid")
soup_courses = BeautifulSoup(content, parse_only=strainer_courses)
rows = soup_courses.select("tr")
courses = []
for r in rows:
if r.has_key('class') and r['class'] == ["datagridhead"]:
continue
cols = r.select("td")
semester_text = cols[0].get_text(strip=True)
teacher = cols[7].get_text(strip=True).replace('/', ',')
time_texts = map(string.strip, cols[8].get_text().split(';'))
locations = map(string.strip, cols[9].get_text().split(';'))
lessons = self.get_lessons(time_texts, locations, semester_text)
course = {
'original_id': cols[3].get_text(strip=True),
'name': cols[4].get_text(strip=True),
'credit': float(cols[6].get_text(strip=True)),
'teacher': teacher,
'lessons': lessons,
}
courses.append(course)
print "#{} {}: {} courses.".format(i, college.encode("utf8"), len(courses))
total_courses += len(courses)
output_dir = os.path.join(os.path.dirname(__file__), 'zju')
if not os.path.exists(output_dir):
os.makedirs(output_dir)
with open(os.path.join(output_dir, str(i) + '.yaml'), 'w') as yaml_file:
yaml_file.write(pretty_format(courses))
# with open(os.path.join(output_dir, str(i) + '.html'), 'w') as html_file:
# html_file.write(soup_courses.prettify().encode("utf8"))
print "Done! Totally exported {} courses.".format(total_courses)