class PiazzaScraper:
# class_code refers to the k12qbt838di4xt in https://piazza.com/class/k12qbt838di4xt
def __init__(self, class_code, username, password):
self.piazza = Piazza()
self.piazza.user_login(username, password)
self.piazza_class = self.piazza.network(class_code)
def get_all_posts(self):
# get_all_posts returns a list of dictionaries. The dictionaries contains information of the thread that resides
# in the Piazza classroom
feed = self.piazza_class.get_feed(limit=999999, offset=0)
thread_ids = [post['id'] for post in feed["feed"]]
posts = []
for thread_id in thread_ids:
posts.append(self.piazza_class.get_post(thread_id))
return posts
def get_post(self, thread_id):
# get_post allows the user to get the dictionary that contains information of a single thread
return self.piazza_class.get_post(thread_id)
def get_users(self):
# get_users allows the user to get a list of dictionaries that contains the profile of a Piazza user
return self.piazza_class.get_all_users()
def get_user(self, user_id):
# get_users allows the user to get a dictionary that contains the profile of a Piazza user
return self.piazza_class.get_users(user_id)
def setUp(self):
username = config.username
password = config.password
class_id = 'idj5lp6gixc6xn'
p = Piazza()
p.user_login(email=username, password=password)
self.cis121 = p.network(class_id)
def create_piazza_post(title: str, content: str, folder: str) -> int:
"""Create a piazza post from the given parameters and return the post's CID."""
EMAIL = Config.get_global().piazza_email
PWD = Config.get_global().piazza_password
ID = Config.get_global().piazza_id
p = Piazza()
p.user_login(EMAIL, PWD)
post = p.network(ID)
while True:
try:
result = post.create_post(
title=title,
content=content,
nid=ID,
folder=folder
)
except piazza_api.exceptions.RequestError as e:
str_e = str(e)
if "posting too quickly" in str_e:
# Piazza has posting time limits, grumble grumble
# "Sorry! It looks like you are posting too quickly--wait a second and try again."
# Try again
time.sleep(0.4)
continue
else:
# Don't handle this
raise e
# All done now!
break
return int(result['nr'])
def main(argv=sys.argv[1:]):
argument_parser = argparse.ArgumentParser()
# argument_parser.add_argument('--username', type=str,
# default='PiazzaBot', help='the bot\'s Keybase username')
args = argument_parser.parse_args(argv)
logger.info('keybase_piazza_bot starting')
event_loop = asyncio.get_event_loop()
piazza = Piazza()
piazza.user_login() # login prompt
cse220 = piazza.network(PIAZZA_NETWORK_ID)
keybase_bot_handler = KeybaseBotHandler(piazza, cse220)
keybase_bot = Bot(
handler=keybase_bot_handler,
loop=event_loop,
# username=args.username,
)
future = keybase_bot.start({})
event_loop.run_until_complete(future)
event_loop.close()
logger.info('keybase_piazza_bot exiting')
def perform_action(action,
course,
as_staff=False,
is_test=None,
kwargs=None):
with connect_db() as db:
if as_staff:
user, pw = db(
"SELECT staff_user, staff_pw FROM piazza_config WHERE course = (%s)",
[course],
).fetchone()
else:
user, pw = db(
"SELECT student_user, student_pw FROM piazza_config WHERE course = (%s)",
[course],
).fetchone()
if is_test:
(course_id, ) = db(
"SELECT test_course_id FROM piazza_config WHERE course = (%s)",
[course],
).fetchone()
else:
(course_id, ) = db(
"SELECT course_id FROM piazza_config WHERE course = (%s)",
[course]).fetchone()
p = Piazza()
p.user_login(user, pw)
course = p.network(course_id)
if kwargs is None:
kwargs = {}
try:
return getattr(course, action)(**kwargs)
except Exception as e:
return str(e), 400
def post_aggregate(email, password, coursecode):
# User credentials
piazza = Piazza()
# Login
piazza.user_login(email, password)
# get course code to hash dictionary
coursetohash = course_hash(email, password)
# get classroom object using the hash
classroom = piazza.network(coursetohash[coursecode])
# go through all the posts, aggregate them in a data-structure
postquestions = []
# board_posts type is generator: cannot access memory in the lazy list
board_posts = classroom.iter_all_posts(limit=const_limit)
# iterate through board posts
for post in board_posts:
# get rid of html tags in question in post
question_string = strip_tags(post["history"][0]["content"])
# append to questions array
postquestions.append(question_string)
return postquestions
def checkPiazza():
client = pymongo.MongoClient(pizzapizzasecret.dbsecret)
db = client.get_database('piazza-posts')
table = db.posts
existingPosts = []
query = table.find()
print(query)
output = {}
i = 0
for x in query:
output[i] = x
output[i].pop('_id')
existingPosts.append(output[i]['ID'])
i += 1
print(existingPosts)
p = Piazza()
p.user_login(pizzapizzasecret.email, pizzapizzasecret.password)
ds = p.network(pizzapizzasecret.net)
#print("ds", ds.iter_all_posts())
posts = ds.iter_all_posts()
for post in posts:
#print("post", post)
if post['history'][0]['subject'] not in existingPosts:
payload = post['history'][0]['subject']
print(payload)
queryObject = {
'ID': payload,
}
queryMongo = table.insert_one(queryObject)
sendPayload(post)
else:
print("piazza channel up to date")
return "success"
class PiazzaWrapper:
def __init__(self, course_id='xxxxxxxxxxx'):
self.p = Piazza()
email_id = input("Enter your Piazza email ID : ")
password = getpass('Piazza Password:'******'feed']:
if post['type'] == 'question':
# print(post['nr'])
# print(post['content_snipet'])
time = post['log'][0]['t']
time = datetime.datetime.strptime(time[:-1],
"%Y-%m-%dT%H:%M:%S")
if time.date() == today.date():
count += 1
if 'has_i' in post.keys():
count_i += 1
elif 'has_s' in post.keys():
count_s += 1
else:
count_unanswered += 1
unanswered_posts.append(post['nr'])
# print(time)
return count, count_i, count_s, count_unanswered, unanswered_posts
def get_unanswered_followup(self):
posts = self.comp_photo19.iter_all_posts()
count = 0
for post in posts:
cid = post['nr']
content = self.comp_photo19.get_post(cid)
count += self.traverse_content_tree(content)
return count
def traverse_content_tree(self, content):
count = 0
if 'children' in content.keys():
if len(content['children']) > 0:
for content_children in content['children']:
count += self.traverse_content_tree(content_children)
if 'no_answer' in content_children.keys():
count += content_children['no_answer']
return count
def get_count_today(self):
posts = self.comp_photo19.get_feed(100, 0)
count, count_i, count_s, count_unanswered, unanswered_posts = self.count_posts(
posts, datetime.datetime.today())
return count, count_i, count_s, count_unanswered, unanswered_posts
def __init__(self, course_code=config.eecs281):
self.piazza = Piazza()
self.piazza.user_login(config.creds['email'], config.creds['password'])
self.course = self.piazza.network(course_code)
# rpc api to post notes
self.piazza_rpc = PiazzaRPC(config.class_code)
self.piazza_rpc.user_login(config.creds['email'], config.creds['password'])
def __init__(self, days_refresh=10):
self.piazza = Piazza()
self.piazza.user_login(email=Config.username, password=Config.password)
self.course = self.piazza.network(Config.courseid)
self.engine = create_engine('sqlite:///test.db', echo=False)
self.Session = sessionmaker(bind=self.engine)
self.session = self.Session()
self.days = days_refresh
def __init__(self):
self.p = Piazza()
self.p.user_login(USER_NAME, USER_PASS)
self.uid = self.p.get_user_profile()['user_id']
# classes = self.p.get_user_classes()
self.si206 = self.p.network("jlp6m1ynp9713y")
def __init__(self, course_id='xxxxxxxxxxx'):
self.p = Piazza()
email_id = input("Enter your Piazza email ID : ")
password = getpass('Piazza Password:')
# course_id = input("Enter your Course ID : ")
self.p.user_login(email_id, password)
user_profile = self.p.get_user_profile()
self.comp_photo19 = self.p.network(course_id)
def main():
p = Piazza()
p.user_login()
classId = input("Enter your class id")
classObj = p.network(classId)
postId = input("Enter post number")
posts = classObj.get_post(postId)
all_users = classObj.get_all_users()
generate_csv(all_users, posts)
def __init__(self, bot, EMAIL, PASSWORD, TARGET, NAME, ID):
self.bot = bot
self.p = Piazza()
self.p.user_login(email=EMAIL, password=PASSWORD)
self.name = NAME
self._nid = ID
self.cls = self.p.network(self._nid)
self.url = f'https://piazza.com/class/{self._nid}?cid='
self.target_channel = TARGET # bot-commands channel
self.sendUpdate.start()
def __init__(self, username, password, course_id):
self.__username = username
self.__password = password
self.__course_id = course_id
self.__path = './data/' + course_id + '.csv'
self.__h = html2text.HTML2Text()
self.p = Piazza()
self.__login()
self.__load_seen_posts()
def main():
"""Get the cli args and start tracking."""
args = parse_arguments()
piazza = Piazza()
piazza.user_login(args.email, args.password)
# Create/load tinydb for the users and posts
userdb = TinyDB(f'{args.class_id}.json', default_table="users")
postdb = TinyDB(f'{args.class_id}.json', default_table="posts")
while True:
track(piazza, args.class_id, userdb, postdb)
time.sleep(2)
def __init__(self, NAME, ID, EMAIL, PASSWORD, GUILD, FETCH_MAX=55, FETCH_MIN=30):
self.name = NAME
self.nid = ID
self._guild = GUILD
self._channels = []
self.url = f"https://piazza.com/class/{self.nid}"
self.p = Piazza()
self.p.user_login(email=EMAIL, password=PASSWORD)
self.network = self.p.network(self.nid)
self.max = FETCH_MAX
self.min = FETCH_MIN
def login(email, password):
piazza = Piazza()
piazza.user_login(email, password)
class_dictionary = piazza.get_user_classes()
return class_dictionary
courseToHash = {}
#print(len(class_dictionary))
for i in range(len(class_dictionary)):
courseToHash.update(
{class_dictionary[i]['num']: class_dictionary[i]['nid']})
def __init__(self, bot, TARGET, CLASS, ID, EMAIL=None, PASSWORD=None):
self.bot = bot
self.p = Piazza()
self.p.user_login(email=EMAIL, password=PASSWORD)
self._nid = ID
self.classname = CLASS
self.cls = self.p.network(self._nid)
self.url = f'https://piazza.com/class/{self._nid}?cid='
self.target_channel = TARGET # bot-commands channel
self.sendUpdate.start(
) # this error is ok, was written this way in the docs
def main():
parser = argparse.ArgumentParser(description='Retrieve Piazza posts and answers in JSON format.')
parser.add_argument('--networkid', '-n', help='piazza network ID', required=True)
parser.add_argument('--output', '-o', help='output filename', required=True)
args = parser.parse_args()
piazza = Piazza()
print('Login to Piazza:')
piazza.user_login()
network = piazza.network(args.networkid)
write_all_posts(network, args.output)
def _connect_to_piazza(self, piazza_credentials):
"""Connect to Piazza"""
_piazza = Piazza()
_piazza.user_login(email=piazza_credentials['piazza_email'],
password=piazza_credentials['piazza_password'])
self._myclass = _piazza.network(piazza_credentials['piazza_network'])
# Get list of cid's from feed
self._feed = self._myclass.get_feed(limit=999999, offset=0)
self._instructor_ids = [
user['id'] for user in self._myclass.get_all_users()
if user['admin'] == True
]
class Bot:
def __init__(self, course_code=config.eecs281):
self.piazza = Piazza()
self.piazza.user_login(config.creds['email'], config.creds['password'])
self.course = self.piazza.network(course_code)
# rpc api to post notes
self.piazza_rpc = PiazzaRPC(config.class_code)
self.piazza_rpc.user_login(config.creds['email'], config.creds['password'])
def get_all_posts_json(self):
documents = []
posts = []
file_name = '{0}.txt'.format(config.class_code)
if not os.path.isfile(file_name):
data = self.course.iter_all_posts(limit=INF)
for post in data:
print('downloading post {0}'.format(post['nr']))
documents.append(post)
posts.append(Post(post))
obj = open(file_name, 'wb')
json.dump(documents, obj)
else:
obj = open(file_name, 'r')
data = json.load(obj)
for post in data:
posts.append(Post(post))
return posts
def get_all_posts(self, start_id=0, limit=100):
documents = []
feed = self.course.get_feed()
ids = [post['nr'] for post in feed['feed']]
for post_id in ids:
if post_id > start_id:
print('downloading post {0}'.format(post_id))
post_json = self.course.get_post(post_id)
documents.append(Post(post_json))
return documents
def get_post(self, id):
return Post(self.course.get_post(id))
def create_post(self, subject, body, folder=['hw1']):
params = {'type':'note','subject':subject, 'content':body, 'folders':folder}
self.piazza_rpc.content_create(params)
def create_answer(self, post_id, content):
params = { 'cid': post_id, 'type': 'i_answer', 'content': content, 'revision': 0}
return self.piazza_rpc.content_instructor_answer(params)
def __init__(self):
self.p = Piazza()
self.p.user_login(PIAZZA_USER, PIAZZA_PASS)
self.uid = self.p.get_user_profile()['user_id']
classes = self.p.get_user_classes()
self.classes = []
print 'Now watching posts for the following {0} classes:'.format(len(classes))
for c in classes:
print '{0} ({1}, {2})'.format(c['num'], c['name'], c['term'])
self.classes.append(self.p.network(c['nid']))
self.filter = UnreadFilter()
def main():
parser = argparse.ArgumentParser(description='Process user input for piazza queries')
parser.add_argument('-q', '--query', nargs="+")
parser.add_argument('-t', '--tags', nargs="+")
parser.add_argument('-r', '--range', nargs=2)
parser.add_argument('-i', '--instructor-only', action='store_true')
parser.add_argument('-p', '--posts', action='store_true')
parser.add_argument('-l', '--force-login', action='store_true')
args = parser.parse_args()
if (args.query is None):
raise(InputError("Query not given!"))
queryObj = QueryObj()
queryObj.add_query(args.query)
queryObj.add_tags(args.tags)
queryObj.add_time_range(args.range)
queryObj.bool_inst_notes(args.instructor_only)
queryObj.bool_pinned(args.posts)
loginfile = os.path.expanduser("~") + "/.pizza"
if not args.force_login:
try:
pkl = pickle.load(open(loginfile,"rb"))
data = {'email', pkl['email'], 'password': pkl['password'].decode('rot13')}
except IOError:
email = raw_input('Piazza Email: ')
password = getpass.getpass()
data = {'email': email, 'password': password}
pkl = {'email': email, 'password': password.encode('rot13')}
pickle.dump(pkl, open(loginfile, "wb"))
piazza = Piazza()
piazza.user_login(data['email'], data['password'])
user_status = piazza.get_user_status()
classes = user_status['networks']
classes = sorted(classes, key=lambda k: k['status'])
# list classes
print("Choose a Class")
counter = 1
for c in classes:
info = c['name']
if c['status'] == 'inactive':
info = '(inactive) ' + info
print '{0:2d}: {1:s}'.format(counter, info)
counter = counter + 1
def __init__(self,
user,
password,
class_id,
corpus=None,
corpus_embeddings=None,
default_bert=True):
self.p = Piazza()
self.p.user_login(user, password)
self.class_id = class_id
self.user_profile = self.p.get_user_profile()
self.network = self.p.network(class_id)
self.DB_manger = MongoDBManger()
self.bert = BertSemanticSearch(corpus, corpus_embeddings, default_bert)
self.parallel_cid_list = []
def course_list(email, password):
# User credentials
piazza = Piazza()
# Login
piazza.user_login(email, password)
# create class dictionary from user class details: we are interested in the "num" key
class_dictionary = piazza.get_user_classes()
course_code_list = []
for index in range((len(class_dictionary))):
course_code_list.append(class_dictionary[index]['num'])
return course_code_list
def course_hash(email, password):
# User credentials
piazza = Piazza()
# Login
piazza.user_login(email, password)
class_dictionary = piazza.get_user_classes()
# dictionary for 'course code: hash_id'
course_to_hash = {}
# print(len(class_dictionary))
for i in range(len(class_dictionary)):
course_to_hash.update(
{class_dictionary[i]['num']: class_dictionary[i]['nid']})
return course_to_hash
def __init__(self,days_refresh=10):
self.piazza = Piazza()
self.piazza.user_login(email=Config.username, password=Config.password)
self.course = self.piazza.network(Config.courseid)
self.engine = create_engine('sqlite:///test.db', echo=False)
self.Session = sessionmaker(bind=self.engine)
self.session = self.Session()
self.days = days_refresh
class PiazzaBot():
def __init__(self):
self.p = Piazza()
self.p.user_login(USER_NAME, USER_PASS)
self.uid = self.p.get_user_profile()['user_id']
# classes = self.p.get_user_classes()
self.si206 = self.p.network("jlp6m1ynp9713y")
def do_bot(self):
posts = self.si206.search_feed("lecture")
for post in posts:
id = post[u'id']
print(self.si206.get_post(id)[u'history'][0][u'content'])
print("\n\n")
def comment_post_aggregate(email, password, coursecode):
# User credentials
piazza = Piazza()
# Login
piazza.user_login(email, password)
# get course code to hash dictionary
coursetohash = course_hash(email, password)
# get classroom object using the hash
classroom = piazza.network(coursetohash[coursecode])
# go through all the posts, aggregate them in a data-structure
postquestions = []
postanswers = []
# board_posts type is generator: cannot access memory in the lazy list
board_posts = classroom.iter_all_posts(limit=const_limit)
# iterate through board posts
for post in board_posts:
# get rid of html tags in question in post
question_string = strip_tags(post["history"][0]["content"])
# append to questions array
postquestions.append(question_string)
# checks if there's an answer associated to the question
if "children" in post.keys(
) and post["children"] and "history" in post["children"][0]:
# for all answers in a single post (iterate)
for answer_index in range(len(post["children"][0]["history"])):
# get rid of html tags for answers in the post, and check if the entry is a string
if type(post["children"][0]["history"][answer_index]
["content"]) == str:
answer_string = strip_tags(post["children"][0]["history"]
[answer_index]["content"])
# append to answers array
postanswers.append(answer_string)
# print(postQuestions + postAnswers)
return postquestions + postanswers
def try_login():
'''
Tries to login to Piazza using the username/password specified in
config.yml
'''
yaml_config = get_config()
print 'Attempting to login to Piazza as %s.....' % yaml_config['username'],
piazza = Piazza()
try:
piazza.user_login(email=yaml_config['username'],
password=yaml_config['password'])
except BaseException:
print 'failed, check username/password'
print 'success'
return piazza
def __init__(self,
name: str,
nid: str,
email: str,
password: str,
guild: discord.Guild,
fetch_max: int = 55,
fetch_min: int = 30):
self._name = name
self.nid = nid
self._guild = guild
self._channels = []
self.url = f"https://piazza.com/class/{self.nid}"
self.p = Piazza()
self.p.user_login(email=email, password=password)
self.network = self.p.network(self.nid)
self.fetch_max = fetch_max
self.fetch_min = fetch_min
course = course.strip()
pid = other_piazza_ids[other_piazza_names.index(course)] if course else piazza_id
posts.add((pid, post, followup))
if len(posts) > 0:
for pid, post, followup in posts:
post_link(channel, user, post, pid, thread, followup)
elif bot_id in text:
process_bot_call(channel, user, text, thread)
if __name__ == "__main__":
from bot_config import *
bot_id = None
sc = SlackClient(token)
p = Piazza()
p.user_login(email=piazza_email, password=piazza_password)
if sc.rtm_connect():
username = sc.server.username
for users in sc.server.login_data['users']:
if users['name'] == username:
bot_id = users['id']
break
if not bot_id:
raise Error("Could not find bot")
while True:
results = sc.rtm_read()
for result in results:
try:
if result['type'] == 'message':
def get(self, course_id, operation):
if operation == 'get':
cursor = g.db.execute('''SELECT course_name, piazza_cid
FROM courses
WHERE course_id=(?)''',
[int(course_id)])
row = cursor.fetchone()
if row is None:
raise InvalidUsage('Given course does not exist')
else:
course_name_str = row['course_name']
piazza_id_str = row['piazza_cid']
return jsonify(message='Returning course info', course_id=course_id, course_name=course_name_str, piazza_cid=piazza_id_str)
# @deprecated: use 'get' end point instead
if operation == 'getname':
cursor = g.db.execute('''SELECT course_name
FROM courses
WHERE course_id=(?)''',
[int(course_id)])
course_name_row = cursor.fetchone()
if course_name_row is None:
raise InvalidUsage('Given course does not have a name')
else:
course_name_str = course_name_row['course_name']
return jsonify(message='Returning name for course', course_id=course_id, course_name=course_name_str)
# @deprecated: use 'get' end point instead
elif operation == 'getpiazza':
cursor = g.db.execute('''SELECT piazza_cid
FROM courses
WHERE course_id=(?)''',
[int(course_id)])
piazza_id_row = cursor.fetchone()
if piazza_id_row is None:
raise InvalidUsage('Given course does not have a Piazza ID')
else:
piazza_id_str = piazza_id_row['piazza_cid']
return jsonify(message='Returning piazza ID for course',
course_id=course_id, piazza_cid=piazza_id_str)
elif operation == 'getpiazzaposts':
cursor = g.db.execute('''SELECT piazza_cid
FROM courses
WHERE course_id=(?)''',
[int(course_id)])
piazza_id_row = cursor.fetchone()
if piazza_id_row is None:
raise InvalidUsage('Given course does not have a Piazza ID')
else:
piazza_id_str = piazza_id_row['piazza_cid']
p = Piazza()
try:
with open(USER_FILE, 'r') as fname:
lines = fname.read().split('\n')
p.user_login(email=lines[0], password=lines[1])
piazza_class = p.network(piazza_id_str)
except IndexError:
raise InvalidUsage('Piazza credentials are improperly formatted',
status_code=501)
except IOError:
raise InvalidUsage('Unable to find piazza credentials',
status_code=500)
except AuthenticationError:
raise InvalidUsage('Invalid pizza credentials')
# Attempt to pull a single post. If it doesn't work, we should
# throw an error
try:
for k in piazza_class.iter_all_posts(limit=1):
single_post = k
except RequestError:
raise InvalidUsage('Invalid piazza course ID',
status_code=500)
def get_posts():
for post in piazza_class.iter_all_posts():
yield json.dumps(post)
return Response(get_posts(), mimetype='application/json')
else:
raise InvalidUsage('Unknown operation type')
class MLStripper(HTMLParser):
def __init__(self):
self.reset()
self.fed = []
def handle_data(self, d):
self.fed.append(d)
def get_data(self):
return ''.join(self.fed)
def strip_tags(html):
s = MLStripper()
s.feed(html)
return s.get_data()
p = Piazza()
p.user_login('*****@*****.**', 'thomaslau')
f = open('userProfile.txt','w')
json.dump(p.get_user_profile(), f)
f.close()
rawUserData = open('userProfile.txt')
jsonUserData = json.load(rawUserData)
rawUserData.close()
masterPath = os.getcwd()
for i in jsonUserData["all_classes"]:
classConnection = p.network(i)
# coding: utf-8
# In[1]:
from piazza_api import Piazza
import bs4
# In[2]:
p = Piazza()
# In[5]:
p.user_login()
# In[6]:
spark = p.network("i9esrcg0gpf8k")
# In[46]:
out =""
failed= []
for i in range(1,spark.get_statistics()["total"]["questions"]):
try :
post = spark.get_post(i)
from piazza_api import Piazza
import sys
import logging
import json
logging.captureWarnings(True)
if len(sys.argv) > 2:
search_query = sys.argv[1]
else:
search_query = "Nothing"
p = Piazza()
password = ""
with open("password.txt") as f:
password = f.read()
p.user_login("*****@*****.**", password)
course = p.network("i4skbzt4mxk3ck")
data = {}
with open("test.txt", "w") as g:
for i in course.iter_all_posts(limit=100):
print i.get("history")[0]["subject"]
print "piazza.com/class/i4skbzt4mxk3ck?cid=%d" % i["nr"]
data[i.get("history")[0]["subject"]] = "piazza.com/class/i4skbzt4mxk3ck?cid=%d" % i["nr"]
json.dump(data, g)
class Scraper:
"""
Usage for Scraper:
>>> s = Scraper() # Initializes scraper/db connection
>>> s.get(10) #Fetches 10 posts and stores them
>>> s.print_topics() # Prints list of current topics/tags
>>> s.print_posts() # Prints all posts in DB
"""
def __init__(self,days_refresh=10):
self.piazza = Piazza()
self.piazza.user_login(email=Config.username, password=Config.password)
self.course = self.piazza.network(Config.courseid)
self.engine = create_engine('sqlite:///test.db', echo=False)
self.Session = sessionmaker(bind=self.engine)
self.session = self.Session()
self.days = days_refresh
def parse(self):
self.refetch_stats()
print("Finished getting stats")
self.delete_recent()
self.get_recent_posts()
def get_recent_posts(self):
"""
Starts the scraper, and stores a certain number of
posts to the database: this is the primary method
to be used in scraping
<Possible Error> Currently, if you run the scraper twice
it will duplicate -- must look into methods to fix that
(probably using ID)
Parameters:
limit - Number of posts to fetch
Returns:
None
"""
for _,post in enumerate(self.course.iter_all_posts()):
if not self.process_one(post):
return
print(_,post['history'][0]['subject'])
def process_one(self, post):
"""
Takes a post from the Piazza API,
converts it into a format ready for the database,
and stores it
"""
time = parse_time(post['created'])
duplicate = self.session.query(Post).filter(Post.time == time).first()
title = remove_tags(post['history'][0]['subject'])
body = remove_tags(post['history'][0]['content'])
views = post['unique_views']
favorites = post['num_favorites']
if duplicate is not None:
if post['bucket_name'] == 'Pinned':
return True
return False
sqlpost = Post(title, body, time, views, favorites)
# Adding Comments
for comment in process_all_children(post):
time = parse_time(comment['created'])
type = comment['type']
if 'history' not in comment:
subject = remove_tags(comment['subject'])
else:
subject = remove_tags(comment['history'][0]['content'])
sqlpost.comments.append(Comment(subject, time, type))
#Adding Tags
for tag in post['tags']:
sqlpost.tags.append(self.get_tag(tag))
#Saving to Database
self.session.add(sqlpost)
self.session.commit()
return True
def get_tag(self, name):
"""
Returns the topic if it exists, and otherwise creates it
"""
tag = self.session.query(Tag).filter(Tag.name == name).first()
if not tag:
tag = Tag(name)
self.session.add(tag)
self.session.commit()
return tag
def print_posts(self):
"""
Prints a list of all posts currently
in the database
"""
posts = self.session.query(Post).all()
for post in posts:
print(post)
for comment in post.children:
print("\t", comment)
def print_tags(self):
"""
Prints a list of topics currently registerd
"""
topics = self.session.query(Tag).all()
for topic in topics:
print(topic)
def delete_all(self):
self.session.query(Post).delete()
self.session.query(Comment).delete()
self.session.query(Tag).delete()
def delete_recent(self):
recent = datetime.now()-timedelta(days=self.days)
self.session.query(Post).filter(Post.time > recent).delete()
def refetch_stats(self):
mostrecent = s.session.query(DayStats).order_by(DayStats.day.desc()).first()
if mostrecent is not None:
delta = datetime.now()-mostrecent.day
if mostrecent is None or delta > timedelta(days=1):
stats = self.course.get_statistics()
self.session.query(DayStats).delete()
for daily in stats['daily']:
day = parse_day(daily['day'])
self.session.add(DayStats(daily['questions'],day,daily['users'],daily['posts']))
self.session.commit()
print("Finished updating statistics")
import json
from time import sleep
from piazza_api import Piazza
import config
username = config.username
password = config.password
class_id = 'is0q8vy2dsm6yx'
p = Piazza()
p.user_login(email=username, password=password)
cis121 = p.network(class_id)
# get the total number of posts
stats = cis121.get_statistics()
# get it using the daily posts in the statistics page
total_num_posts = sum(d['questions'] for d in stats['daily'])
# now load the "class database"
try:
with open(config.LOCAL_DATA_FILE_NAME, 'r') as data_file:
try:
current_data = json.load(data_file)
except ValueError: # empty file
current_data = {}
except IOError: #file didn't exist
current_data = {}
counter = 0
limit = 10
from piazza_api import Piazza
p=Piazza()
p.user_login('*****@*****.**','Cxy3020840!')
eece210 = p.network("ieby1xzit8r1ki")
post=eece210.get_post(29)
s=post['history'][0]
print s['content']
from piazza_api import Piazza
import json
import sys
p = Piazza()
p.user_login()
course = p.network(sys.argv[1])
mapSave = {}
posts = course.iter_all_posts(limit=100000000000)
for post in posts:
content = post["history"][0]["content"]
id = post["nr"]
print (id)
mapSave[id] = content
with open("posts183.json","wb") as f:
f.write(json.dumps(mapSave))
