def __init__(self, split=True, old=True, edxproblem=True, courseinfo=True, edxvideo=True, verbose=False):
'''
Get interface to modulestore backup.
Note: This class presumes modulestore was recently loaded to mongod.
Class also presumes that mongod is running on localhost:27017.
'''
# FIXME: don't presume modulestore is recently loaded and running
self.msdb = mng.MongoClient().modulestore
if verbose:
self.setupLogging(logging.INFO, logFile=None)
else:
self.setupLogging(logging.WARN, logFile=None)
# Need to handle Split and Old modulestore cases
self.split = split
self.old = old
# Switch for updating EdxProblem and CourseInfo separately (useful for testing)
self.update_EP = edxproblem
self.update_CI = courseinfo
self.update_EV = edxvideo
# Initialize MySQL connection from config file
home = os.path.expanduser('~')
dbFile = home + "/.ssh/mysql_user"
if not os.path.isfile(dbFile):
sys.exit("MySQL user credentials not found: " + dbFile)
dbuser = None #@UnusedVariable
dbpass = None #@UnusedVariable
with open(dbFile, 'r') as f:
dbuser = f.readline().rstrip()
dbpass = f.readline().rstrip()
MySQLDB.__init__(self, db="Edx", user=dbuser, passwd=dbpass)
def __init__(self, split=True, old=True, edxproblem=True, courseinfo=True, edxvideo=True):
'''
Get interface to modulestore backup.
Note: This class presumes modulestore was recently loaded to mongod.
Class also presumes that mongod is running on localhost:27017.
'''
# FIXME: don't presume modulestore is recently loaded and running
self.msdb = mng.MongoClient().modulestore
# Need to handle Split and Old modulestore cases
self.split = split
self.old = old
# Switch for updating EdxProblem and CourseInfo separately (useful for testing)
self.update_EP = edxproblem
self.update_CI = courseinfo
self.update_EV = edxvideo
# Initialize MySQL connection from config file
home = os.path.expanduser('~')
dbFile = home + "/.ssh/mysql_user"
if not os.path.isfile(dbFile):
sys.exit("MySQL user credentials not found: " + dbFile)
dbuser = None
dbpass = None
with open(dbFile, 'r') as f:
dbuser = f.readline().rstrip()
dbpass = f.readline().rstrip()
MySQLDB.__init__(self, db="Edx", user=dbuser, passwd=dbpass)
def __init__(self, outdir=""):
'''
Initializes table export tool with db credentials from .ssh
directory. Will write outfiles to current working directory
unless otherwise specified.
'''
# Helper function for making required directories
def ensureExists(dpath):
if not os.path.exists(dpath):
os.mkdir(dpath)
# Configure export logging
n = dt.now()
logDir = os.getcwd()+'/logs/'
ensureExists(logDir)
logging.basicConfig(filename="logs/TableExport_%d-%d-%d_%s.log"
% (n.year, n.month, n.day, n.strftime('%I:%M%p')),
level=logging.INFO)
# Set write directory for outfiles.
self.writeDir = os.getcwd()+'/'+outdir+'/'
ensureExists(self.writeDir)
# Get MySQL database user credentials from .ssh directory
home = os.path.expanduser("~")
dbFile = home + "/.ssh/mysql_user"
if not os.path.isfile(dbFile):
sys.exit("MySQL user credentials not found @ %s." % dbFile)
dbuser, dbpass = None, None
with open(dbFile, 'r') as f:
dbuser, dbpass = f.readline().rstrip(), f.readline().rstrip()
# Read in table lookup from .ssh directory.
tblFile = home + "/.ssh/ds_table_lookup.cfg"
if not os.path.isfile(tblFile):
sys.exit("Table lookup file not found @ %s." % tblFile)
self.tableLookup = dict()
with open(tblFile, 'r') as f:
for line in f:
db, tbl = line.split('.')
self.tableLookup[tbl] = line
# Initialize MySQL database connection
MySQLDB.__init__(self, user=dbuser, passwd=dbpass)
logging.info("Connected to database.")
def __init__(self):
mysql_dbhost='localhost'
#mysql_user='******'#getpass.getuser()
mysql_user=getpass.getuser()
mysql_db='EdxForum'
mysql_passwd=self.getMysqlPasswd()
self.type='all'
print('got passwd from file %s'%(mysql_passwd))
self.db=MySQLDB('127.0.0.1',3306,mysql_user,mysql_passwd,'EdxForum')
def __init__(self):
'''
Initializes extractor object with credentials from .ssh directory.
Set log file directory.
'''
home = expanduser("~")
userFile = home + '/.ssh/qualtrics_user'
tokenFile = home + '/.ssh/qualtrics_token'
dbFile = home + "/.ssh/mysql_user"
if os.path.isfile(userFile) == False:
sys.exit("User file not found: " + userFile)
if os.path.isfile(tokenFile) == False:
sys.exit("Token file not found: " + tokenFile)
if os.path.isfile(dbFile) == False:
sys.exit("MySQL user credentials not found: " + dbFile)
self.apiuser = None
self.apitoken = None
dbuser = None #@UnusedVariable
dbpass = None #@UnusedVariable
with open(userFile, 'r') as f:
self.apiuser = f.readline().rstrip()
with open(tokenFile, 'r') as f:
self.apitoken = f.readline().rstrip()
with open(dbFile, 'r') as f:
dbuser = f.readline().rstrip()
dbpass = f.readline().rstrip()
logging.basicConfig(
filename="EdxQualtricsETL_%d%d%d_%d%d.log" %
(dt.datetime.today().year, dt.datetime.today().month,
dt.datetime.today().day, dt.datetime.now().hour,
dt.datetime.now().minute),
level=logging.INFO)
self.lookup = IpCountryDict()
#************
MySQLDB.__init__(self, db="EdxQualtrics", user=dbuser, passwd=dbpass)
def __init__(self):
'''
Initializes extractor object with credentials from .ssh directory.
Set log file directory.
'''
home = expanduser("~")
userFile = home + '/.ssh/qualtrics_user'
tokenFile = home + '/.ssh/qualtrics_token'
dbFile = home + "/.ssh/mysql_user"
if os.path.isfile(userFile) == False:
sys.exit("User file not found: " + userFile)
if os.path.isfile(tokenFile) == False:
sys.exit("Token file not found: " + tokenFile)
if os.path.isfile(dbFile) == False:
sys.exit("MySQL user credentials not found: " + dbFile)
self.apiuser = None
self.apitoken = None
dbuser = None
dbpass = None
with open(userFile, 'r') as f:
self.apiuser = f.readline().rstrip()
with open(tokenFile, 'r') as f:
self.apitoken = f.readline().rstrip()
with open(dbFile, 'r') as f:
dbuser = f.readline().rstrip()
dbpass = f.readline().rstrip()
logging.basicConfig(filename="EdxQualtricsETL_%d%d%d_%d%d.log" % (dt.datetime.today().year, dt.datetime.today().month, dt.datetime.today().day, dt.datetime.now().hour, dt.datetime.now().minute),
level=logging.INFO)
self.lookup = IpCountryDict()
MySQLDB.__init__(self, db="EdxQualtrics", user=dbuser, passwd=dbpass)
with open(tokenFile, 'r') as f:
token = f.readline()
return uid.strip(), token.strip()
_User,_Token = getUserPwd()
#********
# print _User, _Token
#********
user,pwd = getMysqlUserPwd()
#db=MySQLDB('127.0.0.1',3306,'root','','EdxQualtrics')
# Now connect to the qualtrics db; NOTE the
# db EdxQualtrics must be present:
db=MySQLDB('127.0.0.1',3306,user,pwd,'EdxQualtrics')
initDatabaseIfNeeded()
"""
{u'Meta': {u'Status': u'Success', u'Debug': u''}, u'Result': {u'Surveys': [{u'SurveyCreationDate': u'2014-10-27 15:08:19', u'responses': u'3', u'UserFirstName': u'jagadish', u'SurveyType': u'SV', u'LastModified': u'2014-11-04 12:50:51', u'LastActivated': u'2014-10-27 15:21:36', u'SurveyName': u'cs140-feedback', u'UserLastName': u'venkatraman', u'SurveyID': u'SV_6YegHjmjmngyhVz', u'SurveyStatus': u'Active', u'CreatorID': u'UR_elipjMDGVTxcjZz', u'SurveyStartDate': u'0000-00-00 00:00:00', u'SurveyOwnerID': u'UR_elipjMDGVTxcjZz', u'SurveyExpirationDate': u'0000-00-00 00:00:00'}]}}
Returns all surveys for a particular user, token pair
"""
def getSurveysForUser (User, Token):
url='https://stanforduniversity.qualtrics.com/WRAPI/ControlPanel/api.php?API_SELECT=ControlPanel&Version=2.4&Request=getSurveys&User=%s&Token=%s&Format=JSON&JSONPrettyPrint=1'%(User,Token)
#********
# print url
#********
d=None
try:
d=urllib2.urlopen(url).read()
class PostGetter:
'''
Initialize the class with the appropriate mysql instance.
Initialize the DB instance to point to EdxForum.contents.
'''
def __init__(self):
mysql_dbhost='localhost'
#mysql_user='******'#getpass.getuser()
mysql_user=getpass.getuser()
mysql_db='EdxForum'
mysql_passwd=self.getMysqlPasswd()
self.type='all'
print('got passwd from file %s'%(mysql_passwd))
self.db=MySQLDB('127.0.0.1',3306,mysql_user,mysql_passwd,'EdxForum')
'''
Used only for testing. Test password provided.
'''
def getmysqlpasswd(self):
return '5PinkPenguines'
'''
Password is typically stored in ~user_name/.ssh/mysql
'''
def getMysqlPasswd(self):
homeDir=os.path.expanduser('~'+getpass.getuser())
f_name=homeDir+'/.ssh/mysql'
with open(f_name, 'r') as f:
password = f.readline().strip()
print 'password got from file is %s'%(password)
return password
def getAllDataForCourseAsString(self,course):
courseData=''
query_string='select body from contents where course_display_name=\'%s\''%(course)
for row in self.db.query(query_string):
courseData += row[0]
return courseData
def getMinWeekForCourse(self, course):
query_string='select min(YEARWEEK(created_at)) from contents where course_display_name=\'%s\''%(course)
minval=0
for row in self.db.query(query_string):
minval=row[0]
return int(minval)
def getMaxWeekForCourse(self, course):
query_string='select max(YEARWEEK(created_at)) from contents where course_display_name=\'%s\''%(course)
maxval=0
for row in self.db.query(query_string):
maxval=row[0]
return int(maxval)
def getWeeklyDataForCourseAsString(self,course, week):
courseData=''
query_string='select body from contents where course_display_name=\'%s\' and YEARWEEK(created_at)=%s'%(course,week)
for row in self.db.query(query_string):
courseData += row[0]
return courseData
def getNumPosts(self, course, week):
query_string='select count(*) from contents where course_display_name=\'%s\' and YEARWEEK(created_at)=%s'%(course,week)
numPosts=0
for row in self.db.query(query_string):
numPosts=row[0]
return int(numPosts)
#qq=qq+' limit 50'
#for c in mydb.query("select body from EdxForum.contents where body like '%:)%'"):
for c in mydb.query(qq):
obj=[];
s=c[0]
try:
s=s.encode('ascii', 'ignore')
except :
s='z'
if(s!='z'):
t=(s,code)
all_data.append(s)
categories.append(code)
mydb=MySQLDB('127.0.0.1',3306,'jagadish','5PinkPenguines','EdxForum')
mydb.execute('SET NAMES utf8;');
mydb.execute('SET CHARACTER SET utf8;');
mydb.execute('SET character_set_connection=utf8;');
if (len(sys.argv) !=2):
print 'usage: python confusion.py test/train'
sys.exit (1)
all_data=[]
codes=[]
happy=[]
qq="select body from EdxForum.contents where body like '%:)%' or body like '%:-)%' "