def request_clickstream(self, export_type = "RESEARCH_EVENTING", anonymity_level = "HASHED_IDS_NO_PII",
statement_of_purpose = "Weekly backup of course data"):
if self.verbose:
print "Requesting clickstream data ({}) for period {} to {}".format(self.course_slug, self.interval[0], self.interval[1])
if self.log:
logging.info("Requesting clickstream data ({}) for period {} to {}".format(self.course_slug, self.interval[0], self.interval[1]))
# Construct request
er = ExportRequest(course_id=self.course_id, export_type=export_type, anonymity_level = anonymity_level,
statement_of_purpose = statement_of_purpose, interval=self.interval)
# Fire request
try:
ERM = api.post(er)[0]
except:
errors = ' '.join([str(e) for e in sys.exc_info()])
if self.log:
logging.error("Request failed ({}) with errors {}".format(self.course_slug, errors))
print errors
raise FailedRequest("Failed request")
if self.log:
logging.info("Request successfull ({})".format(self.course_slug))
# Add id to self
vals = ERM.to_json()
self.id_ = vals['id']
self.type_ = "CLICKSTREAM"
self.metadata = vals["metadata"]
self.schemaNames = "NONE"
def request_tables(args):
"""
Create and send a tables data export request with Coursera.
"""
export_request = ExportRequest.from_args(
course_id=args.course_id,
course_slug=args.course_slug,
partner_id=args.partner_id,
partner_short_name=args.partner_short_name,
group_id=args.group_id,
user_id_hashing=args.user_id_hashing,
statement_of_purpose=args.purpose,
export_type=EXPORT_TYPE_TABLES,
schema_names=args.schemas)
export_request_with_metadata = api.post(export_request)[0]
logging.info('Successfully created tables export request {id}.'.format(
id=export_request_with_metadata.id))
logging.debug('Request created with json body:\n{json}'.format(
json=json.dumps(export_request_with_metadata.to_json(), indent=2)))
def request_schemas(self,
export_type="RESEARCH_WITH_SCHEMAS",
anonymity_level="HASHED_IDS_NO_PII",
statement_of_purpose="Weekly backup of course data",
schema_names=[
"demographics", "users", "course_membership",
"course_progress", "feedback", "assessments",
"course_grades", "peer_assignments", "discussions",
"programming_assignments", "course_content"
]):
if self.log:
logging.info("Requesting table data ({})".format(self.course_slug))
# Construct request
er = ExportRequest(course_id=self.course_id,
export_type=export_type,
anonymity_level=anonymity_level,
statement_of_purpose=statement_of_purpose,
schema_names=schema_names)
# Fire request
try:
ERM = api.post(er)[0]
except: # Find out specific error
errors = ' '.join([str(e) for e in sys.exc_info()])
if self.log:
logging.error("Request failed ({}) with errors {}".format(
self.course_slug, errors))
print errors
raise FailedRequest("Failed request")
if self.log:
logging.info("Request successfull ({})".format(self.course_slug))
# Add info to self
vals = ERM.to_json()
self.id_ = vals['id']
self.type_ = "TABLES"
self.metadata = vals["metadata"]
self.schemaNames = vals["schemaNames"]
def request_clickstream(args):
"""
Create and send an clickstream data export request with Coursera. Only
available for data coordinators.
"""
export_request = ExportRequest.from_args(
course_id=args.course_id,
course_slug=args.course_slug,
partner_id=args.partner_id,
partner_short_name=args.partner_short_name,
group_id=args.group_id,
anonymity_level=ANONYMITY_LEVEL_COORDINATOR,
statement_of_purpose=args.purpose,
export_type=EXPORT_TYPE_CLICKSTREAM,
interval=args.interval,
ignore_existing=args.ignore_existing)
export_request_with_metadata = api.post(export_request)[0]
logging.info(
'Successfully created clickstream export request {id}.'.format(
id=export_request_with_metadata.id))
logging.debug('Request created with json body:\n{json}'.format(
json=json.dumps(export_request_with_metadata.to_json(), indent=2)))
def issue_requests(config_json):
range_first = config_json["fist_clickstream_date"] if config_json[
"first_time"] == "yes" else config_json["last_clickstream_date"]
range_last = str(date.today() - timedelta(
days=1)) if datetime.today().hour > 3 else str(date.today() -
timedelta(days=2))
print(range_first, range_last)
TABLES_ISSUE_REQUEST = ExportRequest.ExportRequest(
partner_id=int(config_json["eit_digital_id"]),
statement_of_purpose=config_json["purpose_for_requests"] +
config_json["tables_purpose"],
export_type=EXPORT_TYPE_TABLES,
anonymity_level=ANONYMITY_LEVEL_COORDINATOR,
schema_names=SCHEMA_NAMES)
CLICKSTREAM_ISSUE_REQUEST = ExportRequest.ExportRequest(
partner_id=int(config_json["eit_digital_id"]),
statement_of_purpose=config_json["purpose_for_requests"] +
config_json["clickstream_purpose"],
export_type=EXPORT_TYPE_CLICKSTREAM,
anonymity_level=ANONYMITY_LEVEL_COORDINATOR,
interval=[range_first, range_last])
registered_tables_request = None
registered_clickstream_request = None
exit_with_error = False
error_message = 'Error requesting the '
registered_tables_request, registered_clickstream_request = get_from_todays(
TABLES_ISSUE_REQUEST, CLICKSTREAM_ISSUE_REQUEST)
print registered_tables_request, registered_clickstream_request
while (registered_tables_request == None
or registered_clickstream_request == None):
try:
# try to get the table request
# table_request_id
if registered_tables_request == None:
print 'TIME TO REQUEST TABLES'
tables_response = api.post(TABLES_ISSUE_REQUEST)
print 'THIS IS THE RESPONSE OF THE TABLE-POST' + str(
tables_response[0])
registered_tables_request = api.get(
tables_response[0].to_json()["id"])[0]
print 'THIS IS THE REGISTERED TABLE-REQUEST ' + str(
registered_tables_request)
except Exception as e:
print 'THIS IS THE MESSAGE: '
print e.message
print 'THAT WAS THE MESSAGE'
if int(str(e.message.split(' ')[0])) != 429:
#time to wait
exit_with_error = True
error_message += 'tables: ' + e.message
break
else:
print 'YOU NEED TO WAIT FOR TABLES'
try:
# try to get the clickstream request
if registered_clickstream_request == None:
print 'ABOUT TO REQUEST CLICKSTREAM'
print str(CLICKSTREAM_ISSUE_REQUEST.to_json())
clickstream_response = api.post(CLICKSTREAM_ISSUE_REQUEST)
print 'THIS IS THE RESPONSE OF THE CLICKSTREAM-POST: ' + str(
clickstream_response)
print 'THIS IS THE RESPONSE OF THE CLICKSTREAM-POST: ' + str(
clickstream_response[0])
print type(clickstream_response[0])
registered_clickstream_request = api.get(
clickstream_response[0].to_json()["id"])[0]
print 'THIS IS THE REGISTERED CLICKSTREAM-REQUEST ' + str(
registered_clickstream_request)
except Exception as e:
print 'THIS IS THE MESSAGE: '
print e.message
print 'THAT WAS THE MESSAGE: '
if int(e.message.split(' ')[0]) != 429:
exit_with_error = True
error_message += 'clickstream: ' + e.message
break
else:
print 'YOU NEED TO WAIT FOR EVENTS'
#wait for the necessary time
print registered_tables_request, registered_clickstream_request
if (registered_tables_request == None
or registered_clickstream_request == None):
time.sleep(60)
if exit_with_error:
print error_message
sys.exit()
else:
return registered_tables_request, registered_clickstream_request, range_last
from datetime import date, datetime, timedelta
from courseraresearchexports.exports import api, utils
from courseraresearchexports.models import ExportRequest
from courseraresearchexports.constants.api_constants import ANONYMITY_LEVEL_COORDINATOR, EXPORT_TYPE_TABLES, EXPORT_TYPE_CLICKSTREAM, SCHEMA_NAMES
range_first = str(date.today() - timedelta(days=5))
range_last = str(date.today() - timedelta(days=1))
CLICKSTREAM_ISSUE_REQUEST = ExportRequest.ExportRequest(
partner_id=253,
statement_of_purpose="purpose_for_requests",
export_type=EXPORT_TYPE_CLICKSTREAM,
anonymity_level=ANONYMITY_LEVEL_COORDINATOR,
interval=[range_first, range_last])
clickstream_response = api.post(CLICKSTREAM_ISSUE_REQUEST)
print 'THIS IS THE RESPONSE OF THE POST' + str(clickstream_response[0])
registered_clickstream_request = api.get(
clickstream_response[0].to_json()["id"])
print 'THIS IS THE REGISTERED REQUEST ' + str(
registered_clickstream_request[0])