def handle(self, *args, **options):
if options['update']:
self.check_settings_exist()
self.set_cosmos_client()
version = self.get_latest_version_number()
num_institutions = self.get_number_of_institutions(version)
institutions_list = self.get_institutions(version)
with open(self.fixture_file, 'w') as outfile:
json.dump(institutions_list, outfile, indent=4)
self.success('Saved file to ' + self.fixture_file)
outfile.close()
else:
with open(self.fixture_file, 'r') as file:
contents = json.load(file)
self.mongo = Mongo('institutions')
self.mongo.collection_delete()
try:
self.mongo.insert(contents)
self.success('Inserted institutions into MongoDB')
except Exception as e:
message = e.message if hasattr(e, 'message') else \
'Failed to insert institutions into MongoDB'
raise CommandError(message)
def populate_mongodb(self):
if settings.MONGODB_HOST == self.mongo_host and \
settings.MONGODB_USERNAME == self.mongo_username and \
settings.MONGODB_PASSWORD == self.mongo_password:
mongo = Mongo('institutions')
result = mongo.get_one(
{'institution_id': str(self.institution_id)})
if result is None:
management.call_command('populate_institutions')
management.call_command('populate_courses')
def load_institution_data(institution_id):
if settings.LOCAL:
return InstitutionMocks.get_successful_institution_load_response()
if settings.MONGODB_HOST:
mongo = Mongo('institutions')
return mongo.get_one({'institution_id': str(institution_id)})
else:
headers = {'Ocp-Apim-Subscription-Key': settings.DATASETAPIKEY}
base_url = "%s/institutions/%s"
return requests.get(url=base_url %
(settings.DATASETAPIHOST, institution_id),
headers=headers)
def load_course_data(institution_id, course_id, mode):
if settings.LOCAL:
if course_id == "GN12":
return JointCourseFormatMocks.get_successful_course_load_response()
return CourseFormatMocks.get_successful_course_load_response()
if settings.MONGODB_HOST:
mongo = Mongo('courses')
return mongo.get_one({
'institution_id': str(institution_id),
'course_id': str(course_id),
})
else:
headers = {'Ocp-Apim-Subscription-Key': settings.DATASETAPIKEY}
base_url = "%s/institutions/%s/courses/%s/modes/%s"
response = requests.get(
url=base_url %
(settings.DATASETAPIHOST, institution_id, course_id, mode),
headers=headers)
return response
def __route_mongo(self, sw):
args, db = self.args, Mongo(self.log, sw)
if args['load'] and args['-a']:
db.load()
elif args['load'] and args['--slice']:
db.load(int(args['--slice']))
elif args['search']:
print "Output: %s" % [
x for x in db.search('search', args['--key'], args['--value'])
]
elif args['retr']:
print "Output: %s" % [
x for x in db.search('select_exists', args['--key'], None,
args['--value'], None)
]
elif args['del']:
db.clear_all(True if args['-x'] else False)
class Command(CosmosCommand):
help = 'Copies Institutions from Cosmos DB to the local MongoDB ' + \
'(and into the institutions/fixtures folder)'
request_options = {"enableCrossPartitionQuery": True}
base_url = 'dbs/discoveruni/colls/'
fixture_file = 'institutions/fixtures/institutions.json'
full_fixture_filename = None
mongo = None
version = None
num_institutions = None
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.full_fixture_filename = str(
Path(settings.BASE_DIR + '/' + self.fixture_file))
def add_arguments(self, parser):
parser.add_argument(
'--update',
action='store_true',
help='Gets the latest data from CosmosDB and stores it in ' + \
self.fixture_file,
)
def handle(self, *args, **options):
if options['update']:
self.check_settings_exist()
self.set_cosmos_client()
version = self.get_latest_version_number()
num_institutions = self.get_number_of_institutions(version)
institutions_list = self.get_institutions(version)
with open(self.fixture_file, 'w') as outfile:
json.dump(institutions_list, outfile, indent=4)
self.success('Saved file to ' + self.fixture_file)
outfile.close()
else:
with open(self.fixture_file, 'r') as file:
contents = json.load(file)
self.mongo = Mongo('institutions')
self.mongo.collection_delete()
try:
self.mongo.insert(contents)
self.success('Inserted institutions into MongoDB')
except Exception as e:
message = e.message if hasattr(e, 'message') else \
'Failed to insert institutions into MongoDB'
raise CommandError(message)
def save_institutions_to_mongo_db(self, institutions_json):
self.mongo_collection.insert(institutions_json)
def get_number_of_institutions(self, version):
number_of_institutions = self.base_query(
'institutions',
'SELECT VALUE COUNT(1) from c WHERE c.version = ' + str(version))
try:
number_of_institutions_int = int(number_of_institutions[0])
except Exception:
raise CommandError('number of institutions returned is not an ' + \
'integer in a list (' + str(number_of_institutions) + ')')
self.success('Got number of institutions from CosmosDB (' + \
str(number_of_institutions_int) + ')')
return number_of_institutions_int
def get_institutions(self, version):
sql = 'SELECT * from c WHERE c.version = ' + str(version)
qurey_result = None
try:
qurey_result = self.cosmos_client.QueryItems(
self.base_url + 'institutions', sql, self.request_options)
except Exception:
raise CommandError('CosmosDB instututions query failed')
try:
list_result = list(qurey_result)
except Exception:
raise CommandError('Converting instututions into list failed (' + \
str(query_result) + ')')
self.success('Got institutions from CosmosDB')
return list_result
def get_latest_version_number(self):
version = self.base_query('datasets',
'SELECT VALUE MAX(c.version) from c')
try:
version_int = int(version[0])
except Exception:
raise CommandError('version returned is not an integer in a list (' + \
str(version) + ')')
self.success('Got latest version number from CosmosDB (' + \
str(version_int) + ')')
return version_int
class Command(CosmosCommand):
help = 'Copies Courses from Cosmos DB to the local MongoDB ' + \
'(and into the courses/fixtures folder)'
fixture_file = 'courses/fixtures/courses.json'
full_fixture_filename = None
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.full_fixture_filename = str(
Path(settings.BASE_DIR + '/' + self.fixture_file)
)
def add_arguments(self, parser):
parser.add_argument(
'--update',
action='store_true',
help='Gets the latest data from CosmosDB and stores it in ' + \
self.fixture_file,
)
def handle(self, *args, **options):
if options['update']:
self.check_settings_exist()
self.set_cosmos_client()
version = self.get_latest_version_number()
courses_list = self.get_courses(version)
with open(self.fixture_file, 'w') as outfile:
json.dump(courses_list, outfile, indent=4)
self.success('Saved file to ' + self.fixture_file)
else:
with open(self.fixture_file, 'r') as file:
self.info('Getting courses from ' + self.fixture_file)
contents = json.load(file)
self.success('Retrived courses from ' + self.fixture_file)
self.mongo = Mongo('courses')
self.mongo.collection_delete()
try:
self.mongo.insert(contents)
self.success('Inserted courses into MongoDB')
except Exception as e:
message = e.message if hasattr(e, 'message') else \
'Failed to insert institutions into MongoDB'
raise CommandError(message)
def get_courses(self, version):
if not settings.TEST_COURSES:
raise CommandError('No test corses found in TEST_COURSES')
courses = []
for course_id in settings.TEST_COURSES.split(','):
course_data = self.get_course(course_id, version)
if course_data:
self.success('Got course (' + str(course_id) + ') from CosmosDB')
courses.append(course_data[0])
else:
self.error('Could not get data for course (' + \
str(course_id) + ')')
return courses
def get_course(self, course_id, version):
sql = 'SELECT * from c WHERE c.version = ' + str(version) + ' ' + \
'AND c.course_id = "' + str(course_id) + '"'
qurey_result = None
try:
qurey_result = self.cosmos_client.QueryItems(
self.base_url + 'courses', sql,
self.request_options
)
except Exception:
raise CommandError('CosmosDB course query failed')
try:
list_result = list(qurey_result)
except Exception:
raise CommandError('Converting course into list failed (' + \
str(query_result) + ')')
return list_result