def load_json(self, data_type, path):
logging.info('loading json for data type %s', data_type)
json_file = open(ROOT_PATH + "/data/" + path + str(data_type) + ".json")
json_str = json_file.read()
json_data = simplejson.loads(json_str) # Load JSON file as object
memcache.set("json_" + str(data_type), json_data, 60000)
entities = []
# Setup, Imports
if data_type == 'employers': emp = emp_data()
def special_processes(self, data_type):
if data_type == 'employers':
emp = emp_data()
emp.refresh_employer_images()
def load_data(self, data_type, MIN_SLICE = None, MAX_SLICE = None):
logging.info('loading data type %s from %s to %s' % (data_type, str(MIN_SLICE), str(MAX_SLICE)))
entities = []
# Setup, Imports
if data_type == 'employers': emp = emp_data()
newdata = memcache.get("json_" + str(data_type))
if len(newdata[MIN_SLICE:MAX_SLICE]) == 0:
return "Data Load Is Finished"
for entity in newdata[MIN_SLICE:MAX_SLICE]: # Could this be more efficient?
if data_type == 'proficiencies':
this_entity = Proficiency.get_by_key_name(entity['name'])
if this_entity: save_entity = this_entity #if this exists, modify it
else: save_entity = Proficiency(key_name=entity['name'], name = entity['name'])
save_entity.name = entity['name']
save_entity.status = entity.get("status", "")
these_links = entity.get("links", [])
if len(these_links) > 0: save_entity.links = these_links
save_entity.video_html = entity.get("video_html", "")
save_entity.status = entity.get('status', None)
save_entity.blurb = entity.get('blurb', None)
save_entity.popularity = entity.get("popularity", 50)
save_entity.difficulty = entity.get("difficulty", 50)
if data_type == 'proficiency_topics':
this_proficiency = Proficiency.get_by_key_name(entity['proficiency']['name'])
if not this_proficiency:
print "proficiency ", entity['proficiency']['name'], " not found when inserting topic ", entity['name']
logging.error('proficiency %s not found for topic %s' % (entity['proficiency']['name'], entity['name']))
continue
this_entity = ProficiencyTopic.get_by_key_name(entity['proficiency']['name'] + str('_') + entity['name'])
if this_entity: save_entity = this_entity #if this exists, modify it
else: save_entity = ProficiencyTopic(key_name=entity['proficiency']['name'] + str('_') + entity['name'], name = entity['name'])
save_entity.name = entity['name']
save_entity.proficiency = this_proficiency
if data_type == 'content_pages':
this_proficiency = Proficiency.get_by_key_name(entity['proficiency']['name'])
save_entity = ContentPage(key_name=entity['url'], url = entity['url'], proficiency = this_proficiency)
if data_type == 'links':
this_subject = Proficiency.get_by_key_name(entity['subject']['name'])
this_link = Link.get_by_key_name(entity['subject']['name'] + "_" + entity['url'])
if not this_link: save_entity = Link(key_name=entity['subject']['name'] + "_" + entity['url'], title = entity['title'], url = entity['url'], subject = this_subject)
if data_type == 'raw_items':
this_url = ContentPage.get_by_key_name(entity['page']['url'])
save_entity = RawQuizItem(#key_name?
index = entity['index'],
answer_candidates = entity['answer_candidates'],
pre_content = entity['pre_content'],
content = entity['content'],
post_content = entity['post_content'],
page = this_url,
moderated = False)
if data_type == 'quiz_items':
this_proficiency = Proficiency.get_by_key_name(entity['proficiency']['name'])
if not this_proficiency:
print "proficiency ", entity['proficiency']['name'], " not found when inserting quiz item"
logging.error('proficiency %s not found for quiz item', entity['proficiency']['name'])
continue
this_topic = ProficiencyTopic.get_by_key_name(entity['proficiency']['name'] + str('_') + entity['topic']['name'])
if not this_topic:
print "proficiency topic ", entity['topic']['name'], " not found when inserting quiz item"
logging.error('proficiency topic %s not found for quiz item', entity['topic']['name'])
continue
import string
if entity['index'].lower() not in [ a.lower() for a in entity['answers'] ]: logging.error('Quiz Item Index %s not in answers %s' % ( entity['index'], str(entity['answers'])))
save_entity = QuizItem( content = entity['content'],
theme = entity['theme'],
proficiency = this_proficiency.key(),
answers = [ ".".join( [ string.capwords(l) for l in a.split(".") ] ) for a in entity['answers'] ],
index = entity['index'],
topic = this_topic.key())
if entity['content_url']: save_entity.content_url = entity['content_url']
if data_type == 'employers':
save_entity = False # employers.methods used to save
for e in emp.create_business_account(entity['unique_identifier'], entity.get('email', None), entity['proficiencies'], batch=True):
entities.append(e)
if data_type == 'mailing_list':
save_entity = MailingList(key_name=entity['email'], fullname = entity.get('fullname', ""), email = entity['email'], type = entity.get('type', ""))
if data_type == 'settings':
save_entity = Setting(key_name=entity['name'], name = entity['name'], value = entity['value'])
if save_entity: entities.append(save_entity)
try:
entity_num = 0
self.load_entities[data_type] = entities
for entity in entities: entity_num += 1
logging.info('finished preparing data load for %d %s' % (len(entities), data_type))
return len(entities)
except:
logging.error('Unable to save %s at number %d' % (data_type, entity_num))
return False
