def load_json(self, data_type, path): logging.info('loading json for data type %s', data_type) json_file = open(ROOT_PATH + "/data/" + path + str(data_type) + ".json") json_str = json_file.read() json_data = simplejson.loads(json_str) # Load JSON file as object memcache.set("json_" + str(data_type), json_data, 60000) entities = [] # Setup, Imports if data_type == 'employers': emp = emp_data()
def special_processes(self, data_type): if data_type == 'employers': emp = emp_data() emp.refresh_employer_images()
def load_data(self, data_type, MIN_SLICE = None, MAX_SLICE = None): logging.info('loading data type %s from %s to %s' % (data_type, str(MIN_SLICE), str(MAX_SLICE))) entities = [] # Setup, Imports if data_type == 'employers': emp = emp_data() newdata = memcache.get("json_" + str(data_type)) if len(newdata[MIN_SLICE:MAX_SLICE]) == 0: return "Data Load Is Finished" for entity in newdata[MIN_SLICE:MAX_SLICE]: # Could this be more efficient? if data_type == 'proficiencies': this_entity = Proficiency.get_by_key_name(entity['name']) if this_entity: save_entity = this_entity #if this exists, modify it else: save_entity = Proficiency(key_name=entity['name'], name = entity['name']) save_entity.name = entity['name'] save_entity.status = entity.get("status", "") these_links = entity.get("links", []) if len(these_links) > 0: save_entity.links = these_links save_entity.video_html = entity.get("video_html", "") save_entity.status = entity.get('status', None) save_entity.blurb = entity.get('blurb', None) save_entity.popularity = entity.get("popularity", 50) save_entity.difficulty = entity.get("difficulty", 50) if data_type == 'proficiency_topics': this_proficiency = Proficiency.get_by_key_name(entity['proficiency']['name']) if not this_proficiency: print "proficiency ", entity['proficiency']['name'], " not found when inserting topic ", entity['name'] logging.error('proficiency %s not found for topic %s' % (entity['proficiency']['name'], entity['name'])) continue this_entity = ProficiencyTopic.get_by_key_name(entity['proficiency']['name'] + str('_') + entity['name']) if this_entity: save_entity = this_entity #if this exists, modify it else: save_entity = ProficiencyTopic(key_name=entity['proficiency']['name'] + str('_') + entity['name'], name = entity['name']) save_entity.name = entity['name'] save_entity.proficiency = this_proficiency if data_type == 'content_pages': this_proficiency = Proficiency.get_by_key_name(entity['proficiency']['name']) save_entity = ContentPage(key_name=entity['url'], url = entity['url'], proficiency = this_proficiency) if data_type == 'links': this_subject = Proficiency.get_by_key_name(entity['subject']['name']) this_link = Link.get_by_key_name(entity['subject']['name'] + "_" + entity['url']) if not this_link: save_entity = Link(key_name=entity['subject']['name'] + "_" + entity['url'], title = entity['title'], url = entity['url'], subject = this_subject) if data_type == 'raw_items': this_url = ContentPage.get_by_key_name(entity['page']['url']) save_entity = RawQuizItem(#key_name? index = entity['index'], answer_candidates = entity['answer_candidates'], pre_content = entity['pre_content'], content = entity['content'], post_content = entity['post_content'], page = this_url, moderated = False) if data_type == 'quiz_items': this_proficiency = Proficiency.get_by_key_name(entity['proficiency']['name']) if not this_proficiency: print "proficiency ", entity['proficiency']['name'], " not found when inserting quiz item" logging.error('proficiency %s not found for quiz item', entity['proficiency']['name']) continue this_topic = ProficiencyTopic.get_by_key_name(entity['proficiency']['name'] + str('_') + entity['topic']['name']) if not this_topic: print "proficiency topic ", entity['topic']['name'], " not found when inserting quiz item" logging.error('proficiency topic %s not found for quiz item', entity['topic']['name']) continue import string if entity['index'].lower() not in [ a.lower() for a in entity['answers'] ]: logging.error('Quiz Item Index %s not in answers %s' % ( entity['index'], str(entity['answers']))) save_entity = QuizItem( content = entity['content'], theme = entity['theme'], proficiency = this_proficiency.key(), answers = [ ".".join( [ string.capwords(l) for l in a.split(".") ] ) for a in entity['answers'] ], index = entity['index'], topic = this_topic.key()) if entity['content_url']: save_entity.content_url = entity['content_url'] if data_type == 'employers': save_entity = False # employers.methods used to save for e in emp.create_business_account(entity['unique_identifier'], entity.get('email', None), entity['proficiencies'], batch=True): entities.append(e) if data_type == 'mailing_list': save_entity = MailingList(key_name=entity['email'], fullname = entity.get('fullname', ""), email = entity['email'], type = entity.get('type', "")) if data_type == 'settings': save_entity = Setting(key_name=entity['name'], name = entity['name'], value = entity['value']) if save_entity: entities.append(save_entity) try: entity_num = 0 self.load_entities[data_type] = entities for entity in entities: entity_num += 1 logging.info('finished preparing data load for %d %s' % (len(entities), data_type)) return len(entities) except: logging.error('Unable to save %s at number %d' % (data_type, entity_num)) return False