def update_common_core_map(cc_file):
logging.info("Deferred job <update_common_core_map> started")
reader = csv.reader(cc_file, delimiter='\t')
_ = reader.next()
cc_list = []
cc_standards = {}
for line in reader:
cc_standard = line[0]
cc_cluster = line[1]
try:
cc_description = line[2].encode('utf-8')
except Exception:
cc_description = cc_cluster
exercise_name = line[3]
video_youtube_id = line[4]
if len(cc_standard) == 0:
continue
if cc_standard in cc_standards:
cc = cc_standards[cc_standard]
else:
cc = CommonCoreMap.all().filter('standard = ', cc_standard).get()
if cc is None:
cc = CommonCoreMap()
cc_standards[cc_standard] = cc
cc_list.append(cc)
cc.update_standard(cc_standard, cc_cluster, cc_description)
if len(exercise_name) > 0:
cc.update_exercise(exercise_name)
if len(video_youtube_id) > 0:
cc.update_video(video_youtube_id)
if len(cc_list) > 500:
db.put(cc_list)
cc_list = []
cc_standards = {}
db.put(cc_list)
return
def update_common_core_map(cc_file):
logging.info("Deferred job <update_common_core_map> started")
reader = csv.reader(cc_file, delimiter='\t')
_ = reader.next()
cc_list = []
cc_standards = {}
for line in reader:
cc_standard = line[0]
cc_cluster = line[1]
try:
cc_description = line[2].encode('utf-8')
except Exception:
cc_description = cc_cluster
exercise_name = line[3]
video_youtube_id = line[4]
if len(cc_standard) == 0:
continue
if cc_standard in cc_standards:
cc = cc_standards[cc_standard]
else:
cc = CommonCoreMap.all().filter('standard = ', cc_standard).get()
if cc is None:
cc = CommonCoreMap()
cc_standards[cc_standard] = cc
cc_list.append(cc)
cc.update_standard(cc_standard, cc_cluster, cc_description)
if len(exercise_name) > 0:
cc.update_exercise(exercise_name)
if len(video_youtube_id) > 0:
cc.update_video(video_youtube_id)
if len(cc_list) > 500:
db.put(cc_list)
cc_list = []
cc_standards = {}
db.put(cc_list)
return
cc["domain"] = domains[cc["domain_code"]]['name'].decode("utf8")
cc["cc_cluster"] = row.cluster.decode("utf8")
cc["cc_description"] = row.description.decode("utf8")
cc["videos"] = set()
cc["exercises"] = set()
cc_standards[row.standard] = cc
if row.exercise_name:
cc['exercises'].add(row.exercise_name)
if row.youtube_id:
cc['videos'].add(row.youtube_id)
while reset:
maps = CommonCoreMap.all(keys_only=True).fetch(limit=500)
if not maps:
break
logging.info("Clearing %s Common Core Maps", len(maps))
db.delete(maps)
cc_list = []
for standard, data in cc_standards.iteritems():
changed = False
videos, exercises = data.pop("videos"), data.pop("exercises")
cc = CommonCoreMap.all().filter("standard = ", standard).get()
if not cc:
cc = CommonCoreMap(**data)
changed = True
for ex in exercises: