def get(self): #set all levels to None first diseases = DiseaseGroup.all() for dis in diseases.fetch(10**6): dis.DBLevel = None dis.put() #update level of 'disease' disease = db.GqlQuery('select * from DiseaseGroup where DBID = :1', '4').get() disease.DBLevel = 1 disease.put() added = 0 errorsCaught = [] level = 1 parentq = db.GqlQuery('select * from DiseaseGroup where DBLevel = :1') childq = db.GqlQuery( 'select * from DiseaseGroup where DBID = :1 and DBParent = :2') while True: parentq.bind(level) parents = parentq.fetch(10**6) if parents is None or len(parents) == 0: break # children = [] for disease in parents: # children.extend(disease.DBChildren) for childID in disease.DBChildren: childq.bind(childID, disease.DBID) child = childq.get() try: child.DBLevel = level + 1 child.put() added += 1 except Exception: errorsCaught.append((childID, disease.DBID)) if added % 50 == 0: logging.info('still updating levels') level += 1 self.response.out.write(''' <html> <body> DGs added:''' + str(added) + '<br> errors caught:' + str(errorsCaught)) self.response.out.write(''' <form action='/' method = 'get'> <div><input type='submit' value='Done'></div> </form> ''') self.response.out.write(''' </body> </html> ''')
def get(self): #set all levels to None first diseases = DiseaseGroup.all() for dis in diseases.fetch(10**6): dis.DBLevel = None dis.put() #update level of 'disease' disease = db.GqlQuery('select * from DiseaseGroup where DBID = :1', '4').get() disease.DBLevel = 1 disease.put() added = 0 errorsCaught = [] level = 1 parentq = db.GqlQuery('select * from DiseaseGroup where DBLevel = :1') childq = db.GqlQuery('select * from DiseaseGroup where DBID = :1 and DBParent = :2') while True: parentq.bind(level) parents = parentq.fetch(10**6) if parents is None or len(parents) == 0: break # children = [] for disease in parents: # children.extend(disease.DBChildren) for childID in disease.DBChildren: childq.bind(childID,disease.DBID) child = childq.get() try: child.DBLevel = level + 1 child.put() added += 1 except Exception: errorsCaught.append((childID, disease.DBID)) if added % 50 == 0: logging.info('still updating levels') level += 1 self.response.out.write(''' <html> <body> DGs added:''' + str(added) + '<br> errors caught:' + str(errorsCaught)) self.response.out.write(''' <form action='/' method = 'get'> <div><input type='submit' value='Done'></div> </form> ''') self.response.out.write(''' </body> </html> ''')
def get(self): old = DiseaseGroup.all().fetch(10**6) for d in old: d.delete() added = 0 errorsCaught = [] DGs = [] parser = Parsers.DOParser( 'http://diseaseontology.svn.sourceforge.net/viewvc/diseaseontology/trunk/HumanDO.obo' ) terms = parser.parse( ) #terms = {'DOID':('name', ['alt_ids'], 'def', ['synonyms'], 'subset', ['xrefs'], 'is_a')} # temp = terms.copy() # levels = self.getLevels(temp) # raise Exception('# levels = %s' % len(levels)) # for i in xrange(len(levels)): # level = levels[i] for DOID, attrs in terms.items(): defi = attrs[2] if defi is not None and len(defi) > 500: defi = attrs[2][:500] #500 character limit cutoff parents = attrs[-1] # if level != len(levels) - 1: children = self._getChildren(DOID, terms) # level = self._getlevel(DOID, terms) if parents: # count = 1 for parent in parents: # keyName = '{0}_{1}'.format(DOID, count) keyName = '{0}_{1}'.format(DOID, parent) try: DG = DiseaseGroup(key_name=keyName, DBID=DOID, DBName=attrs[0], DBAlt_ids=attrs[1], DBDef=defi, DBSyns=attrs[3], DBSubset=attrs[4], DBXrefs=attrs[5], DBParent=parent, DBChildren=children, DBLevel=None) DG.put() DGs.append(DG) added += 1 except UnicodeDecodeError: errorsCaught.append(DOID) else: try: DG = DiseaseGroup(key_name=DOID, DBID=DOID, DBName=attrs[0], DBAlt_ids=attrs[1], DBDef=defi, DBSyns=attrs[3], DBSubset=attrs[4], DBXrefs=attrs[5], DBParent=parent, DBChildren=children, DBLevel=None) DG.put() DGs.append(DG) added += 1 except UnicodeDecodeError: errorsCaught.append(DOID) # if len(parent) == 0: # try: # DG = DiseaseGroup(key_name = DOID, # DBID = DOID, # DBName = attrs[0], # DBAlt_ids = attrs[1], # DBDef = defi, # DBSyns = attrs[3], # DBSubset = attrs[4], # DBXrefs = attrs[5], # DBParent = None, # DBChildren = children, # DBLevel = 1) # #DBLevel = i+1) # DG.put() # added += 1 # except UnicodeDecodeError: # errorsCaught.append(DOID) # # elif len(parents) == 1: # try: # parent = parents[0] # level = levels[parent] # DG = DiseaseGroup(key_name = DOID, # DBID = DOID, # DBName = attrs[0], # DBAlt_ids = attrs[1], # DBDef = defi, # DBSyns = attrs[3], # DBSubset = attrs[4], # DBXrefs = attrs[5], # DBParent = parent, # DBChildren = children, # DBLevel = level) # #DBLevel = i+1) # DG.put() # added += 1 # except UnicodeDecodeError: # errorsCaught.append(DOID) # # else: # for j in xrange(len(parents)): # parent = parents[j] # level = levels[parent] # formatted_id = '{0}_{1}'.format(DOID, str(j)) # try: # DG = DiseaseGroup(key_name = formatted_id, # DBID = DOID, # DBName = attrs[0], # DBAlt_ids = attrs[1], # DBDef = defi, # DBSyns = attrs[3], # DBSubset = attrs[4], # DBXrefs = attrs[5], # DBParent = parent, # DBChildren = children, # DBLevel = level) # #DBLevel = i+1) # DG.put() # added += 1 # except UnicodeDecodeError: # errorsCaught.append(DOID) # errors = self._addLevels(DGs) self.response.out.write(''' <html> <body> DGs added:''' + str(added) + '<br> length of terms:' + str(len(terms)) + '<br> errors caught:' + str(errorsCaught) + '<br> UPDATE DISEASE LEVELS NOW') # + '<br> addLevels errors: ' + str(errors)) self.response.out.write(''' <form action='/' method = 'get'> <div><input type='submit' value='Done'></div> </form> ''') self.response.out.write(''' </body> </html> ''') self.redirect('/admin/updateRogueDiseases')
def get(self): old = DiseaseGroup.all().fetch(10**6) for d in old: d.delete() added = 0 errorsCaught = [] DGs = [] parser = Parsers.DOParser('http://diseaseontology.svn.sourceforge.net/viewvc/diseaseontology/trunk/HumanDO.obo') terms = parser.parse() #terms = {'DOID':('name', ['alt_ids'], 'def', ['synonyms'], 'subset', ['xrefs'], 'is_a')} # temp = terms.copy() # levels = self.getLevels(temp) # raise Exception('# levels = %s' % len(levels)) # for i in xrange(len(levels)): # level = levels[i] for DOID, attrs in terms.items(): defi = attrs[2] if defi is not None and len(defi) > 500: defi = attrs[2][:500] #500 character limit cutoff parents = attrs[-1] # if level != len(levels) - 1: children = self._getChildren(DOID, terms) # level = self._getlevel(DOID, terms) if parents: # count = 1 for parent in parents: # keyName = '{0}_{1}'.format(DOID, count) keyName = '{0}_{1}'.format(DOID, parent) try: DG = DiseaseGroup(key_name = keyName, DBID = DOID, DBName = attrs[0], DBAlt_ids = attrs[1], DBDef = defi, DBSyns = attrs[3], DBSubset = attrs[4], DBXrefs = attrs[5], DBParent = parent, DBChildren = children, DBLevel = None) DG.put() DGs.append(DG) added += 1 except UnicodeDecodeError: errorsCaught.append(DOID) else: try: DG = DiseaseGroup(key_name = DOID, DBID = DOID, DBName = attrs[0], DBAlt_ids = attrs[1], DBDef = defi, DBSyns = attrs[3], DBSubset = attrs[4], DBXrefs = attrs[5], DBParent = parent, DBChildren = children, DBLevel = None) DG.put() DGs.append(DG) added += 1 except UnicodeDecodeError: errorsCaught.append(DOID) # if len(parent) == 0: # try: # DG = DiseaseGroup(key_name = DOID, # DBID = DOID, # DBName = attrs[0], # DBAlt_ids = attrs[1], # DBDef = defi, # DBSyns = attrs[3], # DBSubset = attrs[4], # DBXrefs = attrs[5], # DBParent = None, # DBChildren = children, # DBLevel = 1) # #DBLevel = i+1) # DG.put() # added += 1 # except UnicodeDecodeError: # errorsCaught.append(DOID) # # elif len(parents) == 1: # try: # parent = parents[0] # level = levels[parent] # DG = DiseaseGroup(key_name = DOID, # DBID = DOID, # DBName = attrs[0], # DBAlt_ids = attrs[1], # DBDef = defi, # DBSyns = attrs[3], # DBSubset = attrs[4], # DBXrefs = attrs[5], # DBParent = parent, # DBChildren = children, # DBLevel = level) # #DBLevel = i+1) # DG.put() # added += 1 # except UnicodeDecodeError: # errorsCaught.append(DOID) # # else: # for j in xrange(len(parents)): # parent = parents[j] # level = levels[parent] # formatted_id = '{0}_{1}'.format(DOID, str(j)) # try: # DG = DiseaseGroup(key_name = formatted_id, # DBID = DOID, # DBName = attrs[0], # DBAlt_ids = attrs[1], # DBDef = defi, # DBSyns = attrs[3], # DBSubset = attrs[4], # DBXrefs = attrs[5], # DBParent = parent, # DBChildren = children, # DBLevel = level) # #DBLevel = i+1) # DG.put() # added += 1 # except UnicodeDecodeError: # errorsCaught.append(DOID) # errors = self._addLevels(DGs) self.response.out.write(''' <html> <body> DGs added:''' + str(added) + '<br> length of terms:' + str(len(terms)) + '<br> errors caught:' + str(errorsCaught) + '<br> UPDATE DISEASE LEVELS NOW') # + '<br> addLevels errors: ' + str(errors)) self.response.out.write(''' <form action='/' method = 'get'> <div><input type='submit' value='Done'></div> </form> ''') self.response.out.write(''' </body> </html> ''') self.redirect('/admin/updateRogueDiseases')