def adapt_passim_project_name_manu(): oErr = ErrHandle() bResult = True msg = "" name = "Passim" # Issue 412: give all items in the current database the Project name "Passim" # This means: Manuscript, SSG/AF, Historical collection # Sermon: NO, is done in a separate method try: # Iterate over all Manuscripts in the database: for manu in Manuscript.objects.all(): # Try to find if the project already exists: projectfound = Project2.objects.filter(name__iexact = name).first() if projectfound == None: # If the project does not already exist, it needs to be added to the database project = Project2.objects.create(name = name) # And a link should be made between this new project and the manuscript ManuscriptProject.objects.create(project = project, manuscript = manu) else: manuprojlink = ManuscriptProject.objects.filter(project = projectfound, manuscript = manu).first() if manuprojlink == None: # If the project already exists, but not the link, than only a link should be # made between the manuscript and the project ManuscriptProject.objects.create(project = projectfound, manuscript = manu) #print(manu, projectfound) # When iterating over Manuscript, make sure to iterate over Sermons too, how to access them? # Sermons belong to 1 manu # take project of Manuscript, # SermonDescrProject (project = projectmanu, sermon = sermon) except: bResult = False msg = oErr.get_error_message() return bResult, msg
def __init__(self, *args, **kwargs): self.username = kwargs.pop('username', "") self.team_group = kwargs.pop('team_group', "") self.userplus = kwargs.pop('userplus', "") # Start by executing the standard handling super(EqualApprovalForm, self).__init__(*args, **kwargs) oErr = ErrHandle() try: # Some fields are not required self.fields['change'].required = False self.fields['comment'].required = False self.fields['profile'].required = False self.fields['atype'].required = False # Set queryset(s) - for details view self.fields['profilelist'].queryset = Profile.objects.all( ).order_by('user__username') self.fields['passimlist'].queryset = EqualGold.objects.filter( code__isnull=False, moved__isnull=True).order_by('code') self.fields['atypelist'].queryset = FieldChoice.objects.filter( field=APPROVAL_TYPE).order_by("english_name") # Get the instance if 'instance' in kwargs: instance = kwargs['instance'] except: msg = oErr.get_error_message() oErr.DoError("EqualApprovalForm") return None
def adapt_codico_origin(): oErr = ErrHandle() bResult = True msg = "" name = "Passim" # Issue #427: # Create one OriginCod object for each Origin that is part of a Codico # Then set the 'origin' field of that Codico to None try: qs = Codico.objects.all() lst_codico = [] with transaction.atomic(): for codico in qs: # Does this one have an origin set? if not codico.origin is None: # Check if an appropriate OriginCod is already there obj = OriginCod.objects.filter(codico=codico).first() if obj is None: # Create one obj = OriginCod.objects.create(codico=codico, origin=codico.origin) lst_codico.append(codico) # Set the 'origin' field to none with transaction.atomic(): for codico in lst_codico: codico.origin = None codico.save() except: bResult = False msg = oErr.get_error_message() return bResult, msg
def adapt_passim_project_name_equal(): oErr = ErrHandle() bResult = True msg = "" name = "Passim" # Issue 412: give all items in the current database the Project name "Passim" # This means: Manuscript, SSG/AF, Historical collection try: # Iterate over all SSGs/AFs in the database: for equal in EqualGold.objects.all(): # Try to find if the project already exists: projectfound = Project2.objects.filter(name__iexact = name).first() if projectfound == None: # If the project does not already exist, it needs to be added to the database project = Project2.objects.create(name = name) # And a link should be made between this new project and the SSG/AF EqualGoldProject.objects.create(project = project, equal = equal) else: equalprojlink = EqualGoldProject.objects.filter(project = projectfound, equal = equal).first() if equalprojlink == None: # If the project already exists, but not the link, than only a link should be # made between the collection and the project EqualGoldProject.objects.create(project = projectfound, equal = equal) print(equal, projectfound) except: bResult = False msg = oErr.get_error_message() return bResult, msg
def get_ssglists(self, recalculate=False): """Get the set of lists for this particular ResearchSet""" oErr = ErrHandle() lBack = None try: if self.contents != "" and self.contents[0] == "[": lst_contents = json.loads(self.contents) else: lst_contents = [] if not recalculate and len(lst_contents) > 0: # Should the unique_matches be re-calculated? if not 'unique_matches' in lst_contents[0]: # Yes, re-calculate lst_contents = self.calculate_matches(lst_contents) # And make sure this gets saved! self.contents = json.dumps(lst_contents) self.save() lBack = lst_contents else: # Re-calculate the lists self.update_ssglists() # Get the result lBack = json.loads(self.contents) except: msg = oErr.get_error_message() oErr.DoError("ResearchSet/get_ssglists") return lBack
def get_review_list(profile, all=False): """Get the list of objects this editor needs to review""" oErr = ErrHandle() lstBack = [] try: # Default: return the empty list # lstBack = EqualChange.objects.none() # Get the list of projects for this user lst_project_id = profile.projects.all().values("id") if len(lst_project_id) > 0: # Get the list of EqualChange objects linked to any of these projects lstQ = [] lstQ.append( Q(super__equal_proj__project__id__in=lst_project_id)) if not all: lstQ.append(Q(atype='def')) lstBack = [ x['id'] for x in EqualChange.objects.exclude( profile=profile).filter(*lstQ).distinct().values('id') ] # lstBack = EqualChange.objects.exclude(profile=profile).filter(*lstQ).distinct() except: msg = oErr.get_error_message() oErr.DoError("EqualChange/get_review_list") return lstBack
def get_approver_list(self, excl=None): """Get the list of editors that need to approve this change If [excl] is specified, then this object is excluded from the list of Profile objects returned """ oErr = ErrHandle() lstBack = None try: # Default: return the empty list lstBack = Profile.objects.none() # Get all the projects to which this SSG 'belongs' lst_project = [ x['id'] for x in self.super.projects.all().values("id") ] # Note: only SSGs that belong to more than one project need to be reviewed if len(lst_project) > 1: # Get all the editors associated with these projects lst_profile_id = [ x['profile_id'] for x in ProjectEditor.objects.filter( project__id__in=lst_project).values( 'profile_id').distinct() ] if len(lst_profile_id) > 0: if excl == None: lstBack = Profile.objects.filter(id__in=lst_profile_id) else: lstBack = Profile.objects.filter( id__in=lst_profile_id).exclude(id=excl.id) except: msg = oErr.get_error_message() oErr.DoError("EqualChange/get_approver_list") return lstBack
def get_testsets(request): """Get a list of testunits per testset of one round""" oErr = ErrHandle() try: data = 'fail' qd = request.GET if request.method == "GET" else request.POST if request.is_ajax(): round = qd.get("round", "") lstQ = [] lstQ.append(Q(testset__round=round)) items = TestsetUnit.objects.filter(*lstQ).order_by("testset__number").values( 'testset__round', 'testset__number', 'testunit__speaker__name', 'testunit__speaker__gender', 'testunit__fname', 'testunit__sentence__name', 'testunit__ntype') results = [] for idx, obj in enumerate(items): number = obj.get('testset__number') speaker = obj.get('testunit__speaker__name') gender = obj.get('testunit__speaker__gender') fname = obj.get('testunit__sentence__name') sentence = obj.get('testunit__fname') ntype = "Lom" if obj.get('testunit__ntype') == "n" else "Nat" co_json = {'idx': idx+1, 'testset': number, 'speaker': speaker, 'gender': gender, 'filename': fname, 'sentence': sentence, 'ntype': ntype } results.append(co_json) data = json.dumps(results) else: data = "Request is not ajax" except: msg = oErr.get_error_message() data = "error: {}".format(msg) mimetype = "application/json" return HttpResponse(data, mimetype)
def adapt_ssg_bidirectional(): oErr = ErrHandle() bResult = True msg = "" try: # Put all links in a list lst_link = [] lst_remove = [] lst_reverse = [] for obj in EqualGoldLink.objects.filter(linktype__in=LINK_BIDIR): # Check for any other eqg-links with the same source lst_src = EqualGoldLink.objects.filter(src=obj.src, dst=obj.dst).exclude(id=obj.id) if lst_src.count() > 0: # Add them to the removal for item in lst_src: lst_remove.append(item) else: # Add the obj to the list lst_link.append(obj) for obj in lst_link: # Find the reverse link reverse = EqualGoldLink.objects.filter(src=obj.dst, dst=obj.src) if reverse.count() == 0: # Create the reversal reverse = EqualGoldLink.objects.create(src=obj.dst, dst=obj.src, linktype=obj.linktype) except: bResult = False msg = oErr.get_error_message() return bResult, msg
def calculate_matches(self, ssglists): """Calculate the number of pm-matches for each list from ssglists""" oErr = ErrHandle() lBack = [] try: # Preparation: create a list of SSG ids per list for oItem in ssglists: oItem['ssgid'] = [x['super'] for x in oItem['ssglist']] oItem['unique_matches'] = 0 # Calculate the number of matches for each SSglist for idx_list in range(len(ssglists)): # Take this as the possible pivot list lPivot = ssglists[idx_list]['ssgid'] # Walk all lists that are not this pivot and count matches lUnique = [] oMatches = {} if 'matchset' in ssglists[idx_list]['title']: oMatches = ssglists[idx_list]['title']['matchset'] for idx, setlist in enumerate(ssglists): if idx != idx_list: # Get this ssglist lSsgId = setlist['ssgid'] # Start calculating the number of matches this list has with the currently suggested PM sKey = str(ssglists[idx]['title']['order']) lMatches = [] # Consider all ssg id's in this list for ssg_id in lSsgId: # Global unique matches if ssg_id in lPivot and not ssg_id in lUnique: lUnique.append(ssg_id) # Calculation with respect to the currently suggested PM if ssg_id in lPivot: # and not ssg_id in lMatches: lMatches.append(ssg_id) # Keep track of the matches (for sorting) oMatches[sKey] = len(lMatches) # Store the between-list matches ssglists[idx_list]['title']['matchset'] = oMatches # Store the number of matches for this list unique_matches = len(lUnique) ssglists[idx_list]['unique_matches'] = unique_matches # What we return lBack = ssglists except: msg = oErr.get_error_message() oErr.DoError("ResearchSet/calculate_matches") lBack = None # Return the status return lBack
def update_ssglists(self, force=False): """Re-calculate the set of lists for this particular ResearchSet""" oErr = ErrHandle() bResult = True lst_ssglists = [] try: oPMlist = None # Get the lists of SSGs for each list in the set for idx, setlist in enumerate( self.researchset_setlists.all().order_by('order')): # Check for the contents if force or setlist.contents == "" or len( setlist.contents) < 3 or setlist.contents[0] == "[": setlist.calculate_contents() # Retrieve the SSG-list from the contents oSsgList = json.loads(setlist.contents) # If this is not the *first* setlist, calculate the number of matches with the first if idx == 0: oPMlist = copy.copy(oSsgList) else: # Calculate the matches oSsgList['title']['matches'] = get_list_matches( oPMlist, oSsgList) # Always pass on the default order oSsgList['title']['order'] = idx + 1 # Add the list object to the list lst_ssglists.append(oSsgList) # Calculate the unique_matches for each list lst_ssglists = self.calculate_matches(lst_ssglists) # Put it in the ResearchSet and save it self.contents = json.dumps(lst_ssglists) self.save() # All related SetDef items should be warned with transaction.atomic(): for obj in SetDef.objects.filter(researchset=self.id): contents = json.loads(obj.contents) contents['recalc'] = True obj.contents = json.dumps(contents) obj.save() # Return this list of lists bResult = True except: msg = oErr.get_error_message() oErr.DoError("ResearchSet/update_ssglists") bResult = False return bResult
def adapt_nicknames(): oErr = ErrHandle() bResult = True msg = "" try: # Perform adaptations bResult, msg = SermonDescr.adapt_nicknames() except: bResult = False msg = oErr.get_error_message() return bResult, msg
def adapt_coll_ownerless(): """Find collections without owner and delete these""" oErr = ErrHandle() bResult = True msg = "" name = "Passim" try: qs = Collection.objects.filter(owner__isnull=True) qs.delete() except: bResult = False msg = oErr.get_error_message() return bResult, msg
def add_names(main_list, fragment): oErr = ErrHandle() try: for sentence in fragment.replace("_", "").split("."): # WOrk through non-initial words words = re.split(r'\s+', sentence) for word in words[1:]: if re.match(re_name, word): if not word in main_list: main_list.append(word) except: msg = oErr.get_error_message() oErr.DoError("add_names")
def adapt_sermon_gsig(): oErr = ErrHandle() bResult = True msg = "" try: # Perform adaptations bResult = SermonSignature.adapt_gsig() except: bResult = False msg = oErr.get_error_message() return bResult, msg
def calculate_pm(self): """Calculate the Pivot Manuscript/item for this research set""" oErr = ErrHandle() iBack = -1 try: # Get the lists that we have made ssglists = self.get_ssglists() # Calculate the matches ssglists = self.calculate_matches(ssglists) # Figure out what the first best list is idx_pm = -1 max_matches = -1 min_order = len(ssglists) + 2 min_year_start = 3000 min_year_finish = 3000 for idx, oListItem in enumerate(ssglists): unique_matches = oListItem['unique_matches'] year_start = oListItem['title']['yearstart'] year_finish = oListItem['title']['yearfinish'] order = oListItem['title']['order'] # Check which is the best so far bTakeThis = False bTakeThis = (unique_matches > max_matches) if not bTakeThis and unique_matches == max_matches: bTakeThis = (year_start < min_year_start) if not bTakeThis and year_start == min_year_start: bTakeThis = (year_finish < min_year_finish) if not bTakeThis and year_finish == min_year_finish: bTakeThis = (order < min_order) # Adapt if this is the one if bTakeThis: max_matches = unique_matches min_year_start = year_start min_year_finish = year_finish min_order = order idx_pm = idx # What we return is the 'order' value of the best matching list iBack = ssglists[idx_pm]['title']['order'] except: msg = oErr.get_error_message() oErr.DoError("ResearchSet/calculate_pm") iBack = -1 # Return the PM that we have found return iBack
def adapt_s_to_ssg_link(): oErr = ErrHandle() bResult = True msg = "" try: qs = SermonDescrEqual.objects.all() with transaction.atomic(): for obj in qs: obj.linktype = LINK_UNSPECIFIED obj.save() except: bResult = False msg = oErr.get_error_message() return bResult, msg
def get_watermark(): """Create and return a watermark""" oErr = ErrHandle() watermark_template = "seeker/passim_watermark.html" watermark = "" try: # create a watermark with the right datestamp context_wm = dict( datestamp=get_crpp_date(get_current_datetime(), True)) watermark = render_to_string(watermark_template, context_wm) except: msg = oErr.get_error_message() oErr.DoError("get_watermark") # Return the result return watermark
def adapt_biblerefs(): oErr = ErrHandle() bResult = True msg = "" try: # Remove any previous BibRange objects BibRange.objects.all().delete() # Perform adaptations for sermon in SermonDescr.objects.exclude(bibleref__isnull=True).exclude(bibleref__exact=''): sermon.do_ranges(force=True) except: bResult = False msg = oErr.get_error_message() return bResult, msg
def get_ssg_list(self): """Create a list of SSGs,depending on the type I am""" oErr = ErrHandle() lBack = None collection_types = ['hist', 'ssgd'] try: if self.setlisttype == "manu": # Create a list of SSG's from the manuscript lBack = self.ssgs_manuscript(self.manuscript) elif self.setlisttype in collection_types: lBack = self.ssgs_collection(self.collection) except: msg = oErr.get_error_message() oErr.DoError("SetList/get_ssg_list") return lBack
def adapt_author_anonymus(): oErr = ErrHandle() bResult = True msg = "" try: # Get all SSGs with anyonymus with transaction.atomic(): ano = "anonymus" qs = EqualGold.objects.filter(Q(author__name__iexact=ano)) for ssg in qs: ssg.save() except: bResult = False msg = oErr.get_error_message() return bResult, msg
def get_ssg_passim(ssg_id, obj=None): oErr = ErrHandle() code = "" try: # Get the Passim Code if obj == None: obj = EqualGold.objects.filter(id=ssg_id).first() if obj.code == None: code = "eqg_{}".format(obj.id) elif " " in obj.code: code = obj.code.split(" ")[1] else: code = obj.code except: msg = oErr.get_error_message() oErr.DoError("get_ssg_passim") return code
def adapt_hccount(): oErr = ErrHandle() bResult = True msg = "" try: # Walk all SSGs with transaction.atomic(): for ssg in EqualGold.objects.all(): hccount = ssg.collections.filter(settype="hc").count() if hccount != ssg.hccount: ssg.hccount = hccount ssg.save() except: bResult = False msg = oErr.get_error_message() return bResult, msg
def adapt_ssgcount(): oErr = ErrHandle() bResult = True msg = "" try: # Walk all SSGs with transaction.atomic(): for ssg in EqualGold.objects.all(): ssgcount = ssg.relations.count() if ssgcount != ssg.ssgcount: ssg.ssgcount = ssgcount ssg.save() except: bResult = False msg = oErr.get_error_message() return bResult, msg
def get_galway(self, catalogue_id): # Read the manuscript with the indicated id oBack = None bResult = False oErr = ErrHandle() try: url = "https://elmss.nuigalway.ie/api/v1/catalogue/{}".format( catalogue_id) try: r = requests.get(url) except: sMsg = oErr.get_error_message() oErr.DoError("Request problem") return False, sMsg if r.status_code == 200: # Read the response sText = r.text oBack = json.loads(sText) bResult = True else: bResult = False sResult = "download_file received status {} for {}".format( r.status_code, url) oErr.Status("get_galway reading error: {}".format(sResult)) except: msg = oErr.get_error_message() oErr.DoError("get_galway") oBack = None return oBack
def add_item(super, profile, field, oChange, oCurrent=None): """Add one item""" oErr = ErrHandle() obj = None try: # Make sure to stringify, sorting the keys change = json.dumps(oChange, sort_keys=True) if oCurrent is None: current = None else: current = json.dumps(oCurrent, sort_keys=True) # Look for this particular change, supposing it has not changed yet obj = EqualChange.objects.filter(super=super, profile=profile, field=field, current=current, change=change).first() if obj == None or obj.changeapprovals.count() > 0: # Less restricted: look for any suggestion for a change on this field that has not been reviewed by anyone yet. bFound = False for obj in EqualChange.objects.filter(super=super, profile=profile, field=field, atype="def"): if obj.changeapprovals.exclude(atype="def").count() == 0: # We can use this one bFound = True obj.current = current obj.change = change obj.save() break # What if nothing has been found? if not bFound: # Only in that case do we make a new suggestion obj = EqualChange.objects.create(super=super, profile=profile, field=field, current=current, change=change) except: msg = oErr.get_error_message() oErr.DoError("EqualChange/add_item") return obj
def adapt_passim_code(): oErr = ErrHandle() bResult = True msg = "" try: # Walk all SSGs need_correction = {} for obj in EqualGold.objects.all().order_by("-id"): code = obj.code if code != None and code != "ZZZ_DETERMINE": count = EqualGold.objects.filter(code=code).count() if count > 1: oErr.Status("Duplicate code={} id={}".format(code, obj.id)) if code in need_correction: need_correction[code].append(obj.id) else: need_correction[code] = [obj.id] oErr.Status(json.dumps(need_correction)) for k,v in need_correction.items(): code = k ssg_list = v for ssg_id in ssg_list[:-1]: oErr.Status("Changing CODE for id {}".format(ssg_id)) obj = EqualGold.objects.filter(id=ssg_id).first() if obj != None: obj.code = None obj.number = None obj.save() oErr.Status("Re-saved id {}, code is now: {}".format(obj.id, obj.code)) except: bResult = False msg = oErr.get_error_message() return bResult, msg
def adapt_templatecleanup(): oErr = ErrHandle() bResult = True msg = "" try: # Get a list of all the templates and the manuscript id's in it template_manu_id = [x.manu.id for x in Template.objects.all().order_by('manu__id')] # Get all manuscripts that are supposed to be template, but whose ID is not in [templat_manu_id] qs_manu = Manuscript.objects.filter(mtype='tem').exclude(id__in=template_manu_id) # Remove these manuscripts (and their associated msitems, sermondescr, sermonhead qs_manu.delete() except: bResult = False msg = oErr.get_error_message() return bResult, msg
def listview_adaptations(lv): """Perform adaptations specific for this listview""" oErr = ErrHandle() try: if lv in adaptation_list: for adapt in adaptation_list.get(lv): sh_done = Information.get_kvalue(adapt) if sh_done == None or sh_done != "done": # Do the adaptation, depending on what it is method_to_call = "adapt_{}".format(adapt) bResult, msg = globals()[method_to_call]() if bResult: # Success Information.set_kvalue(adapt, "done") except: msg = oErr.get_error_message() oErr.DoError("listview_adaptations")
def adapt_manuprov_m2m(): oErr = ErrHandle() bResult = True msg = "" try: # Issue #289: back to m2m connection prov_changes = 0 prov_added = 0 # Keep a list of provenance id's that may be kept keep_id = [] # Remove all previous ProvenanceMan connections ProvenanceMan.objects.all().delete() # Get all the manuscripts for manu in Manuscript.objects.all(): # Treat all the M2O provenances for this manuscript for prov in manu.manuprovenances.all(): # Get the *name* and the *loc* for this prov name = prov.name loc = prov.location note = prov.note # Get the very *first* provenance with name/loc firstprov = Provenance.objects.filter(name__iexact=name, location=loc).first() if firstprov == None: # Create one firstprov = Provenance.objects.create(name=name, location=loc) keep_id.append(firstprov.id) # Add the link link = ProvenanceMan.objects.create(manuscript=manu, provenance=firstprov, note=note) # Walk all provenances to remove the unused ones delete_id = [] for prov in Provenance.objects.all().values('id'): if not prov['id'] in keep_id: delete_id.append(prov['id']) oErr.Status("Deleting provenances: {}".format(len(delete_id))) Provenance.objects.filter(id__in=delete_id).delete() # Success oErr.Status("adapt_manuprov_m2m: {} changes, {} additions".format(prov_changes, prov_added)) except: bResult = False msg = oErr.get_error_message() return bResult, msg