def adapt_passim_project_name_manu():
oErr = ErrHandle()
bResult = True
msg = ""
name = "Passim"
# Issue 412: give all items in the current database the Project name "Passim"
# This means: Manuscript, SSG/AF, Historical collection
# Sermon: NO, is done in a separate method
try:
# Iterate over all Manuscripts in the database:
for manu in Manuscript.objects.all():
# Try to find if the project already exists:
projectfound = Project2.objects.filter(name__iexact = name).first()
if projectfound == None:
# If the project does not already exist, it needs to be added to the database
project = Project2.objects.create(name = name)
# And a link should be made between this new project and the manuscript
ManuscriptProject.objects.create(project = project, manuscript = manu)
else:
manuprojlink = ManuscriptProject.objects.filter(project = projectfound, manuscript = manu).first()
if manuprojlink == None:
# If the project already exists, but not the link, than only a link should be
# made between the manuscript and the project
ManuscriptProject.objects.create(project = projectfound, manuscript = manu)
#print(manu, projectfound)
# When iterating over Manuscript, make sure to iterate over Sermons too, how to access them?
# Sermons belong to 1 manu
# take project of Manuscript,
# SermonDescrProject (project = projectmanu, sermon = sermon)
except:
bResult = False
msg = oErr.get_error_message()
return bResult, msg
def __init__(self, *args, **kwargs):
self.username = kwargs.pop('username', "")
self.team_group = kwargs.pop('team_group', "")
self.userplus = kwargs.pop('userplus', "")
# Start by executing the standard handling
super(EqualApprovalForm, self).__init__(*args, **kwargs)
oErr = ErrHandle()
try:
# Some fields are not required
self.fields['change'].required = False
self.fields['comment'].required = False
self.fields['profile'].required = False
self.fields['atype'].required = False
# Set queryset(s) - for details view
self.fields['profilelist'].queryset = Profile.objects.all(
).order_by('user__username')
self.fields['passimlist'].queryset = EqualGold.objects.filter(
code__isnull=False, moved__isnull=True).order_by('code')
self.fields['atypelist'].queryset = FieldChoice.objects.filter(
field=APPROVAL_TYPE).order_by("english_name")
# Get the instance
if 'instance' in kwargs:
instance = kwargs['instance']
except:
msg = oErr.get_error_message()
oErr.DoError("EqualApprovalForm")
return None
def adapt_codico_origin():
oErr = ErrHandle()
bResult = True
msg = ""
name = "Passim"
# Issue #427:
# Create one OriginCod object for each Origin that is part of a Codico
# Then set the 'origin' field of that Codico to None
try:
qs = Codico.objects.all()
lst_codico = []
with transaction.atomic():
for codico in qs:
# Does this one have an origin set?
if not codico.origin is None:
# Check if an appropriate OriginCod is already there
obj = OriginCod.objects.filter(codico=codico).first()
if obj is None:
# Create one
obj = OriginCod.objects.create(codico=codico, origin=codico.origin)
lst_codico.append(codico)
# Set the 'origin' field to none
with transaction.atomic():
for codico in lst_codico:
codico.origin = None
codico.save()
except:
bResult = False
msg = oErr.get_error_message()
return bResult, msg
def adapt_passim_project_name_equal():
oErr = ErrHandle()
bResult = True
msg = ""
name = "Passim"
# Issue 412: give all items in the current database the Project name "Passim"
# This means: Manuscript, SSG/AF, Historical collection
try:
# Iterate over all SSGs/AFs in the database:
for equal in EqualGold.objects.all():
# Try to find if the project already exists:
projectfound = Project2.objects.filter(name__iexact = name).first()
if projectfound == None:
# If the project does not already exist, it needs to be added to the database
project = Project2.objects.create(name = name)
# And a link should be made between this new project and the SSG/AF
EqualGoldProject.objects.create(project = project, equal = equal)
else:
equalprojlink = EqualGoldProject.objects.filter(project = projectfound, equal = equal).first()
if equalprojlink == None:
# If the project already exists, but not the link, than only a link should be
# made between the collection and the project
EqualGoldProject.objects.create(project = projectfound, equal = equal)
print(equal, projectfound)
except:
bResult = False
msg = oErr.get_error_message()
return bResult, msg
def get_ssglists(self, recalculate=False):
"""Get the set of lists for this particular ResearchSet"""
oErr = ErrHandle()
lBack = None
try:
if self.contents != "" and self.contents[0] == "[":
lst_contents = json.loads(self.contents)
else:
lst_contents = []
if not recalculate and len(lst_contents) > 0:
# Should the unique_matches be re-calculated?
if not 'unique_matches' in lst_contents[0]:
# Yes, re-calculate
lst_contents = self.calculate_matches(lst_contents)
# And make sure this gets saved!
self.contents = json.dumps(lst_contents)
self.save()
lBack = lst_contents
else:
# Re-calculate the lists
self.update_ssglists()
# Get the result
lBack = json.loads(self.contents)
except:
msg = oErr.get_error_message()
oErr.DoError("ResearchSet/get_ssglists")
return lBack
def get_review_list(profile, all=False):
"""Get the list of objects this editor needs to review"""
oErr = ErrHandle()
lstBack = []
try:
# Default: return the empty list
# lstBack = EqualChange.objects.none()
# Get the list of projects for this user
lst_project_id = profile.projects.all().values("id")
if len(lst_project_id) > 0:
# Get the list of EqualChange objects linked to any of these projects
lstQ = []
lstQ.append(
Q(super__equal_proj__project__id__in=lst_project_id))
if not all:
lstQ.append(Q(atype='def'))
lstBack = [
x['id'] for x in EqualChange.objects.exclude(
profile=profile).filter(*lstQ).distinct().values('id')
]
# lstBack = EqualChange.objects.exclude(profile=profile).filter(*lstQ).distinct()
except:
msg = oErr.get_error_message()
oErr.DoError("EqualChange/get_review_list")
return lstBack
def get_approver_list(self, excl=None):
"""Get the list of editors that need to approve this change
If [excl] is specified, then this object is excluded from the list of Profile objects returned
"""
oErr = ErrHandle()
lstBack = None
try:
# Default: return the empty list
lstBack = Profile.objects.none()
# Get all the projects to which this SSG 'belongs'
lst_project = [
x['id'] for x in self.super.projects.all().values("id")
]
# Note: only SSGs that belong to more than one project need to be reviewed
if len(lst_project) > 1:
# Get all the editors associated with these projects
lst_profile_id = [
x['profile_id'] for x in ProjectEditor.objects.filter(
project__id__in=lst_project).values(
'profile_id').distinct()
]
if len(lst_profile_id) > 0:
if excl == None:
lstBack = Profile.objects.filter(id__in=lst_profile_id)
else:
lstBack = Profile.objects.filter(
id__in=lst_profile_id).exclude(id=excl.id)
except:
msg = oErr.get_error_message()
oErr.DoError("EqualChange/get_approver_list")
return lstBack
def get_testsets(request):
"""Get a list of testunits per testset of one round"""
oErr = ErrHandle()
try:
data = 'fail'
qd = request.GET if request.method == "GET" else request.POST
if request.is_ajax():
round = qd.get("round", "")
lstQ = []
lstQ.append(Q(testset__round=round))
items = TestsetUnit.objects.filter(*lstQ).order_by("testset__number").values(
'testset__round', 'testset__number', 'testunit__speaker__name',
'testunit__speaker__gender', 'testunit__fname',
'testunit__sentence__name', 'testunit__ntype')
results = []
for idx, obj in enumerate(items):
number = obj.get('testset__number')
speaker = obj.get('testunit__speaker__name')
gender = obj.get('testunit__speaker__gender')
fname = obj.get('testunit__sentence__name')
sentence = obj.get('testunit__fname')
ntype = "Lom" if obj.get('testunit__ntype') == "n" else "Nat"
co_json = {'idx': idx+1, 'testset': number, 'speaker': speaker, 'gender': gender,
'filename': fname, 'sentence': sentence, 'ntype': ntype }
results.append(co_json)
data = json.dumps(results)
else:
data = "Request is not ajax"
except:
msg = oErr.get_error_message()
data = "error: {}".format(msg)
mimetype = "application/json"
return HttpResponse(data, mimetype)
def adapt_ssg_bidirectional():
oErr = ErrHandle()
bResult = True
msg = ""
try:
# Put all links in a list
lst_link = []
lst_remove = []
lst_reverse = []
for obj in EqualGoldLink.objects.filter(linktype__in=LINK_BIDIR):
# Check for any other eqg-links with the same source
lst_src = EqualGoldLink.objects.filter(src=obj.src, dst=obj.dst).exclude(id=obj.id)
if lst_src.count() > 0:
# Add them to the removal
for item in lst_src:
lst_remove.append(item)
else:
# Add the obj to the list
lst_link.append(obj)
for obj in lst_link:
# Find the reverse link
reverse = EqualGoldLink.objects.filter(src=obj.dst, dst=obj.src)
if reverse.count() == 0:
# Create the reversal
reverse = EqualGoldLink.objects.create(src=obj.dst, dst=obj.src, linktype=obj.linktype)
except:
bResult = False
msg = oErr.get_error_message()
return bResult, msg
def calculate_matches(self, ssglists):
"""Calculate the number of pm-matches for each list from ssglists"""
oErr = ErrHandle()
lBack = []
try:
# Preparation: create a list of SSG ids per list
for oItem in ssglists:
oItem['ssgid'] = [x['super'] for x in oItem['ssglist']]
oItem['unique_matches'] = 0
# Calculate the number of matches for each SSglist
for idx_list in range(len(ssglists)):
# Take this as the possible pivot list
lPivot = ssglists[idx_list]['ssgid']
# Walk all lists that are not this pivot and count matches
lUnique = []
oMatches = {}
if 'matchset' in ssglists[idx_list]['title']:
oMatches = ssglists[idx_list]['title']['matchset']
for idx, setlist in enumerate(ssglists):
if idx != idx_list:
# Get this ssglist
lSsgId = setlist['ssgid']
# Start calculating the number of matches this list has with the currently suggested PM
sKey = str(ssglists[idx]['title']['order'])
lMatches = []
# Consider all ssg id's in this list
for ssg_id in lSsgId:
# Global unique matches
if ssg_id in lPivot and not ssg_id in lUnique:
lUnique.append(ssg_id)
# Calculation with respect to the currently suggested PM
if ssg_id in lPivot: # and not ssg_id in lMatches:
lMatches.append(ssg_id)
# Keep track of the matches (for sorting)
oMatches[sKey] = len(lMatches)
# Store the between-list matches
ssglists[idx_list]['title']['matchset'] = oMatches
# Store the number of matches for this list
unique_matches = len(lUnique)
ssglists[idx_list]['unique_matches'] = unique_matches
# What we return
lBack = ssglists
except:
msg = oErr.get_error_message()
oErr.DoError("ResearchSet/calculate_matches")
lBack = None
# Return the status
return lBack
def update_ssglists(self, force=False):
"""Re-calculate the set of lists for this particular ResearchSet"""
oErr = ErrHandle()
bResult = True
lst_ssglists = []
try:
oPMlist = None
# Get the lists of SSGs for each list in the set
for idx, setlist in enumerate(
self.researchset_setlists.all().order_by('order')):
# Check for the contents
if force or setlist.contents == "" or len(
setlist.contents) < 3 or setlist.contents[0] == "[":
setlist.calculate_contents()
# Retrieve the SSG-list from the contents
oSsgList = json.loads(setlist.contents)
# If this is not the *first* setlist, calculate the number of matches with the first
if idx == 0:
oPMlist = copy.copy(oSsgList)
else:
# Calculate the matches
oSsgList['title']['matches'] = get_list_matches(
oPMlist, oSsgList)
# Always pass on the default order
oSsgList['title']['order'] = idx + 1
# Add the list object to the list
lst_ssglists.append(oSsgList)
# Calculate the unique_matches for each list
lst_ssglists = self.calculate_matches(lst_ssglists)
# Put it in the ResearchSet and save it
self.contents = json.dumps(lst_ssglists)
self.save()
# All related SetDef items should be warned
with transaction.atomic():
for obj in SetDef.objects.filter(researchset=self.id):
contents = json.loads(obj.contents)
contents['recalc'] = True
obj.contents = json.dumps(contents)
obj.save()
# Return this list of lists
bResult = True
except:
msg = oErr.get_error_message()
oErr.DoError("ResearchSet/update_ssglists")
bResult = False
return bResult
def adapt_nicknames():
oErr = ErrHandle()
bResult = True
msg = ""
try:
# Perform adaptations
bResult, msg = SermonDescr.adapt_nicknames()
except:
bResult = False
msg = oErr.get_error_message()
return bResult, msg
def adapt_coll_ownerless():
"""Find collections without owner and delete these"""
oErr = ErrHandle()
bResult = True
msg = ""
name = "Passim"
try:
qs = Collection.objects.filter(owner__isnull=True)
qs.delete()
except:
bResult = False
msg = oErr.get_error_message()
return bResult, msg
def add_names(main_list, fragment):
oErr = ErrHandle()
try:
for sentence in fragment.replace("_", "").split("."):
# WOrk through non-initial words
words = re.split(r'\s+', sentence)
for word in words[1:]:
if re.match(re_name, word):
if not word in main_list:
main_list.append(word)
except:
msg = oErr.get_error_message()
oErr.DoError("add_names")
def adapt_sermon_gsig():
oErr = ErrHandle()
bResult = True
msg = ""
try:
# Perform adaptations
bResult = SermonSignature.adapt_gsig()
except:
bResult = False
msg = oErr.get_error_message()
return bResult, msg
def calculate_pm(self):
"""Calculate the Pivot Manuscript/item for this research set"""
oErr = ErrHandle()
iBack = -1
try:
# Get the lists that we have made
ssglists = self.get_ssglists()
# Calculate the matches
ssglists = self.calculate_matches(ssglists)
# Figure out what the first best list is
idx_pm = -1
max_matches = -1
min_order = len(ssglists) + 2
min_year_start = 3000
min_year_finish = 3000
for idx, oListItem in enumerate(ssglists):
unique_matches = oListItem['unique_matches']
year_start = oListItem['title']['yearstart']
year_finish = oListItem['title']['yearfinish']
order = oListItem['title']['order']
# Check which is the best so far
bTakeThis = False
bTakeThis = (unique_matches > max_matches)
if not bTakeThis and unique_matches == max_matches:
bTakeThis = (year_start < min_year_start)
if not bTakeThis and year_start == min_year_start:
bTakeThis = (year_finish < min_year_finish)
if not bTakeThis and year_finish == min_year_finish:
bTakeThis = (order < min_order)
# Adapt if this is the one
if bTakeThis:
max_matches = unique_matches
min_year_start = year_start
min_year_finish = year_finish
min_order = order
idx_pm = idx
# What we return is the 'order' value of the best matching list
iBack = ssglists[idx_pm]['title']['order']
except:
msg = oErr.get_error_message()
oErr.DoError("ResearchSet/calculate_pm")
iBack = -1
# Return the PM that we have found
return iBack
def adapt_s_to_ssg_link():
oErr = ErrHandle()
bResult = True
msg = ""
try:
qs = SermonDescrEqual.objects.all()
with transaction.atomic():
for obj in qs:
obj.linktype = LINK_UNSPECIFIED
obj.save()
except:
bResult = False
msg = oErr.get_error_message()
return bResult, msg
def get_watermark():
"""Create and return a watermark"""
oErr = ErrHandle()
watermark_template = "seeker/passim_watermark.html"
watermark = ""
try:
# create a watermark with the right datestamp
context_wm = dict(
datestamp=get_crpp_date(get_current_datetime(), True))
watermark = render_to_string(watermark_template, context_wm)
except:
msg = oErr.get_error_message()
oErr.DoError("get_watermark")
# Return the result
return watermark
def adapt_biblerefs():
oErr = ErrHandle()
bResult = True
msg = ""
try:
# Remove any previous BibRange objects
BibRange.objects.all().delete()
# Perform adaptations
for sermon in SermonDescr.objects.exclude(bibleref__isnull=True).exclude(bibleref__exact=''):
sermon.do_ranges(force=True)
except:
bResult = False
msg = oErr.get_error_message()
return bResult, msg
def get_ssg_list(self):
"""Create a list of SSGs,depending on the type I am"""
oErr = ErrHandle()
lBack = None
collection_types = ['hist', 'ssgd']
try:
if self.setlisttype == "manu":
# Create a list of SSG's from the manuscript
lBack = self.ssgs_manuscript(self.manuscript)
elif self.setlisttype in collection_types:
lBack = self.ssgs_collection(self.collection)
except:
msg = oErr.get_error_message()
oErr.DoError("SetList/get_ssg_list")
return lBack
def adapt_author_anonymus():
oErr = ErrHandle()
bResult = True
msg = ""
try:
# Get all SSGs with anyonymus
with transaction.atomic():
ano = "anonymus"
qs = EqualGold.objects.filter(Q(author__name__iexact=ano))
for ssg in qs:
ssg.save()
except:
bResult = False
msg = oErr.get_error_message()
return bResult, msg
def get_ssg_passim(ssg_id, obj=None):
oErr = ErrHandle()
code = ""
try:
# Get the Passim Code
if obj == None:
obj = EqualGold.objects.filter(id=ssg_id).first()
if obj.code == None:
code = "eqg_{}".format(obj.id)
elif " " in obj.code:
code = obj.code.split(" ")[1]
else:
code = obj.code
except:
msg = oErr.get_error_message()
oErr.DoError("get_ssg_passim")
return code
def adapt_hccount():
oErr = ErrHandle()
bResult = True
msg = ""
try:
# Walk all SSGs
with transaction.atomic():
for ssg in EqualGold.objects.all():
hccount = ssg.collections.filter(settype="hc").count()
if hccount != ssg.hccount:
ssg.hccount = hccount
ssg.save()
except:
bResult = False
msg = oErr.get_error_message()
return bResult, msg
def adapt_ssgcount():
oErr = ErrHandle()
bResult = True
msg = ""
try:
# Walk all SSGs
with transaction.atomic():
for ssg in EqualGold.objects.all():
ssgcount = ssg.relations.count()
if ssgcount != ssg.ssgcount:
ssg.ssgcount = ssgcount
ssg.save()
except:
bResult = False
msg = oErr.get_error_message()
return bResult, msg
def get_galway(self, catalogue_id):
# Read the manuscript with the indicated id
oBack = None
bResult = False
oErr = ErrHandle()
try:
url = "https://elmss.nuigalway.ie/api/v1/catalogue/{}".format(
catalogue_id)
try:
r = requests.get(url)
except:
sMsg = oErr.get_error_message()
oErr.DoError("Request problem")
return False, sMsg
if r.status_code == 200:
# Read the response
sText = r.text
oBack = json.loads(sText)
bResult = True
else:
bResult = False
sResult = "download_file received status {} for {}".format(
r.status_code, url)
oErr.Status("get_galway reading error: {}".format(sResult))
except:
msg = oErr.get_error_message()
oErr.DoError("get_galway")
oBack = None
return oBack
def add_item(super, profile, field, oChange, oCurrent=None):
"""Add one item"""
oErr = ErrHandle()
obj = None
try:
# Make sure to stringify, sorting the keys
change = json.dumps(oChange, sort_keys=True)
if oCurrent is None:
current = None
else:
current = json.dumps(oCurrent, sort_keys=True)
# Look for this particular change, supposing it has not changed yet
obj = EqualChange.objects.filter(super=super,
profile=profile,
field=field,
current=current,
change=change).first()
if obj == None or obj.changeapprovals.count() > 0:
# Less restricted: look for any suggestion for a change on this field that has not been reviewed by anyone yet.
bFound = False
for obj in EqualChange.objects.filter(super=super,
profile=profile,
field=field,
atype="def"):
if obj.changeapprovals.exclude(atype="def").count() == 0:
# We can use this one
bFound = True
obj.current = current
obj.change = change
obj.save()
break
# What if nothing has been found?
if not bFound:
# Only in that case do we make a new suggestion
obj = EqualChange.objects.create(super=super,
profile=profile,
field=field,
current=current,
change=change)
except:
msg = oErr.get_error_message()
oErr.DoError("EqualChange/add_item")
return obj
def adapt_passim_code():
oErr = ErrHandle()
bResult = True
msg = ""
try:
# Walk all SSGs
need_correction = {}
for obj in EqualGold.objects.all().order_by("-id"):
code = obj.code
if code != None and code != "ZZZ_DETERMINE":
count = EqualGold.objects.filter(code=code).count()
if count > 1:
oErr.Status("Duplicate code={} id={}".format(code, obj.id))
if code in need_correction:
need_correction[code].append(obj.id)
else:
need_correction[code] = [obj.id]
oErr.Status(json.dumps(need_correction))
for k,v in need_correction.items():
code = k
ssg_list = v
for ssg_id in ssg_list[:-1]:
oErr.Status("Changing CODE for id {}".format(ssg_id))
obj = EqualGold.objects.filter(id=ssg_id).first()
if obj != None:
obj.code = None
obj.number = None
obj.save()
oErr.Status("Re-saved id {}, code is now: {}".format(obj.id, obj.code))
except:
bResult = False
msg = oErr.get_error_message()
return bResult, msg
def adapt_templatecleanup():
oErr = ErrHandle()
bResult = True
msg = ""
try:
# Get a list of all the templates and the manuscript id's in it
template_manu_id = [x.manu.id for x in Template.objects.all().order_by('manu__id')]
# Get all manuscripts that are supposed to be template, but whose ID is not in [templat_manu_id]
qs_manu = Manuscript.objects.filter(mtype='tem').exclude(id__in=template_manu_id)
# Remove these manuscripts (and their associated msitems, sermondescr, sermonhead
qs_manu.delete()
except:
bResult = False
msg = oErr.get_error_message()
return bResult, msg
def listview_adaptations(lv):
"""Perform adaptations specific for this listview"""
oErr = ErrHandle()
try:
if lv in adaptation_list:
for adapt in adaptation_list.get(lv):
sh_done = Information.get_kvalue(adapt)
if sh_done == None or sh_done != "done":
# Do the adaptation, depending on what it is
method_to_call = "adapt_{}".format(adapt)
bResult, msg = globals()[method_to_call]()
if bResult:
# Success
Information.set_kvalue(adapt, "done")
except:
msg = oErr.get_error_message()
oErr.DoError("listview_adaptations")
def adapt_manuprov_m2m():
oErr = ErrHandle()
bResult = True
msg = ""
try:
# Issue #289: back to m2m connection
prov_changes = 0
prov_added = 0
# Keep a list of provenance id's that may be kept
keep_id = []
# Remove all previous ProvenanceMan connections
ProvenanceMan.objects.all().delete()
# Get all the manuscripts
for manu in Manuscript.objects.all():
# Treat all the M2O provenances for this manuscript
for prov in manu.manuprovenances.all():
# Get the *name* and the *loc* for this prov
name = prov.name
loc = prov.location
note = prov.note
# Get the very *first* provenance with name/loc
firstprov = Provenance.objects.filter(name__iexact=name, location=loc).first()
if firstprov == None:
# Create one
firstprov = Provenance.objects.create(name=name, location=loc)
keep_id.append(firstprov.id)
# Add the link
link = ProvenanceMan.objects.create(manuscript=manu, provenance=firstprov, note=note)
# Walk all provenances to remove the unused ones
delete_id = []
for prov in Provenance.objects.all().values('id'):
if not prov['id'] in keep_id:
delete_id.append(prov['id'])
oErr.Status("Deleting provenances: {}".format(len(delete_id)))
Provenance.objects.filter(id__in=delete_id).delete()
# Success
oErr.Status("adapt_manuprov_m2m: {} changes, {} additions".format(prov_changes, prov_added))
except:
bResult = False
msg = oErr.get_error_message()
return bResult, msg