def get_galway(self, catalogue_id):
# Read the manuscript with the indicated id
oBack = None
bResult = False
oErr = ErrHandle()
try:
url = "https://elmss.nuigalway.ie/api/v1/catalogue/{}".format(
catalogue_id)
try:
r = requests.get(url)
except:
sMsg = oErr.get_error_message()
oErr.DoError("Request problem")
return False, sMsg
if r.status_code == 200:
# Read the response
sText = r.text
oBack = json.loads(sText)
bResult = True
else:
bResult = False
sResult = "download_file received status {} for {}".format(
r.status_code, url)
oErr.Status("get_galway reading error: {}".format(sResult))
except:
msg = oErr.get_error_message()
oErr.DoError("get_galway")
oBack = None
return oBack
def get_ssglists(self, recalculate=False):
"""Get the set of lists for this particular ResearchSet"""
oErr = ErrHandle()
lBack = None
try:
if self.contents != "" and self.contents[0] == "[":
lst_contents = json.loads(self.contents)
else:
lst_contents = []
if not recalculate and len(lst_contents) > 0:
# Should the unique_matches be re-calculated?
if not 'unique_matches' in lst_contents[0]:
# Yes, re-calculate
lst_contents = self.calculate_matches(lst_contents)
# And make sure this gets saved!
self.contents = json.dumps(lst_contents)
self.save()
lBack = lst_contents
else:
# Re-calculate the lists
self.update_ssglists()
# Get the result
lBack = json.loads(self.contents)
except:
msg = oErr.get_error_message()
oErr.DoError("ResearchSet/get_ssglists")
return lBack
def __init__(self, *args, **kwargs):
self.username = kwargs.pop('username', "")
self.team_group = kwargs.pop('team_group', "")
self.userplus = kwargs.pop('userplus', "")
# Start by executing the standard handling
super(EqualApprovalForm, self).__init__(*args, **kwargs)
oErr = ErrHandle()
try:
# Some fields are not required
self.fields['change'].required = False
self.fields['comment'].required = False
self.fields['profile'].required = False
self.fields['atype'].required = False
# Set queryset(s) - for details view
self.fields['profilelist'].queryset = Profile.objects.all(
).order_by('user__username')
self.fields['passimlist'].queryset = EqualGold.objects.filter(
code__isnull=False, moved__isnull=True).order_by('code')
self.fields['atypelist'].queryset = FieldChoice.objects.filter(
field=APPROVAL_TYPE).order_by("english_name")
# Get the instance
if 'instance' in kwargs:
instance = kwargs['instance']
except:
msg = oErr.get_error_message()
oErr.DoError("EqualApprovalForm")
return None
def get_approver_list(self, excl=None):
"""Get the list of editors that need to approve this change
If [excl] is specified, then this object is excluded from the list of Profile objects returned
"""
oErr = ErrHandle()
lstBack = None
try:
# Default: return the empty list
lstBack = Profile.objects.none()
# Get all the projects to which this SSG 'belongs'
lst_project = [
x['id'] for x in self.super.projects.all().values("id")
]
# Note: only SSGs that belong to more than one project need to be reviewed
if len(lst_project) > 1:
# Get all the editors associated with these projects
lst_profile_id = [
x['profile_id'] for x in ProjectEditor.objects.filter(
project__id__in=lst_project).values(
'profile_id').distinct()
]
if len(lst_profile_id) > 0:
if excl == None:
lstBack = Profile.objects.filter(id__in=lst_profile_id)
else:
lstBack = Profile.objects.filter(
id__in=lst_profile_id).exclude(id=excl.id)
except:
msg = oErr.get_error_message()
oErr.DoError("EqualChange/get_approver_list")
return lstBack
def get_review_list(profile, all=False):
"""Get the list of objects this editor needs to review"""
oErr = ErrHandle()
lstBack = []
try:
# Default: return the empty list
# lstBack = EqualChange.objects.none()
# Get the list of projects for this user
lst_project_id = profile.projects.all().values("id")
if len(lst_project_id) > 0:
# Get the list of EqualChange objects linked to any of these projects
lstQ = []
lstQ.append(
Q(super__equal_proj__project__id__in=lst_project_id))
if not all:
lstQ.append(Q(atype='def'))
lstBack = [
x['id'] for x in EqualChange.objects.exclude(
profile=profile).filter(*lstQ).distinct().values('id')
]
# lstBack = EqualChange.objects.exclude(profile=profile).filter(*lstQ).distinct()
except:
msg = oErr.get_error_message()
oErr.DoError("EqualChange/get_review_list")
return lstBack
def calculate_matches(self, ssglists):
"""Calculate the number of pm-matches for each list from ssglists"""
oErr = ErrHandle()
lBack = []
try:
# Preparation: create a list of SSG ids per list
for oItem in ssglists:
oItem['ssgid'] = [x['super'] for x in oItem['ssglist']]
oItem['unique_matches'] = 0
# Calculate the number of matches for each SSglist
for idx_list in range(len(ssglists)):
# Take this as the possible pivot list
lPivot = ssglists[idx_list]['ssgid']
# Walk all lists that are not this pivot and count matches
lUnique = []
oMatches = {}
if 'matchset' in ssglists[idx_list]['title']:
oMatches = ssglists[idx_list]['title']['matchset']
for idx, setlist in enumerate(ssglists):
if idx != idx_list:
# Get this ssglist
lSsgId = setlist['ssgid']
# Start calculating the number of matches this list has with the currently suggested PM
sKey = str(ssglists[idx]['title']['order'])
lMatches = []
# Consider all ssg id's in this list
for ssg_id in lSsgId:
# Global unique matches
if ssg_id in lPivot and not ssg_id in lUnique:
lUnique.append(ssg_id)
# Calculation with respect to the currently suggested PM
if ssg_id in lPivot: # and not ssg_id in lMatches:
lMatches.append(ssg_id)
# Keep track of the matches (for sorting)
oMatches[sKey] = len(lMatches)
# Store the between-list matches
ssglists[idx_list]['title']['matchset'] = oMatches
# Store the number of matches for this list
unique_matches = len(lUnique)
ssglists[idx_list]['unique_matches'] = unique_matches
# What we return
lBack = ssglists
except:
msg = oErr.get_error_message()
oErr.DoError("ResearchSet/calculate_matches")
lBack = None
# Return the status
return lBack
def update_ssglists(self, force=False):
"""Re-calculate the set of lists for this particular ResearchSet"""
oErr = ErrHandle()
bResult = True
lst_ssglists = []
try:
oPMlist = None
# Get the lists of SSGs for each list in the set
for idx, setlist in enumerate(
self.researchset_setlists.all().order_by('order')):
# Check for the contents
if force or setlist.contents == "" or len(
setlist.contents) < 3 or setlist.contents[0] == "[":
setlist.calculate_contents()
# Retrieve the SSG-list from the contents
oSsgList = json.loads(setlist.contents)
# If this is not the *first* setlist, calculate the number of matches with the first
if idx == 0:
oPMlist = copy.copy(oSsgList)
else:
# Calculate the matches
oSsgList['title']['matches'] = get_list_matches(
oPMlist, oSsgList)
# Always pass on the default order
oSsgList['title']['order'] = idx + 1
# Add the list object to the list
lst_ssglists.append(oSsgList)
# Calculate the unique_matches for each list
lst_ssglists = self.calculate_matches(lst_ssglists)
# Put it in the ResearchSet and save it
self.contents = json.dumps(lst_ssglists)
self.save()
# All related SetDef items should be warned
with transaction.atomic():
for obj in SetDef.objects.filter(researchset=self.id):
contents = json.loads(obj.contents)
contents['recalc'] = True
obj.contents = json.dumps(contents)
obj.save()
# Return this list of lists
bResult = True
except:
msg = oErr.get_error_message()
oErr.DoError("ResearchSet/update_ssglists")
bResult = False
return bResult
def add_names(main_list, fragment):
oErr = ErrHandle()
try:
for sentence in fragment.replace("_", "").split("."):
# WOrk through non-initial words
words = re.split(r'\s+', sentence)
for word in words[1:]:
if re.match(re_name, word):
if not word in main_list:
main_list.append(word)
except:
msg = oErr.get_error_message()
oErr.DoError("add_names")
def calculate_pm(self):
"""Calculate the Pivot Manuscript/item for this research set"""
oErr = ErrHandle()
iBack = -1
try:
# Get the lists that we have made
ssglists = self.get_ssglists()
# Calculate the matches
ssglists = self.calculate_matches(ssglists)
# Figure out what the first best list is
idx_pm = -1
max_matches = -1
min_order = len(ssglists) + 2
min_year_start = 3000
min_year_finish = 3000
for idx, oListItem in enumerate(ssglists):
unique_matches = oListItem['unique_matches']
year_start = oListItem['title']['yearstart']
year_finish = oListItem['title']['yearfinish']
order = oListItem['title']['order']
# Check which is the best so far
bTakeThis = False
bTakeThis = (unique_matches > max_matches)
if not bTakeThis and unique_matches == max_matches:
bTakeThis = (year_start < min_year_start)
if not bTakeThis and year_start == min_year_start:
bTakeThis = (year_finish < min_year_finish)
if not bTakeThis and year_finish == min_year_finish:
bTakeThis = (order < min_order)
# Adapt if this is the one
if bTakeThis:
max_matches = unique_matches
min_year_start = year_start
min_year_finish = year_finish
min_order = order
idx_pm = idx
# What we return is the 'order' value of the best matching list
iBack = ssglists[idx_pm]['title']['order']
except:
msg = oErr.get_error_message()
oErr.DoError("ResearchSet/calculate_pm")
iBack = -1
# Return the PM that we have found
return iBack
def get_ssg_list(self):
"""Create a list of SSGs,depending on the type I am"""
oErr = ErrHandle()
lBack = None
collection_types = ['hist', 'ssgd']
try:
if self.setlisttype == "manu":
# Create a list of SSG's from the manuscript
lBack = self.ssgs_manuscript(self.manuscript)
elif self.setlisttype in collection_types:
lBack = self.ssgs_collection(self.collection)
except:
msg = oErr.get_error_message()
oErr.DoError("SetList/get_ssg_list")
return lBack
def get_watermark():
"""Create and return a watermark"""
oErr = ErrHandle()
watermark_template = "seeker/passim_watermark.html"
watermark = ""
try:
# create a watermark with the right datestamp
context_wm = dict(
datestamp=get_crpp_date(get_current_datetime(), True))
watermark = render_to_string(watermark_template, context_wm)
except:
msg = oErr.get_error_message()
oErr.DoError("get_watermark")
# Return the result
return watermark
def get_ssg_passim(ssg_id, obj=None):
oErr = ErrHandle()
code = ""
try:
# Get the Passim Code
if obj == None:
obj = EqualGold.objects.filter(id=ssg_id).first()
if obj.code == None:
code = "eqg_{}".format(obj.id)
elif " " in obj.code:
code = obj.code.split(" ")[1]
else:
code = obj.code
except:
msg = oErr.get_error_message()
oErr.DoError("get_ssg_passim")
return code
def add_item(super, profile, field, oChange, oCurrent=None):
"""Add one item"""
oErr = ErrHandle()
obj = None
try:
# Make sure to stringify, sorting the keys
change = json.dumps(oChange, sort_keys=True)
if oCurrent is None:
current = None
else:
current = json.dumps(oCurrent, sort_keys=True)
# Look for this particular change, supposing it has not changed yet
obj = EqualChange.objects.filter(super=super,
profile=profile,
field=field,
current=current,
change=change).first()
if obj == None or obj.changeapprovals.count() > 0:
# Less restricted: look for any suggestion for a change on this field that has not been reviewed by anyone yet.
bFound = False
for obj in EqualChange.objects.filter(super=super,
profile=profile,
field=field,
atype="def"):
if obj.changeapprovals.exclude(atype="def").count() == 0:
# We can use this one
bFound = True
obj.current = current
obj.change = change
obj.save()
break
# What if nothing has been found?
if not bFound:
# Only in that case do we make a new suggestion
obj = EqualChange.objects.create(super=super,
profile=profile,
field=field,
current=current,
change=change)
except:
msg = oErr.get_error_message()
oErr.DoError("EqualChange/add_item")
return obj
def listview_adaptations(lv):
"""Perform adaptations specific for this listview"""
oErr = ErrHandle()
try:
if lv in adaptation_list:
for adapt in adaptation_list.get(lv):
sh_done = Information.get_kvalue(adapt)
if sh_done == None or sh_done != "done":
# Do the adaptation, depending on what it is
method_to_call = "adapt_{}".format(adapt)
bResult, msg = globals()[method_to_call]()
if bResult:
# Success
Information.set_kvalue(adapt, "done")
except:
msg = oErr.get_error_message()
oErr.DoError("listview_adaptations")
def check_projects(profile):
"""Check of [profile] needs to have any EqualApprove objects
And if he does: create them for him
"""
oErr = ErrHandle()
iCount = 0
try:
# Walk through all the changes that I have suggested
qs = EqualChange.objects.filter(profile=profile)
for change in qs:
# Check the approval of this particular one
iCount += change.check_approval()
# All should be up to date now
except:
msg = oErr.get_error_message()
oErr.DoError("EqualChange/check_projects")
return iCount
def get_contents(self, pivot_id=None, recalculate=False):
"""Get the set of lists for this particular DCT"""
oErr = ErrHandle()
oBack = None
try:
# Get the SSGlists from the research set
ssglists = self.researchset.get_ssglists(recalculate)
# Possibly get (stored) parameters from myself
params = {}
if self.contents != "" and self.contents[0] == "{":
params = json.loads(self.contents)
# Return the parameters and the lists
oBack = dict(params=params, ssglists=ssglists)
except:
msg = oErr.get_error_message()
oErr.DoError("SetDef/get_contents")
return oBack
def __init__(self, *args, **kwargs):
# Obligatory for this type of form!!!
self.username = kwargs.pop('username', "")
self.team_group = kwargs.pop('team_group', "")
self.userplus = kwargs.pop('userplus', "")
# Start by executing the standard handling
super(SetDefForm, self).__init__(*args, **kwargs)
oErr = ErrHandle()
try:
# Some fields are not required
self.fields['name'].required = False
self.fields['notes'].required = False
# Set queryset(s) - for details view
self.fields['manulist'].queryset = Manuscript.objects.none()
# Set the widgets correctly
self.fields['histlist'].widget = CollOneHistWidget(
attrs={
'username': self.username,
'team_group': self.team_group,
'settype': 'hc',
'data-placeholder': 'Select a historical collection...',
'style': 'width: 100%;',
'class': 'searching'
})
# Note: the collection filters must use the SCOPE of the collection
self.fields['histlist'].queryset = Collection.get_scoped_queryset(
'super', self.username, self.team_group, settype="hc")
# Get the instance
if 'instance' in kwargs:
instance = kwargs['instance']
except:
msg = oErr.get_error_message()
oErr.DoError("SetDefForm")
return None
def get_ssg_sig(ssg_id):
oErr = ErrHandle()
sig = ""
editype_preferences = ['gr', 'cl', 'ot']
issue_375 = True
try:
for editype in editype_preferences:
siglist = Signature.objects.filter(
gold__equal__id=ssg_id,
editype=editype).order_by('code').values('code')
if len(siglist) > 0:
sig = siglist[0]['code']
break
# We now have the most appropriate signature, or the empty string, if there is none
# The folloing is old code, not quite clear why it was necessary
if not issue_375 and ',' in sig:
sig = sig.split(",")[0].strip()
except:
msg = oErr.get_error_message()
oErr.DoError("get_ssg_sig")
return sig
def get_ssg_corpus(profile, instance):
oErr = ErrHandle()
lock_status = "new"
ssg_corpus = None
try:
# Set the lock, or return if we are busy with this ssg_corpus
ssg_corpus = EqualGoldCorpus.objects.filter(profile=profile,
ssg=instance).last()
if ssg_corpus != None:
lock_status = ssg_corpus.status
# Check the status
if lock_status == "busy":
# Already busy
return context
else:
# Need to create a lock
ssg_corpus = EqualGoldCorpus.objects.create(profile=profile,
ssg=instance)
lock_status = "busy"
# Save the status
ssg_corpus.status = lock_status
ssg_corpus.save()
if lock_status == "new":
# Remove earlier corpora made by me based on this SSG
EqualGoldCorpus.objects.filter(profile=profile,
ssg=instance).delete()
# Create a new one
ssg_corpus = EqualGoldCorpus.objects.create(profile=profile,
ssg=instance,
status="busy")
except:
msg = oErr.get_error_message()
oErr.DoError("get_ssg_corpus")
return ssg_corpus, lock_status
def check_approval(self):
"""Check if all who need it have an EqualApprove object for this one"""
oErr = ErrHandle()
iCount = 0
try:
# Check which editors should have an approval object (excluding myself)
change = self
profile = self.profile
lst_approver = change.get_approver_list(profile)
for approver in lst_approver:
# Check if an EqualApprove exists
approval = EqualApproval.objects.filter(
change=change, profile=approver).first()
if approval is None:
# Create one
approval = EqualApproval.objects.create(change=change,
profile=approver)
iCount = 1
except:
msg = oErr.get_error_message()
oErr.DoError("EqualChange/check_approval")
return iCount
def get_setlist(self, pivot_id=None):
"""Get the set of lists for this particular DCT"""
oErr = ErrHandle()
oBack = None
try:
# Get to the research set
id_list = [
x['id']
for x in self.researchset.researchset_setlists.all().order_by(
'order').values("id")
]
lst_rset = []
if pivot_id != None and pivot_id in id_list:
lst_rset.append(pivot_id)
# Add the remaining ID's
for id in id_list:
if not id in lst_rset: lst_rset.append(id)
# Check the number of lists in the research set
if lst_rset == None or len(lst_rset) < 2:
oErr.Status("Not enough SSG-lists to compare")
return None
# We have enough lists: Get the lists of SSGs for each
lst_ssglists = []
for setlist_id in lst_rset:
# Get the actual setlist
setlist = SetList.objects.filter(id=setlist_id).first()
if setlist != None:
# Create an empty SSG-list
oSsgList = {}
# Add the object itself
oSsgList['obj'] = setlist
# Add the name object for this list
oSsgList['title'] = setlist.get_title_object()
# Get the list of SSGs for this list
oSsgList['ssglist'] = setlist.get_ssg_list()
# Add the list object to the list
lst_ssglists.append(oSsgList)
# Return this list of lists
oBack = dict(ssglists=lst_ssglists)
# Prepare and create an appropriate table = list of rows
rows = []
# Create header row
oRow = []
oRow.append('Gr/Cl/Ot')
for oSsgList in lst_ssglists:
# Add the title *object*
oRow.append(oSsgList['title'])
rows.append(oRow)
# Start out with the pivot: the *first* one in 'ssglist'
lst_pivot = lst_ssglists[0]
for oPivot in lst_pivot['ssglist']:
# Create a row based on this pivot
oRow = []
# (1) row header
oRow.append(oPivot['sig'])
# (2) pivot SSG number
oRow.append(oPivot['order'])
# (3) SSG number in all other manuscripts
ssg_id = oPivot['super']
for lst_this in lst_ssglists[1:]:
bFound = False
order = ""
for oItem in lst_this['ssglist']:
if ssg_id == oItem['super']:
# Found it!
order = oItem['order']
bFound = True
break
oRow.append(order)
# (4) add the row to the list
rows.append(oRow)
# Make sure we return the right information
oBack['setlist'] = rows
except:
msg = oErr.get_error_message()
oErr.DoError("SetDef/get_setlist")
return oBack
def ssgs_manuscript(self, bDebug=False):
"""Get the ordered list of SSGs related to a manuscript"""
oErr = ErrHandle()
lBack = None
try:
manu = self.manuscript
# Get a list of SSGs to which the sermons in this manuscript point
# Per issue #402 additional info needs to be there:
# author,
# PASSIM code,
# incipit, explicit, HC (if the SSG is part of a historical collection),
# number of SGs contained in the equality set,
# number of links to other SSGs
# Per issue #402, phase 2, even more info needs to be added:
# for MANUSCRIPT source-lists, there should be an option for the user
# to select additional SERMON manifestation details fields to be shown
# (options: attributed author, section title, lectio, title, incipit, explicit,
# postscriptum, feast, bible reference, cod. notes, notes, keywords)
qs = manu.sermondescr_super.all().order_by(
'sermon__msitem__order').values(
'sermon', 'sermon__msitem__order', 'super', 'super__code',
'super__author__name', 'super__incipit', 'super__explicit',
'super__sgcount', 'super__ssgcount',
'sermon__author__name', 'sermon__sectiontitle',
'sermon__quote', 'sermon__title', 'sermon__incipit',
'sermon__explicit', 'sermon__postscriptum',
'sermon__feast__name', 'sermon__bibleref',
'sermon__additional', 'sermon__note')
# NOTE: the 'keywords' for issue #402 are a bit more cumbersome to collect...
lBack = []
with transaction.atomic():
for obj in qs:
# Get the order number
order = obj['sermon__msitem__order']
# Get the SSG and all its characteristics
super = obj['super']
code = obj['super__code']
authorname = obj['super__author__name']
incipit = obj['super__incipit']
explicit = obj['super__explicit']
sgcount = obj['super__sgcount']
ssgcount = obj['super__ssgcount']
# Sermon characteristics
sermon = obj['sermon']
srm_author = obj['sermon__author__name']
srm_sectiontitle = obj['sermon__sectiontitle']
srm_lectio = obj['sermon__quote']
srm_title = obj['sermon__title']
srm_incipit = obj['sermon__incipit']
srm_explicit = obj['sermon__explicit']
srm_postscriptum = obj['sermon__postscriptum']
srm_feast = obj['sermon__feast__name']
srm_bibleref = obj['sermon__bibleref']
srm_codnotes = obj['sermon__additional']
srm_notes = obj['sermon__note']
# Get the name(s) of the HC
lst_hc = CollectionSuper.objects.filter(
super=super, collection__settype="hc").values(
'collection__id', 'collection__name')
hcs = ", ".join([x['collection__name'] for x in lst_hc])
# ======== DEBUGGING ============
if hcs != "":
iStop = 1
# ===============================
# TODO: Get the keywords
lst_kw = SermonDescrKeyword.objects.filter(
sermon=sermon).values('keyword__name')
kws = ", ".join([x['keyword__name'] for x in lst_kw])
# Get a URL for this ssg
url = reverse('equalgold_details', kwargs={'pk': super})
# Treat signatures for this SSG: get the best for showing
sigbest = get_goldsig_dct(super)
if sigbest == "":
sigbest = "ssg_{}".format(super)
# Signatures for this SSG: get the full list
siglist = get_goldsiglist_dct(super)
# Put into object
oItem = dict(super=super,
sig=sigbest,
siglist=siglist,
hcs=hcs,
kws=kws,
order=order,
code=code,
url=url,
author=authorname,
type='ms',
incipit=incipit,
explicit=explicit,
sgcount=sgcount,
ssgcount=ssgcount,
srm_author=srm_author,
srm_sectiontitle=srm_sectiontitle,
srm_lectio=srm_lectio,
srm_title=srm_title,
srm_incipit=srm_incipit,
srm_explicit=srm_explicit,
srm_postscriptum=srm_postscriptum,
srm_feast=srm_feast,
srm_bibleref=srm_bibleref,
srm_codnotes=srm_codnotes,
srm_notes=srm_notes)
# Add to list
lBack.append(oItem)
# Debugging: print the list
if bDebug:
print("Manuscript id={}".format(manu.id))
for oItem in lBack:
order = oItem.get("order")
super = oItem.get("super")
sig = oItem.get("sig")
code = get_passimcode(super, code)
print("sermon {}: ssg={} sig=[{}]".format(
order, code, sig))
except:
msg = oErr.get_error_message()
oErr.DoError("ssgs_manuscript")
return lBack
def do_overlap(self, ssg_link, degree):
"""Calculate the overlap network up until 'degree'"""
def add_nodeset(ssg_id, group):
node_key = ssg_id
# ---------- DEBUG --------------
if node_key == 4968:
iStop = 1
# -------------------------------
# Possibly add the node to the set
if not node_key in node_set:
code_sig = get_ssg_sig(ssg_id)
ssg = EqualGold.objects.filter(id=ssg_id).first()
code_pas = get_ssg_passim(ssg_id, ssg)
scount = ssg.scount
# Get a list of the HCs that this SSG is part of
hc_list = ssg.collections.filter(settype="hc").values(
"id", "name")
hcs = []
for oItem in hc_list:
id = oItem['id']
if not id in hist_set:
hist_set[id] = oItem['name']
hcs.append(id)
node_value = dict(label=code_sig,
id=ssg_id,
group=group,
passim=code_pas,
scount=scount,
hcs=hcs)
node_set[node_key] = node_value
node_list = []
link_list = []
node_set = {}
link_set = {}
hist_set = {}
max_value = 0
max_group = 1
oErr = ErrHandle()
try:
# Degree 0: direct contacts
ssg_id = self.obj.id
ssg_queue = [ssg_id]
ssg_add = []
group = 1
degree -= 1
while len(ssg_queue) > 0 and degree >= 0:
# Get the next available node
node_key = ssg_queue.pop()
# Possibly add the node to the set
add_nodeset(node_key, group)
# Add links from this SSG to others
if node_key in ssg_link:
dst_list = ssg_link[node_key]
for oDst in dst_list:
dst = oDst['dst']
# Add this one to the add list
ssg_add.append(dst)
link_key = "{}_{}".format(node_key, dst)
if not link_key in link_set:
oLink = dict(source=node_key,
target=dst,
spectype=oDst['spectype'],
linktype=oDst['linktype'],
spec=oDst['spec'],
link=oDst['link'],
alternatives=oDst['alternatives'],
note=oDst['note'],
value=0)
# Add the link to the set
link_set[link_key] = oLink
# Add the destination node to nodeset
add_nodeset(dst, group + 1)
# Increment the link value
link_set[link_key]['value'] += 1
# Go to the next degree
if len(ssg_queue) == 0 and degree >= 0:
# Decrement the degree
degree -= 1
group += 1
# Add the items from ssg_add into the queue
while len(ssg_add) > 0:
ssg_queue.append(ssg_add.pop())
# Turn the sets into lists
node_list = [v for k, v in node_set.items()]
link_list = [v for k, v in link_set.items()]
# Calculate max_value
for oItem in link_list:
value = oItem['value']
if value > max_value: max_value = value
for oItem in node_list:
group = oItem['group']
if group > max_group: max_group = group
except:
msg = oErr.get_error_message()
oErr.DoError("EqualGoldOverlap/do_overlap")
return node_list, link_list, hist_set, max_value, max_group
def add_to_context(self, context):
def add_to_dict(this_dict, item):
if item != "":
if not item in this_dict:
this_dict[item] = 1
else:
this_dict[item] += 1
oErr = ErrHandle()
try:
# Need to figure out who I am
profile = Profile.get_user_profile(self.request.user.username)
instance = self.obj
networkslider = self.qd.get("network_trans_slider", "1")
if isinstance(networkslider, str):
networkslider = int(networkslider)
# Get the 'manuscript-corpus': all manuscripts in which a sermon is that belongs to the same SSG
manu_list = SermonDescrEqual.objects.filter(
super=instance).distinct().values("manu_id")
manu_dict = {}
for idx, manu in enumerate(manu_list):
manu_dict[manu['manu_id']] = idx + 1
ssg_corpus, lock_status = get_ssg_corpus(profile, instance)
if lock_status != "ready":
# Create an EqualGoldCorpus based on the SSGs in these manuscripts
ssg_list = SermonDescrEqual.objects.filter(
manu__id__in=manu_list).order_by(
'super_id').distinct().values('super_id')
ssg_list_id = [x['super_id'] for x in ssg_list]
with transaction.atomic():
for ssg in EqualGold.objects.filter(id__in=ssg_list_id):
# Get the name of the author
authorname = "empty" if ssg.author == None else ssg.author.name
# Get the scount
scount = ssg.scount
# Create new ssg_corpus item
obj = EqualGoldCorpusItem.objects.create(
corpus=ssg_corpus,
equal=ssg,
authorname=authorname,
scount=scount)
# Add this list to the ssg_corpus
ssg_corpus.status = "ready"
ssg_corpus.save()
node_list, link_list, author_list, max_value = self.do_manu_method(
ssg_corpus, manu_list, networkslider)
# Add the information to the context in data
context['data'] = dict(node_list=node_list,
link_list=link_list,
watermark=get_watermark(),
author_list=author_list,
max_value=max_value,
networkslider=networkslider,
legend="SSG network")
# Can remove the lock
ssg_corpus.status = "ready"
ssg_corpus.save()
except:
msg = oErr.get_error_message()
oErr.DoError("EqualGoldTrans/add_to_context")
return context
def ssgs_collection(self, bDebug=False):
"""Get the ordered list of SSGs in the [super] type collection"""
oErr = ErrHandle()
lBack = None
try:
coll = self.collection
# Get the list of SSGs inside this collection
# Per issue #402 additional info needs to be there:
# author,
# PASSIM code,
# incipit, explicit, HC (if the SSG is part of a historical collection),
# number of SGs contained in the equality set,
# number of links to other SSGs
# HC/PD specific:
# name
# description
# size
# HC-specific:
# literature
qs = CollectionSuper.objects.filter(
collection=coll).order_by('order').values(
'order', 'collection__name', 'collection__descrip',
'collection__settype', 'super', 'super__code',
'super__author__name', 'super__incipit', 'super__explicit',
'super__sgcount', 'super__ssgcount')
lBack = []
with transaction.atomic():
for obj in qs:
# Get the order number
order = obj['order']
# Get the collection-specific information
name = obj['collection__name']
descr = obj['collection__descrip']
settype = obj['collection__settype']
# Get the SSG
super = obj['super']
code = obj['super__code']
authorname = obj['super__author__name']
incipit = obj['super__incipit']
explicit = obj['super__explicit']
sgcount = obj['super__sgcount']
ssgcount = obj['super__ssgcount']
# Get the name(s) of the HC
lst_hc = CollectionSuper.objects.filter(
super=super, collection__settype="hc").values(
'collection__id', 'collection__name')
hcs = ", ".join([x['collection__name'] for x in lst_hc])
# ======== DEBUGGING ============
if hcs != "":
iStop = 1
# ===============================
# Get a URL for this ssg
url = reverse('equalgold_details', kwargs={'pk': super})
# Treat signatures for this SSG
sigbest = get_goldsig_dct(super)
if sigbest == "":
sigbest = "ssg_{}".format(super)
# Signatures for this SSG: get the full list
siglist = get_goldsiglist_dct(super)
# Put into object
oItem = dict(super=super,
sig=sigbest,
siglist=siglist,
name=name,
descr=descr,
type=settype,
order=order,
code=code,
url=url,
author=authorname,
incipit=incipit,
explicit=explicit,
sgcount=sgcount,
ssgcount=ssgcount,
hcs=hcs)
# Add to list
lBack.append(oItem)
# Debugging: print the list
if bDebug:
print("Collection id={}".format(coll.id))
for oItem in lBack:
order = oItem.get("order")
super = oItem.get("super")
sig = oItem.get("sig")
code = get_passimcode(super, code)
print("sermon {}: ssg={} sig=[{}]".format(
order, code, sig))
except:
msg = oErr.get_error_message()
oErr.DoError("ssgs_collection")
return lBack
def do_manu_method(self, ssg_corpus, manu_list, min_value):
"""Calculation like 'MedievalManuscriptTransmission' description
The @min_value is the minimum link-value the user wants to see
"""
oErr = ErrHandle()
author_dict = {}
node_list = []
link_list = []
max_value = 0 # Maximum number of manuscripts in which an SSG occurs
max_scount = 1 # Maximum number of sermons associated with one SSG
try:
# Initializations
ssg_dict = {}
link_dict = {}
scount_dict = {}
node_listT = []
link_listT = []
node_set = {} # Put the nodes in a set, with their SSG_ID as key
title_set = {} # Link from title to SSG_ID
# Walk the ssg_corpus: each SSG is one 'text', having a title and a category (=author code)
for idx, item in enumerate(
EqualGoldCorpusItem.objects.filter(
corpus=ssg_corpus).values('authorname', 'scount',
'equal__code', 'equal__id')):
# Determine the name for this row
category = item['authorname']
ssg_id = item['equal__id']
code = item['equal__code']
if code == None or code == "" or not " " in code or not "." in code:
title = "eqg{}".format(ssg_id)
else:
title = code.split(" ")[1]
# Get the Signature that is most appropriate
sig = get_ssg_sig(ssg_id)
# Add author to dictionary
if not category in author_dict: author_dict[category] = 0
author_dict[category] += 1
# the scount and store it in a dictionary
scount = item['scount']
scount_dict[ssg_id] = scount
if scount > max_scount:
max_scount = scount
node_key = ssg_id
node_value = dict(label=title,
category=category,
scount=scount,
sig=sig,
rating=0)
if node_key in node_set:
oErr.Status(
"EqualGoldGraph/do_manu_method: attempt to add same title '{}' for {} and {}"
.format(title, ssg_id, title_set[title]))
else:
node_set[node_key] = node_value
title_set[title] = ssg_id
# Create list of authors
author_list = [
dict(category=k, count=v) for k, v in author_dict.items()
]
author_list = sorted(author_list,
key=lambda x:
(-1 * x['count'], x['category'].lower()))
# Create a dictionary of manuscripts, each having a list of SSG ids
manu_set = {}
for manu_item in manu_list:
manu_id = manu_item["manu_id"]
# Get a list of all SSGs in this manuscript
ssg_list = SermonDescrEqual.objects.filter(
manu__id=manu_id).order_by('super_id').distinct()
# Add the SSG id list to the manuset
manu_set[manu_id] = [x.super for x in ssg_list]
# Create a list of edges based on the above
link_dict = {}
for manu_id, ssg_list in manu_set.items():
# Only treat ssg_lists that are larger than 1
if len(ssg_list) > 1:
# itertool.combinations creates all combinations of SSG to SSG in one manuscript
for subset in itertools.combinations(ssg_list, 2):
source = subset[0]
target = subset[1]
source_id = source.id
target_id = target.id
link_code = "{}_{}".format(source_id, target_id)
if link_code in link_dict:
oLink = link_dict[link_code]
else:
oLink = dict(source=source_id,
target=target_id,
value=0)
link_dict[link_code] = oLink
# Add 1
oLink['value'] += 1
if oLink['value'] > max_value:
max_value = oLink['value']
# Turn the link_dict into a list
# link_list = [v for k,v in link_dict.items()]
# Only accept the links that have a value >= min_value
node_dict = {}
link_list = []
for k, oItem in link_dict.items():
if oItem['value'] >= min_value:
link_list.append(copy.copy(oItem))
# Take note of the nodes
src = oItem['source']
dst = oItem['target']
if not src in node_dict: node_dict[src] = node_set[src]
if not dst in node_dict: node_dict[dst] = node_set[dst]
# Walk the nodes
node_list = []
for ssg_id, oItem in node_dict.items():
oItem['id'] = ssg_id
oItem['scount'] = 100 * scount_dict[oItem['id']] / max_scount
node_list.append(copy.copy(oItem))
except:
msg = oErr.get_error_message()
oErr.DoError("do_manu_method")
return node_list, link_list, author_list, max_value
def do_manu_method(self, ssg_corpus, manu_list, min_value):
"""Calculation like Manuscript_Transmission_Of_se172
The @min_value is the minimum link-value the user wants to see
"""
oErr = ErrHandle()
node_list = []
link_list = []
max_value = 0 # Maximum number of manuscripts in which an SSG occurs
max_scount = 1 # Maximum number of sermons associated with one SSG
manu_count = []
def get_nodes(crp):
nodes = []
for idx, item in enumerate(crp.target_ints):
oNode = dict(group=item,
author=crp.target_idx[item],
id=crp.titles[idx])
nodes.append(oNode)
return nodes
try:
# Create a pystyl-corpus object (see above)
sty_corpus = Corpus(texts=[],
titles=[],
target_ints=[],
target_idx=[])
ssg_dict = {}
link_dict = {}
scount_dict = {}
node_listT = []
link_listT = []
# Initialize manu_count: the number of SSG co-occurring in N manuscripts
for item in manu_list:
manu_count.append(0)
# Walk the ssg_corpus: each SSG is one 'text', having a title and a category (=author code)
for idx, item in enumerate(
EqualGoldCorpusItem.objects.filter(
corpus=ssg_corpus).values('words', 'authorname',
'scount', 'equal__code',
'equal__id')):
# Determine the name for this row
category = item['authorname']
code = item['equal__code']
if code == None or code == "" or not " " in code or not "." in code:
title = "eqg{}".format(item['equal__id'])
else:
title = code.split(" ")[1]
# The text = the words
text = " ".join(json.loads(item['words']))
# the scount and store it in a dictionary
scount_dict[title] = item['scount']
if item['scount'] > max_scount:
max_scount = item['scount']
# Add the text to the corpus
if title in sty_corpus.titles:
ssg_id = -1
bFound = False
for k, v in ssg_dict.items():
if v == title:
ssg_id = k
bFound = True
break
oErr.Status(
"EqualGoldGraph/do_manu_method: attempt to add same title '{}' for {} and {}"
.format(title, ssg_id, item['equal__id']))
else:
# Also make sure to buidl an SSG-dictionary
ssg_dict[item['equal__id']] = title
sty_corpus.add_text(text, title, category)
# Get a list of nodes
node_listT = get_nodes(sty_corpus)
# Walk the manuscripts
for manu_item in manu_list:
manu_id = manu_item["manu_id"]
# Get a list of all SSGs in this manuscript
ssg_list = SermonDescrEqual.objects.filter(
manu__id=manu_id).order_by('super_id').distinct().values(
'super_id')
ssg_list_id = [x['super_id'] for x in ssg_list]
# evaluate links between a source and target SSG
for idx_s, source_id in enumerate(ssg_list_id):
# sanity check
if source_id in ssg_dict:
# Get the title of the source
source = ssg_dict[source_id]
for idx_t in range(idx_s + 1, len(ssg_list_id) - 1):
target_id = ssg_list_id[idx_t]
# Double check
if target_id in ssg_dict:
# Get the title of the target
target = ssg_dict[target_id]
# Retrieve or create a link from the link_listT
link_code = "{}_{}".format(
source_id, target_id)
if link_code in link_dict:
oLink = link_listT[link_dict[link_code]]
else:
oLink = dict(source=source,
source_id=source_id,
target=target,
target_id=target_id,
value=0)
link_listT.append(oLink)
link_dict[link_code] = len(link_listT) - 1
# Now add to the value
oLink['value'] += 1
if oLink['value'] > max_value:
max_value = oLink['value']
# Only accept the links that have a value >= min_value
node_dict = []
for oItem in link_listT:
if oItem['value'] >= min_value:
link_list.append(copy.copy(oItem))
# Take note of the nodes
src = oItem['source']
dst = oItem['target']
if not src in node_dict: node_dict.append(src)
if not dst in node_dict: node_dict.append(dst)
# Walk the nodes
for oItem in node_listT:
if oItem['id'] in node_dict:
oItem['scount'] = 100 * scount_dict[
oItem['id']] / max_scount
node_list.append(copy.copy(oItem))
except:
msg = oErr.get_error_message()
oErr.DoError("do_hier_method1")
return node_list, link_list, max_value
def add_to_context(self, context):
names_list = [x.lower() for x in COMMON_NAMES]
def get_author(code):
"""Get the author id from the passim code"""
author = 10000
if "PASSIM" in code:
author = int(code.replace("PASSIM", "").strip().split(".")[0])
return author
def add_to_dict(this_dict, item):
if item != "":
if not item in this_dict:
this_dict[item] = 1
else:
this_dict[item] += 1
oErr = ErrHandle()
try:
# Need to figure out who I am
profile = Profile.get_user_profile(self.request.user.username)
instance = self.obj
# Get the 'manuscript-corpus': the manuscripts in which the same kind of sermon like me is
manu_list = SermonDescrEqual.objects.filter(
super=instance).distinct().values("manu_id")
manu_dict = {}
for idx, manu in enumerate(manu_list):
manu_dict[manu['manu_id']] = idx + 1
author_dict = {}
lock_status = "new"
# Set the lock, or return if we are busy with this ssg_corpus
ssg_corpus = EqualGoldCorpus.objects.filter(profile=profile,
ssg=instance).last()
if ssg_corpus != None:
lock_status = ssg_corpus.status
# Check the status
if lock_status == "busy":
# Already busy
return context
else:
# Need to create a lock
ssg_corpus = EqualGoldCorpus.objects.create(profile=profile,
ssg=instance)
lock_status = "busy"
# Save the status
ssg_corpus.status = lock_status
ssg_corpus.save()
if lock_status == "new":
# Remove earlier corpora made by me based on this SSG
EqualGoldCorpus.objects.filter(profile=profile,
ssg=instance).delete()
# Create a new one
ssg_corpus = EqualGoldCorpus.objects.create(profile=profile,
ssg=instance,
status="busy")
if lock_status != "ready":
# Create an EqualGoldCorpus based on the SSGs in these manuscripts
ssg_list = SermonDescrEqual.objects.filter(
manu__id__in=manu_list).order_by(
'super_id').distinct().values('super_id')
ssg_list_id = [x['super_id'] for x in ssg_list]
all_words = {}
with transaction.atomic():
for ssg in EqualGold.objects.filter(id__in=ssg_list_id):
# Add this item to the ssg_corpus
latin = {}
if ssg.incipit != None:
for item in ssg.srchincipit.replace(
",", "").replace("…", "").split(" "):
add_to_dict(latin, item)
add_to_dict(all_words, item)
if ssg.explicit != None:
for item in ssg.srchexplicit.replace(
",", "").replace("…", "").split(" "):
add_to_dict(latin, item)
add_to_dict(all_words, item)
# Get the name of the author
authorname = "empty" if ssg.author == None else ssg.author.name
# Create new ssg_corpus item
obj = EqualGoldCorpusItem.objects.create(
corpus=ssg_corpus,
words=json.dumps(latin),
equal=ssg,
authorname=authorname)
# What are the 100 MFWs in all_words?
mfw = [dict(word=k, count=v) for k, v in all_words.items()]
mfw_sorted = sorted(mfw, key=lambda x: -1 * x['count'])
mfw_cento = mfw_sorted[:100]
mfw = []
for item in mfw_cento:
mfw.append(item['word'])
# Add this list to the ssg_corpus
ssg_corpus.mfw = json.dumps(mfw)
ssg_corpus.status = "ready"
ssg_corpus.save()
node_list, link_list, max_value = self.do_hier_method3(
ssg_corpus, names_list)
# Add the information to the context in data
context['data'] = dict(node_list=node_list,
link_list=link_list,
watermark=get_watermark(),
max_value=max_value,
legend="SSG network")
# Can remove the lock
ssg_corpus.status = "ready"
ssg_corpus.save()
except:
msg = oErr.get_error_message()
oErr.DoError("EqualGoldPca/add_to_context")
return context
def do_hier_method3(self, ssg_corpus, names_list):
"""Calculate distance matrix with PYSTYL, do the rest myself"""
oErr = ErrHandle()
node_list = []
link_list = []
max_value = 0
def dm_to_leo(matrix, crp):
links = []
i = 0
maxv = 0
for row_id, row in enumerate(matrix):
# Calculate the link
minimum = None
min_id = -1
for col_idx in range(row_id + 1, len(row) - 1):
value = row[col_idx]
if minimum == None:
if value > 0: minimum = value
elif value < minimum:
minimum = value
min_id = col_idx
if minimum != None and min_id >= 0:
# Create a link
oLink = dict(source_id=row_id,
source=crp.titles[row_id],
target_id=min_id,
target=crp.titles[min_id],
value=minimum)
links.append(oLink)
# Keep track of max_value
if minimum > maxv:
maxv = minimum
return links, maxv
def get_nodes(crp):
nodes = []
for idx, item in enumerate(crp.target_ints):
oNode = dict(group=item,
author=crp.target_idx[item],
id=crp.titles[idx],
scount=5)
nodes.append(oNode)
return nodes
do_example = False
try:
# Create a pystyl-corpus object (see above)
sty_corpus = Corpus(texts=[],
titles=[],
target_ints=[],
target_idx=[])
ssg_dict = {}
# Walk the ssg_corpus: each SSG is one 'text', having a title and a category (=author code)
for idx, item in enumerate(
EqualGoldCorpusItem.objects.filter(
corpus=ssg_corpus).values('words', 'authorname',
'equal__code', 'equal__id')):
# Determine the name for this row
category = item['authorname']
code = item['equal__code']
if code == None or code == "" or not " " in code or not "." in code:
title = "eqg{}".format(item['equal__id'])
else:
title = code.split(" ")[1]
# The text = the words
text = " ".join(json.loads(item['words']))
# Add the text to the corpus
if title in sty_corpus.titles:
ssg_id = -1
bFound = False
for k, v in ssg_dict.items():
if v == title:
ssg_id = k
bFound = True
break
oErr.Status(
"EqualGoldGraph/do_manu_method: attempt to add same title '{}' for {} and {}"
.format(title, ssg_id, item['equal__id']))
else:
# Also make sure to buidl an SSG-dictionary
ssg_dict[item['equal__id']] = title
sty_corpus.add_text(text, title, category)
# We now 'have' the corpus, so we can work with it...
sty_corpus.preprocess(alpha_only=True, lowercase=True)
sty_corpus.tokenize()
# REmove the common names
sty_corpus.remove_tokens(rm_tokens=names_list, rm_pronouns=False)
# Vectorize the corpus
sty_corpus.vectorize(mfi=200,
ngram_type="word",
ngram_size=1,
vector_space="tf_std")
# Get a list of nodes
node_list = get_nodes(sty_corpus)
# Create a distance matrix
dm = distance_matrix(sty_corpus, "minmax")
# Convert the distance matrix into a list of 'nearest links'
link_list, max_value = dm_to_leo(dm, sty_corpus)
# Convert the cluster_tree into a node_list and a link_list (i.e. get the list of edges)
iRead = 1
except:
msg = oErr.get_error_message()
oErr.DoError("do_hier_method1")
return node_list, link_list, max_value
def add_codico_to_manuscript(manu):
"""Check if a manuscript has a Codico, and if not create it"""
def get_number(items, bFirst):
"""Extract the first or last consecutive number from the string"""
number = -1
if len(items) > 0:
if bFirst:
# Find the first number
for sInput in items:
arNumber = re.findall(r'\d+', sInput)
if len(arNumber) > 0:
number = int(arNumber[0])
break
else:
# Find the last number
for sInput in reversed(items):
arNumber = re.findall(r'\d+', sInput)
if len(arNumber) > 0:
number = int(arNumber[-1])
break
return number
oErr = ErrHandle()
bResult = False
msg = ""
try:
# Check if the codico exists
codi = Codico.objects.filter(manuscript=manu).first()
if codi == None:
# Get first and last sermons and then their pages
items = [x['itemsermons__locus'] for x in manu.manuitems.filter(itemsermons__locus__isnull=False).order_by(
'order').values('itemsermons__locus')]
if len(items) > 0:
pagefirst = get_number(items, True)
pagelast = get_number(items, False)
else:
pagefirst = 1
pagelast = 1
# Create the codico
codi = Codico.objects.create(
name=manu.name, support=manu.support, extent=manu.extent,
format=manu.format, order=1, pagefirst=pagefirst, pagelast=pagelast,
origin=manu.origin, manuscript=manu
)
else:
# Possibly copy stuff from manu to codi
bNeedSaving = False
if codi.name == "SUPPLY A NAME" and manu.name != "":
codi.name = manu.name ; bNeedSaving = True
if codi.support == None and manu.support != None:
codi.support = manu.support ; bNeedSaving = True
if codi.extent == None and manu.extent != None:
codi.extent = manu.extent ; bNeedSaving = True
if codi.format == None and manu.format != None:
codi.format = manu.format ; bNeedSaving = True
if codi.order == 0:
codi.order = 1 ; bNeedSaving = True
if codi.origin == None and manu.origin != None:
codi.origin = manu.origin ; bNeedSaving = True
# Possibly save changes
if bNeedSaving:
codi.save()
# Copy provenances
if codi.codico_provenances.count() == 0:
for mp in manu.manuscripts_provenances.all():
obj = ProvenanceCod.objects.filter(
provenance=mp.provenance, codico=codi, note=mp.note).first()
if obj == None:
obj = ProvenanceCod.objects.create(
provenance=mp.provenance, codico=codi, note=mp.note)
# Copy keywords
if codi.codico_kw.count() == 0:
for mk in manu.manuscript_kw.all():
obj = CodicoKeyword.objects.filter(
codico=codi, keyword=mk.keyword).first()
if obj == None:
obj = CodicoKeyword.objects.create(
codico=codi, keyword=mk.keyword)
# Copy date ranges
if codi.codico_dateranges.count() == 0:
for md in manu.manuscript_dateranges.all():
if md.codico_id == None or md.codico_id == 0 or md.codico == None or md.codic.id != codi.id:
md.codico = codi
md.save()
# Tie all MsItems that need be to the Codico
for msitem in manu.manuitems.all().order_by('order'):
if msitem.codico_id == None or msitem.codico == None or msitem.codico.id != codi.id:
msitem.codico = codi
msitem.save()
bResult = True
except:
msg = oErr.get_error_message()
oErr.DoError("add_codico_to_manuscript")
bResult = False
return bResult, msg