示例#1
0
    def get_galway(self, catalogue_id):
        # Read the manuscript with the indicated id

        oBack = None
        bResult = False
        oErr = ErrHandle()
        try:
            url = "https://elmss.nuigalway.ie/api/v1/catalogue/{}".format(
                catalogue_id)
            try:
                r = requests.get(url)
            except:
                sMsg = oErr.get_error_message()
                oErr.DoError("Request problem")
                return False, sMsg
            if r.status_code == 200:
                # Read the response
                sText = r.text
                oBack = json.loads(sText)
                bResult = True
            else:
                bResult = False
                sResult = "download_file received status {} for {}".format(
                    r.status_code, url)
                oErr.Status("get_galway reading error: {}".format(sResult))
        except:
            msg = oErr.get_error_message()
            oErr.DoError("get_galway")
            oBack = None

        return oBack
示例#2
0
    def get_ssglists(self, recalculate=False):
        """Get the set of lists for this particular ResearchSet"""

        oErr = ErrHandle()
        lBack = None
        try:
            if self.contents != "" and self.contents[0] == "[":
                lst_contents = json.loads(self.contents)
            else:
                lst_contents = []
            if not recalculate and len(lst_contents) > 0:
                # Should the unique_matches be re-calculated?
                if not 'unique_matches' in lst_contents[0]:
                    # Yes, re-calculate
                    lst_contents = self.calculate_matches(lst_contents)
                    # And make sure this gets saved!
                    self.contents = json.dumps(lst_contents)
                    self.save()
                lBack = lst_contents
            else:
                # Re-calculate the lists
                self.update_ssglists()
                # Get the result
                lBack = json.loads(self.contents)

        except:
            msg = oErr.get_error_message()
            oErr.DoError("ResearchSet/get_ssglists")
        return lBack
示例#3
0
    def __init__(self, *args, **kwargs):
        self.username = kwargs.pop('username', "")
        self.team_group = kwargs.pop('team_group', "")
        self.userplus = kwargs.pop('userplus', "")
        # Start by executing the standard handling
        super(EqualApprovalForm, self).__init__(*args, **kwargs)

        oErr = ErrHandle()
        try:
            # Some fields are not required
            self.fields['change'].required = False
            self.fields['comment'].required = False
            self.fields['profile'].required = False
            self.fields['atype'].required = False

            # Set queryset(s) - for details view
            self.fields['profilelist'].queryset = Profile.objects.all(
            ).order_by('user__username')
            self.fields['passimlist'].queryset = EqualGold.objects.filter(
                code__isnull=False, moved__isnull=True).order_by('code')
            self.fields['atypelist'].queryset = FieldChoice.objects.filter(
                field=APPROVAL_TYPE).order_by("english_name")

            # Get the instance
            if 'instance' in kwargs:
                instance = kwargs['instance']

        except:
            msg = oErr.get_error_message()
            oErr.DoError("EqualApprovalForm")
        return None
示例#4
0
    def get_approver_list(self, excl=None):
        """Get the list of editors that need to approve this change
        
        If [excl] is specified, then this object is excluded from the list of Profile objects returned
        """

        oErr = ErrHandle()
        lstBack = None
        try:
            # Default: return the empty list
            lstBack = Profile.objects.none()
            # Get all the projects to which this SSG 'belongs'
            lst_project = [
                x['id'] for x in self.super.projects.all().values("id")
            ]
            # Note: only SSGs that belong to more than one project need to be reviewed
            if len(lst_project) > 1:
                # Get all the editors associated with these projects
                lst_profile_id = [
                    x['profile_id'] for x in ProjectEditor.objects.filter(
                        project__id__in=lst_project).values(
                            'profile_id').distinct()
                ]
                if len(lst_profile_id) > 0:
                    if excl == None:
                        lstBack = Profile.objects.filter(id__in=lst_profile_id)
                    else:
                        lstBack = Profile.objects.filter(
                            id__in=lst_profile_id).exclude(id=excl.id)
        except:
            msg = oErr.get_error_message()
            oErr.DoError("EqualChange/get_approver_list")
        return lstBack
示例#5
0
    def get_review_list(profile, all=False):
        """Get the list of objects this editor needs to review"""

        oErr = ErrHandle()
        lstBack = []
        try:
            # Default: return the empty list
            # lstBack = EqualChange.objects.none()
            # Get the list of projects for this user
            lst_project_id = profile.projects.all().values("id")
            if len(lst_project_id) > 0:
                # Get the list of EqualChange objects linked to any of these projects
                lstQ = []
                lstQ.append(
                    Q(super__equal_proj__project__id__in=lst_project_id))
                if not all:
                    lstQ.append(Q(atype='def'))
                lstBack = [
                    x['id'] for x in EqualChange.objects.exclude(
                        profile=profile).filter(*lstQ).distinct().values('id')
                ]
                # lstBack = EqualChange.objects.exclude(profile=profile).filter(*lstQ).distinct()
        except:
            msg = oErr.get_error_message()
            oErr.DoError("EqualChange/get_review_list")
        return lstBack
示例#6
0
    def calculate_matches(self, ssglists):
        """Calculate the number of pm-matches for each list from ssglists"""

        oErr = ErrHandle()
        lBack = []
        try:
            # Preparation: create a list of SSG ids per list
            for oItem in ssglists:
                oItem['ssgid'] = [x['super'] for x in oItem['ssglist']]
                oItem['unique_matches'] = 0

            # Calculate the number of matches for each SSglist
            for idx_list in range(len(ssglists)):
                # Take this as the possible pivot list
                lPivot = ssglists[idx_list]['ssgid']

                # Walk all lists that are not this pivot and count matches
                lUnique = []
                oMatches = {}
                if 'matchset' in ssglists[idx_list]['title']:
                    oMatches = ssglists[idx_list]['title']['matchset']
                for idx, setlist in enumerate(ssglists):
                    if idx != idx_list:
                        # Get this ssglist
                        lSsgId = setlist['ssgid']

                        # Start calculating the number of matches this list has with the currently suggested PM
                        sKey = str(ssglists[idx]['title']['order'])
                        lMatches = []

                        # Consider all ssg id's in this list
                        for ssg_id in lSsgId:
                            # Global unique matches
                            if ssg_id in lPivot and not ssg_id in lUnique:
                                lUnique.append(ssg_id)
                            # Calculation with respect to the currently suggested PM
                            if ssg_id in lPivot:  # and not ssg_id in lMatches:
                                lMatches.append(ssg_id)
                        # Keep track of the matches (for sorting)
                        oMatches[sKey] = len(lMatches)
                # Store the between-list matches
                ssglists[idx_list]['title']['matchset'] = oMatches

                # Store the number of matches for this list
                unique_matches = len(lUnique)
                ssglists[idx_list]['unique_matches'] = unique_matches

            # What we return
            lBack = ssglists
        except:
            msg = oErr.get_error_message()
            oErr.DoError("ResearchSet/calculate_matches")
            lBack = None
        # Return the status
        return lBack
示例#7
0
    def update_ssglists(self, force=False):
        """Re-calculate the set of lists for this particular ResearchSet"""

        oErr = ErrHandle()
        bResult = True
        lst_ssglists = []
        try:
            oPMlist = None
            # Get the lists of SSGs for each list in the set
            for idx, setlist in enumerate(
                    self.researchset_setlists.all().order_by('order')):
                # Check for the contents
                if force or setlist.contents == "" or len(
                        setlist.contents) < 3 or setlist.contents[0] == "[":
                    setlist.calculate_contents()

                # Retrieve the SSG-list from the contents
                oSsgList = json.loads(setlist.contents)

                # If this is not the *first* setlist, calculate the number of matches with the first
                if idx == 0:
                    oPMlist = copy.copy(oSsgList)
                else:
                    # Calculate the matches
                    oSsgList['title']['matches'] = get_list_matches(
                        oPMlist, oSsgList)
                # Always pass on the default order
                oSsgList['title']['order'] = idx + 1

                # Add the list object to the list
                lst_ssglists.append(oSsgList)

            # Calculate the unique_matches for each list
            lst_ssglists = self.calculate_matches(lst_ssglists)

            # Put it in the ResearchSet and save it
            self.contents = json.dumps(lst_ssglists)
            self.save()

            # All related SetDef items should be warned
            with transaction.atomic():
                for obj in SetDef.objects.filter(researchset=self.id):
                    contents = json.loads(obj.contents)
                    contents['recalc'] = True
                    obj.contents = json.dumps(contents)
                    obj.save()

            # Return this list of lists
            bResult = True
        except:
            msg = oErr.get_error_message()
            oErr.DoError("ResearchSet/update_ssglists")
            bResult = False
        return bResult
示例#8
0
 def add_names(main_list, fragment):
     oErr = ErrHandle()
     try:
         for sentence in fragment.replace("_", "").split("."):
             # WOrk through non-initial words
             words = re.split(r'\s+', sentence)
             for word in words[1:]:
                 if re.match(re_name, word):
                     if not word in main_list:
                         main_list.append(word)
     except:
         msg = oErr.get_error_message()
         oErr.DoError("add_names")
示例#9
0
    def calculate_pm(self):
        """Calculate the Pivot Manuscript/item for this research set"""

        oErr = ErrHandle()
        iBack = -1
        try:
            # Get the lists that we have made
            ssglists = self.get_ssglists()

            # Calculate the matches
            ssglists = self.calculate_matches(ssglists)

            # Figure out what the first best list is
            idx_pm = -1
            max_matches = -1
            min_order = len(ssglists) + 2
            min_year_start = 3000
            min_year_finish = 3000
            for idx, oListItem in enumerate(ssglists):
                unique_matches = oListItem['unique_matches']
                year_start = oListItem['title']['yearstart']
                year_finish = oListItem['title']['yearfinish']
                order = oListItem['title']['order']

                # Check which is the best so far
                bTakeThis = False
                bTakeThis = (unique_matches > max_matches)
                if not bTakeThis and unique_matches == max_matches:
                    bTakeThis = (year_start < min_year_start)
                    if not bTakeThis and year_start == min_year_start:
                        bTakeThis = (year_finish < min_year_finish)
                        if not bTakeThis and year_finish == min_year_finish:
                            bTakeThis = (order < min_order)

                # Adapt if this is the one
                if bTakeThis:
                    max_matches = unique_matches
                    min_year_start = year_start
                    min_year_finish = year_finish
                    min_order = order
                    idx_pm = idx

            # What we return is the 'order' value of the best matching list
            iBack = ssglists[idx_pm]['title']['order']

        except:
            msg = oErr.get_error_message()
            oErr.DoError("ResearchSet/calculate_pm")
            iBack = -1
        # Return the PM that we have found
        return iBack
示例#10
0
    def get_ssg_list(self):
        """Create a list of SSGs,depending on the type I am"""

        oErr = ErrHandle()
        lBack = None
        collection_types = ['hist', 'ssgd']
        try:
            if self.setlisttype == "manu":
                # Create a list of SSG's from the manuscript
                lBack = self.ssgs_manuscript(self.manuscript)
            elif self.setlisttype in collection_types:
                lBack = self.ssgs_collection(self.collection)
        except:
            msg = oErr.get_error_message()
            oErr.DoError("SetList/get_ssg_list")
        return lBack
示例#11
0
def get_watermark():
    """Create and return a watermark"""

    oErr = ErrHandle()
    watermark_template = "seeker/passim_watermark.html"
    watermark = ""
    try:
        # create a watermark with the right datestamp
        context_wm = dict(
            datestamp=get_crpp_date(get_current_datetime(), True))
        watermark = render_to_string(watermark_template, context_wm)
    except:
        msg = oErr.get_error_message()
        oErr.DoError("get_watermark")
    # Return the result
    return watermark
示例#12
0
def get_ssg_passim(ssg_id, obj=None):
    oErr = ErrHandle()
    code = ""
    try:
        # Get the Passim Code
        if obj == None:
            obj = EqualGold.objects.filter(id=ssg_id).first()
        if obj.code == None:
            code = "eqg_{}".format(obj.id)
        elif " " in obj.code:
            code = obj.code.split(" ")[1]
        else:
            code = obj.code
    except:
        msg = oErr.get_error_message()
        oErr.DoError("get_ssg_passim")
    return code
示例#13
0
    def add_item(super, profile, field, oChange, oCurrent=None):
        """Add one item"""

        oErr = ErrHandle()
        obj = None
        try:
            # Make sure to stringify, sorting the keys
            change = json.dumps(oChange, sort_keys=True)
            if oCurrent is None:
                current = None
            else:
                current = json.dumps(oCurrent, sort_keys=True)

            # Look for this particular change, supposing it has not changed yet
            obj = EqualChange.objects.filter(super=super,
                                             profile=profile,
                                             field=field,
                                             current=current,
                                             change=change).first()
            if obj == None or obj.changeapprovals.count() > 0:
                # Less restricted: look for any suggestion for a change on this field that has not been reviewed by anyone yet.
                bFound = False
                for obj in EqualChange.objects.filter(super=super,
                                                      profile=profile,
                                                      field=field,
                                                      atype="def"):
                    if obj.changeapprovals.exclude(atype="def").count() == 0:
                        # We can use this one
                        bFound = True
                        obj.current = current
                        obj.change = change
                        obj.save()
                        break
                # What if nothing has been found?
                if not bFound:
                    # Only in that case do we make a new suggestion
                    obj = EqualChange.objects.create(super=super,
                                                     profile=profile,
                                                     field=field,
                                                     current=current,
                                                     change=change)
        except:
            msg = oErr.get_error_message()
            oErr.DoError("EqualChange/add_item")
        return obj
示例#14
0
def listview_adaptations(lv):
    """Perform adaptations specific for this listview"""

    oErr = ErrHandle()
    try:
        if lv in adaptation_list:
            for adapt in adaptation_list.get(lv):
                sh_done  = Information.get_kvalue(adapt)
                if sh_done == None or sh_done != "done":
                    # Do the adaptation, depending on what it is
                    method_to_call = "adapt_{}".format(adapt)
                    bResult, msg = globals()[method_to_call]()
                    if bResult:
                        # Success
                        Information.set_kvalue(adapt, "done")
    except:
        msg = oErr.get_error_message()
        oErr.DoError("listview_adaptations")
示例#15
0
    def check_projects(profile):
        """Check of [profile] needs to have any EqualApprove objects

        And if he does: create them for him
        """

        oErr = ErrHandle()
        iCount = 0
        try:
            # Walk through all the changes that I have suggested
            qs = EqualChange.objects.filter(profile=profile)
            for change in qs:
                # Check the approval of this particular one
                iCount += change.check_approval()
            # All should be up to date now
        except:
            msg = oErr.get_error_message()
            oErr.DoError("EqualChange/check_projects")
        return iCount
示例#16
0
    def get_contents(self, pivot_id=None, recalculate=False):
        """Get the set of lists for this particular DCT"""

        oErr = ErrHandle()
        oBack = None
        try:
            # Get the SSGlists from the research set
            ssglists = self.researchset.get_ssglists(recalculate)

            # Possibly get (stored) parameters from myself
            params = {}
            if self.contents != "" and self.contents[0] == "{":
                params = json.loads(self.contents)

            # Return the parameters and the lists
            oBack = dict(params=params, ssglists=ssglists)

        except:
            msg = oErr.get_error_message()
            oErr.DoError("SetDef/get_contents")
        return oBack
示例#17
0
    def __init__(self, *args, **kwargs):
        # Obligatory for this type of form!!!
        self.username = kwargs.pop('username', "")
        self.team_group = kwargs.pop('team_group', "")
        self.userplus = kwargs.pop('userplus', "")
        # Start by executing the standard handling
        super(SetDefForm, self).__init__(*args, **kwargs)

        oErr = ErrHandle()
        try:
            # Some fields are not required
            self.fields['name'].required = False
            self.fields['notes'].required = False

            # Set queryset(s) - for details view
            self.fields['manulist'].queryset = Manuscript.objects.none()

            # Set the widgets correctly
            self.fields['histlist'].widget = CollOneHistWidget(
                attrs={
                    'username': self.username,
                    'team_group': self.team_group,
                    'settype': 'hc',
                    'data-placeholder': 'Select a historical collection...',
                    'style': 'width: 100%;',
                    'class': 'searching'
                })
            # Note: the collection filters must use the SCOPE of the collection
            self.fields['histlist'].queryset = Collection.get_scoped_queryset(
                'super', self.username, self.team_group, settype="hc")

            # Get the instance
            if 'instance' in kwargs:
                instance = kwargs['instance']

        except:
            msg = oErr.get_error_message()
            oErr.DoError("SetDefForm")
        return None
示例#18
0
def get_ssg_sig(ssg_id):

    oErr = ErrHandle()
    sig = ""
    editype_preferences = ['gr', 'cl', 'ot']
    issue_375 = True
    try:
        for editype in editype_preferences:
            siglist = Signature.objects.filter(
                gold__equal__id=ssg_id,
                editype=editype).order_by('code').values('code')
            if len(siglist) > 0:
                sig = siglist[0]['code']
                break
        # We now have the most appropriate signature, or the empty string, if there is none
        # The folloing is old code, not quite clear why it was necessary
        if not issue_375 and ',' in sig:
            sig = sig.split(",")[0].strip()
    except:
        msg = oErr.get_error_message()
        oErr.DoError("get_ssg_sig")
    return sig
示例#19
0
def get_ssg_corpus(profile, instance):
    oErr = ErrHandle()
    lock_status = "new"
    ssg_corpus = None
    try:

        # Set the lock, or return if we are busy with this ssg_corpus
        ssg_corpus = EqualGoldCorpus.objects.filter(profile=profile,
                                                    ssg=instance).last()
        if ssg_corpus != None:
            lock_status = ssg_corpus.status
            # Check the status
            if lock_status == "busy":
                # Already busy
                return context
        else:
            # Need to create a lock
            ssg_corpus = EqualGoldCorpus.objects.create(profile=profile,
                                                        ssg=instance)
            lock_status = "busy"

        # Save the status
        ssg_corpus.status = lock_status
        ssg_corpus.save()

        if lock_status == "new":
            # Remove earlier corpora made by me based on this SSG
            EqualGoldCorpus.objects.filter(profile=profile,
                                           ssg=instance).delete()

            # Create a new one
            ssg_corpus = EqualGoldCorpus.objects.create(profile=profile,
                                                        ssg=instance,
                                                        status="busy")

    except:
        msg = oErr.get_error_message()
        oErr.DoError("get_ssg_corpus")
    return ssg_corpus, lock_status
示例#20
0
    def check_approval(self):
        """Check if all who need it have an EqualApprove object for this one"""

        oErr = ErrHandle()
        iCount = 0
        try:
            # Check which editors should have an approval object (excluding myself)
            change = self
            profile = self.profile
            lst_approver = change.get_approver_list(profile)
            for approver in lst_approver:
                # Check if an EqualApprove exists
                approval = EqualApproval.objects.filter(
                    change=change, profile=approver).first()
                if approval is None:
                    # Create one
                    approval = EqualApproval.objects.create(change=change,
                                                            profile=approver)
                    iCount = 1
        except:
            msg = oErr.get_error_message()
            oErr.DoError("EqualChange/check_approval")
        return iCount
示例#21
0
    def get_setlist(self, pivot_id=None):
        """Get the set of lists for this particular DCT"""

        oErr = ErrHandle()
        oBack = None
        try:
            # Get to the research set
            id_list = [
                x['id']
                for x in self.researchset.researchset_setlists.all().order_by(
                    'order').values("id")
            ]
            lst_rset = []
            if pivot_id != None and pivot_id in id_list:
                lst_rset.append(pivot_id)
            # Add the remaining ID's
            for id in id_list:
                if not id in lst_rset: lst_rset.append(id)

            # Check the number of lists in the research set
            if lst_rset == None or len(lst_rset) < 2:
                oErr.Status("Not enough SSG-lists to compare")
                return None

            # We have enough lists: Get the lists of SSGs for each
            lst_ssglists = []
            for setlist_id in lst_rset:
                # Get the actual setlist
                setlist = SetList.objects.filter(id=setlist_id).first()
                if setlist != None:
                    # Create an empty SSG-list
                    oSsgList = {}
                    # Add the object itself
                    oSsgList['obj'] = setlist
                    # Add the name object for this list
                    oSsgList['title'] = setlist.get_title_object()
                    # Get the list of SSGs for this list
                    oSsgList['ssglist'] = setlist.get_ssg_list()
                    # Add the list object to the list
                    lst_ssglists.append(oSsgList)

            # Return this list of lists
            oBack = dict(ssglists=lst_ssglists)
            # Prepare and create an appropriate table = list of rows
            rows = []
            # Create header row
            oRow = []
            oRow.append('Gr/Cl/Ot')
            for oSsgList in lst_ssglists:
                # Add the title *object*
                oRow.append(oSsgList['title'])
            rows.append(oRow)

            # Start out with the pivot: the *first* one in 'ssglist'
            lst_pivot = lst_ssglists[0]
            for oPivot in lst_pivot['ssglist']:
                # Create a row based on this pivot
                oRow = []
                # (1) row header
                oRow.append(oPivot['sig'])
                # (2) pivot SSG number
                oRow.append(oPivot['order'])
                # (3) SSG number in all other manuscripts
                ssg_id = oPivot['super']
                for lst_this in lst_ssglists[1:]:
                    bFound = False
                    order = ""
                    for oItem in lst_this['ssglist']:
                        if ssg_id == oItem['super']:
                            # Found it!
                            order = oItem['order']
                            bFound = True
                            break
                    oRow.append(order)
                # (4) add the row to the list
                rows.append(oRow)
            # Make sure we return the right information
            oBack['setlist'] = rows
        except:
            msg = oErr.get_error_message()
            oErr.DoError("SetDef/get_setlist")
        return oBack
示例#22
0
    def ssgs_manuscript(self, bDebug=False):
        """Get the ordered list of SSGs related to a manuscript"""

        oErr = ErrHandle()
        lBack = None
        try:
            manu = self.manuscript
            # Get a list of SSGs to which the sermons in this manuscript point
            # Per issue #402 additional info needs to be there:
            #   author,
            #   PASSIM code,
            #   incipit, explicit, HC (if the SSG is part of a historical collection),
            #   number of SGs contained in the equality set,
            #   number of links to other SSGs
            # Per issue #402, phase 2, even more info needs to be added:
            #   for MANUSCRIPT source-lists, there should be an option for the user
            #       to select additional SERMON manifestation details fields to be shown
            #       (options: attributed author, section title, lectio, title, incipit, explicit,
            #       postscriptum, feast, bible reference, cod. notes, notes, keywords)
            qs = manu.sermondescr_super.all().order_by(
                'sermon__msitem__order').values(
                    'sermon', 'sermon__msitem__order', 'super', 'super__code',
                    'super__author__name', 'super__incipit', 'super__explicit',
                    'super__sgcount', 'super__ssgcount',
                    'sermon__author__name', 'sermon__sectiontitle',
                    'sermon__quote', 'sermon__title', 'sermon__incipit',
                    'sermon__explicit', 'sermon__postscriptum',
                    'sermon__feast__name', 'sermon__bibleref',
                    'sermon__additional', 'sermon__note')
            # NOTE: the 'keywords' for issue #402 are a bit more cumbersome to collect...
            lBack = []
            with transaction.atomic():
                for obj in qs:
                    # Get the order number
                    order = obj['sermon__msitem__order']
                    # Get the SSG and all its characteristics
                    super = obj['super']
                    code = obj['super__code']
                    authorname = obj['super__author__name']
                    incipit = obj['super__incipit']
                    explicit = obj['super__explicit']
                    sgcount = obj['super__sgcount']
                    ssgcount = obj['super__ssgcount']
                    # Sermon characteristics
                    sermon = obj['sermon']
                    srm_author = obj['sermon__author__name']
                    srm_sectiontitle = obj['sermon__sectiontitle']
                    srm_lectio = obj['sermon__quote']
                    srm_title = obj['sermon__title']
                    srm_incipit = obj['sermon__incipit']
                    srm_explicit = obj['sermon__explicit']
                    srm_postscriptum = obj['sermon__postscriptum']
                    srm_feast = obj['sermon__feast__name']
                    srm_bibleref = obj['sermon__bibleref']
                    srm_codnotes = obj['sermon__additional']
                    srm_notes = obj['sermon__note']

                    # Get the name(s) of the HC
                    lst_hc = CollectionSuper.objects.filter(
                        super=super, collection__settype="hc").values(
                            'collection__id', 'collection__name')
                    hcs = ", ".join([x['collection__name'] for x in lst_hc])
                    # ======== DEBUGGING ============
                    if hcs != "":
                        iStop = 1
                    # ===============================

                    # TODO: Get the keywords
                    lst_kw = SermonDescrKeyword.objects.filter(
                        sermon=sermon).values('keyword__name')
                    kws = ", ".join([x['keyword__name'] for x in lst_kw])

                    # Get a URL for this ssg
                    url = reverse('equalgold_details', kwargs={'pk': super})
                    # Treat signatures for this SSG: get the best for showing
                    sigbest = get_goldsig_dct(super)
                    if sigbest == "":
                        sigbest = "ssg_{}".format(super)
                    # Signatures for this SSG: get the full list
                    siglist = get_goldsiglist_dct(super)
                    # Put into object
                    oItem = dict(super=super,
                                 sig=sigbest,
                                 siglist=siglist,
                                 hcs=hcs,
                                 kws=kws,
                                 order=order,
                                 code=code,
                                 url=url,
                                 author=authorname,
                                 type='ms',
                                 incipit=incipit,
                                 explicit=explicit,
                                 sgcount=sgcount,
                                 ssgcount=ssgcount,
                                 srm_author=srm_author,
                                 srm_sectiontitle=srm_sectiontitle,
                                 srm_lectio=srm_lectio,
                                 srm_title=srm_title,
                                 srm_incipit=srm_incipit,
                                 srm_explicit=srm_explicit,
                                 srm_postscriptum=srm_postscriptum,
                                 srm_feast=srm_feast,
                                 srm_bibleref=srm_bibleref,
                                 srm_codnotes=srm_codnotes,
                                 srm_notes=srm_notes)
                    # Add to list
                    lBack.append(oItem)

            # Debugging: print the list
            if bDebug:
                print("Manuscript id={}".format(manu.id))
                for oItem in lBack:
                    order = oItem.get("order")
                    super = oItem.get("super")
                    sig = oItem.get("sig")
                    code = get_passimcode(super, code)
                    print("sermon {}: ssg={} sig=[{}]".format(
                        order, code, sig))

        except:
            msg = oErr.get_error_message()
            oErr.DoError("ssgs_manuscript")
        return lBack
示例#23
0
    def do_overlap(self, ssg_link, degree):
        """Calculate the overlap network up until 'degree'"""
        def add_nodeset(ssg_id, group):
            node_key = ssg_id

            # ---------- DEBUG --------------
            if node_key == 4968:
                iStop = 1
            # -------------------------------

            # Possibly add the node to the set
            if not node_key in node_set:
                code_sig = get_ssg_sig(ssg_id)
                ssg = EqualGold.objects.filter(id=ssg_id).first()
                code_pas = get_ssg_passim(ssg_id, ssg)
                scount = ssg.scount

                # Get a list of the HCs that this SSG is part of
                hc_list = ssg.collections.filter(settype="hc").values(
                    "id", "name")
                hcs = []
                for oItem in hc_list:
                    id = oItem['id']
                    if not id in hist_set:
                        hist_set[id] = oItem['name']
                    hcs.append(id)

                node_value = dict(label=code_sig,
                                  id=ssg_id,
                                  group=group,
                                  passim=code_pas,
                                  scount=scount,
                                  hcs=hcs)
                node_set[node_key] = node_value

        node_list = []
        link_list = []
        node_set = {}
        link_set = {}
        hist_set = {}
        max_value = 0
        max_group = 1
        oErr = ErrHandle()

        try:
            # Degree 0: direct contacts
            ssg_id = self.obj.id

            ssg_queue = [ssg_id]
            ssg_add = []
            group = 1
            degree -= 1
            while len(ssg_queue) > 0 and degree >= 0:
                # Get the next available node
                node_key = ssg_queue.pop()

                # Possibly add the node to the set
                add_nodeset(node_key, group)

                # Add links from this SSG to others
                if node_key in ssg_link:
                    dst_list = ssg_link[node_key]
                    for oDst in dst_list:
                        dst = oDst['dst']
                        # Add this one to the add list
                        ssg_add.append(dst)
                        link_key = "{}_{}".format(node_key, dst)
                        if not link_key in link_set:
                            oLink = dict(source=node_key,
                                         target=dst,
                                         spectype=oDst['spectype'],
                                         linktype=oDst['linktype'],
                                         spec=oDst['spec'],
                                         link=oDst['link'],
                                         alternatives=oDst['alternatives'],
                                         note=oDst['note'],
                                         value=0)
                            # Add the link to the set
                            link_set[link_key] = oLink
                            # Add the destination node to nodeset
                            add_nodeset(dst, group + 1)

                        # Increment the link value
                        link_set[link_key]['value'] += 1

                # Go to the next degree
                if len(ssg_queue) == 0 and degree >= 0:
                    # Decrement the degree
                    degree -= 1

                    group += 1
                    # Add the items from ssg_add into the queue
                    while len(ssg_add) > 0:
                        ssg_queue.append(ssg_add.pop())

            # Turn the sets into lists
            node_list = [v for k, v in node_set.items()]
            link_list = [v for k, v in link_set.items()]

            # Calculate max_value
            for oItem in link_list:
                value = oItem['value']
                if value > max_value: max_value = value
            for oItem in node_list:
                group = oItem['group']
                if group > max_group: max_group = group

        except:
            msg = oErr.get_error_message()
            oErr.DoError("EqualGoldOverlap/do_overlap")

        return node_list, link_list, hist_set, max_value, max_group
示例#24
0
    def add_to_context(self, context):
        def add_to_dict(this_dict, item):
            if item != "":
                if not item in this_dict:
                    this_dict[item] = 1
                else:
                    this_dict[item] += 1

        oErr = ErrHandle()
        try:
            # Need to figure out who I am
            profile = Profile.get_user_profile(self.request.user.username)
            instance = self.obj

            networkslider = self.qd.get("network_trans_slider", "1")
            if isinstance(networkslider, str):
                networkslider = int(networkslider)

            # Get the 'manuscript-corpus': all manuscripts in which a sermon is that belongs to the same SSG
            manu_list = SermonDescrEqual.objects.filter(
                super=instance).distinct().values("manu_id")
            manu_dict = {}
            for idx, manu in enumerate(manu_list):
                manu_dict[manu['manu_id']] = idx + 1

            ssg_corpus, lock_status = get_ssg_corpus(profile, instance)

            if lock_status != "ready":

                # Create an EqualGoldCorpus based on the SSGs in these manuscripts
                ssg_list = SermonDescrEqual.objects.filter(
                    manu__id__in=manu_list).order_by(
                        'super_id').distinct().values('super_id')
                ssg_list_id = [x['super_id'] for x in ssg_list]
                with transaction.atomic():
                    for ssg in EqualGold.objects.filter(id__in=ssg_list_id):
                        # Get the name of the author
                        authorname = "empty" if ssg.author == None else ssg.author.name
                        # Get the scount
                        scount = ssg.scount
                        # Create new ssg_corpus item
                        obj = EqualGoldCorpusItem.objects.create(
                            corpus=ssg_corpus,
                            equal=ssg,
                            authorname=authorname,
                            scount=scount)

                # Add this list to the ssg_corpus
                ssg_corpus.status = "ready"
                ssg_corpus.save()

            node_list, link_list, author_list, max_value = self.do_manu_method(
                ssg_corpus, manu_list, networkslider)

            # Add the information to the context in data
            context['data'] = dict(node_list=node_list,
                                   link_list=link_list,
                                   watermark=get_watermark(),
                                   author_list=author_list,
                                   max_value=max_value,
                                   networkslider=networkslider,
                                   legend="SSG network")

            # Can remove the lock
            ssg_corpus.status = "ready"
            ssg_corpus.save()

        except:
            msg = oErr.get_error_message()
            oErr.DoError("EqualGoldTrans/add_to_context")

        return context
示例#25
0
    def ssgs_collection(self, bDebug=False):
        """Get the ordered list of SSGs in the [super] type collection"""

        oErr = ErrHandle()
        lBack = None
        try:
            coll = self.collection
            # Get the list of SSGs inside this collection
            # Per issue #402 additional info needs to be there:
            #   author,
            #   PASSIM code,
            #   incipit, explicit, HC (if the SSG is part of a historical collection),
            #   number of SGs contained in the equality set,
            #   number of links to other SSGs
            # HC/PD specific:
            #   name
            #   description
            #   size
            # HC-specific:
            #   literature
            qs = CollectionSuper.objects.filter(
                collection=coll).order_by('order').values(
                    'order', 'collection__name', 'collection__descrip',
                    'collection__settype', 'super', 'super__code',
                    'super__author__name', 'super__incipit', 'super__explicit',
                    'super__sgcount', 'super__ssgcount')
            lBack = []
            with transaction.atomic():
                for obj in qs:
                    # Get the order number
                    order = obj['order']
                    # Get the collection-specific information
                    name = obj['collection__name']
                    descr = obj['collection__descrip']
                    settype = obj['collection__settype']
                    # Get the SSG
                    super = obj['super']
                    code = obj['super__code']
                    authorname = obj['super__author__name']
                    incipit = obj['super__incipit']
                    explicit = obj['super__explicit']
                    sgcount = obj['super__sgcount']
                    ssgcount = obj['super__ssgcount']
                    # Get the name(s) of the HC
                    lst_hc = CollectionSuper.objects.filter(
                        super=super, collection__settype="hc").values(
                            'collection__id', 'collection__name')
                    hcs = ", ".join([x['collection__name'] for x in lst_hc])
                    # ======== DEBUGGING ============
                    if hcs != "":
                        iStop = 1
                    # ===============================
                    # Get a URL for this ssg
                    url = reverse('equalgold_details', kwargs={'pk': super})
                    # Treat signatures for this SSG
                    sigbest = get_goldsig_dct(super)
                    if sigbest == "":
                        sigbest = "ssg_{}".format(super)
                    # Signatures for this SSG: get the full list
                    siglist = get_goldsiglist_dct(super)
                    # Put into object
                    oItem = dict(super=super,
                                 sig=sigbest,
                                 siglist=siglist,
                                 name=name,
                                 descr=descr,
                                 type=settype,
                                 order=order,
                                 code=code,
                                 url=url,
                                 author=authorname,
                                 incipit=incipit,
                                 explicit=explicit,
                                 sgcount=sgcount,
                                 ssgcount=ssgcount,
                                 hcs=hcs)
                    # Add to list
                    lBack.append(oItem)

            # Debugging: print the list
            if bDebug:
                print("Collection id={}".format(coll.id))
                for oItem in lBack:
                    order = oItem.get("order")
                    super = oItem.get("super")
                    sig = oItem.get("sig")
                    code = get_passimcode(super, code)
                    print("sermon {}: ssg={} sig=[{}]".format(
                        order, code, sig))

        except:
            msg = oErr.get_error_message()
            oErr.DoError("ssgs_collection")
        return lBack
示例#26
0
    def do_manu_method(self, ssg_corpus, manu_list, min_value):
        """Calculation like 'MedievalManuscriptTransmission' description
        
        The @min_value is the minimum link-value the user wants to see
        """

        oErr = ErrHandle()
        author_dict = {}
        node_list = []
        link_list = []
        max_value = 0  # Maximum number of manuscripts in which an SSG occurs
        max_scount = 1  # Maximum number of sermons associated with one SSG

        try:
            # Initializations
            ssg_dict = {}
            link_dict = {}
            scount_dict = {}
            node_listT = []
            link_listT = []
            node_set = {}  # Put the nodes in a set, with their SSG_ID as key
            title_set = {}  # Link from title to SSG_ID

            # Walk the ssg_corpus: each SSG is one 'text', having a title and a category (=author code)
            for idx, item in enumerate(
                    EqualGoldCorpusItem.objects.filter(
                        corpus=ssg_corpus).values('authorname', 'scount',
                                                  'equal__code', 'equal__id')):
                # Determine the name for this row
                category = item['authorname']
                ssg_id = item['equal__id']
                code = item['equal__code']
                if code == None or code == "" or not " " in code or not "." in code:
                    title = "eqg{}".format(ssg_id)
                else:
                    title = code.split(" ")[1]
                # Get the Signature that is most appropriate
                sig = get_ssg_sig(ssg_id)

                # Add author to dictionary
                if not category in author_dict: author_dict[category] = 0
                author_dict[category] += 1

                # the scount and store it in a dictionary
                scount = item['scount']
                scount_dict[ssg_id] = scount
                if scount > max_scount:
                    max_scount = scount

                node_key = ssg_id
                node_value = dict(label=title,
                                  category=category,
                                  scount=scount,
                                  sig=sig,
                                  rating=0)
                if node_key in node_set:
                    oErr.Status(
                        "EqualGoldGraph/do_manu_method: attempt to add same title '{}' for {} and {}"
                        .format(title, ssg_id, title_set[title]))
                else:
                    node_set[node_key] = node_value
                    title_set[title] = ssg_id

            # Create list of authors
            author_list = [
                dict(category=k, count=v) for k, v in author_dict.items()
            ]
            author_list = sorted(author_list,
                                 key=lambda x:
                                 (-1 * x['count'], x['category'].lower()))

            # Create a dictionary of manuscripts, each having a list of SSG ids
            manu_set = {}
            for manu_item in manu_list:
                manu_id = manu_item["manu_id"]
                # Get a list of all SSGs in this manuscript
                ssg_list = SermonDescrEqual.objects.filter(
                    manu__id=manu_id).order_by('super_id').distinct()
                # Add the SSG id list to the manuset
                manu_set[manu_id] = [x.super for x in ssg_list]

            # Create a list of edges based on the above
            link_dict = {}
            for manu_id, ssg_list in manu_set.items():
                # Only treat ssg_lists that are larger than 1
                if len(ssg_list) > 1:
                    # itertool.combinations creates all combinations of SSG to SSG in one manuscript
                    for subset in itertools.combinations(ssg_list, 2):
                        source = subset[0]
                        target = subset[1]
                        source_id = source.id
                        target_id = target.id
                        link_code = "{}_{}".format(source_id, target_id)
                        if link_code in link_dict:
                            oLink = link_dict[link_code]
                        else:
                            oLink = dict(source=source_id,
                                         target=target_id,
                                         value=0)
                            link_dict[link_code] = oLink
                        # Add 1
                        oLink['value'] += 1
                        if oLink['value'] > max_value:
                            max_value = oLink['value']
            # Turn the link_dict into a list
            # link_list = [v for k,v in link_dict.items()]

            # Only accept the links that have a value >= min_value
            node_dict = {}
            link_list = []
            for k, oItem in link_dict.items():
                if oItem['value'] >= min_value:
                    link_list.append(copy.copy(oItem))
                    # Take note of the nodes
                    src = oItem['source']
                    dst = oItem['target']
                    if not src in node_dict: node_dict[src] = node_set[src]
                    if not dst in node_dict: node_dict[dst] = node_set[dst]
            # Walk the nodes
            node_list = []
            for ssg_id, oItem in node_dict.items():
                oItem['id'] = ssg_id
                oItem['scount'] = 100 * scount_dict[oItem['id']] / max_scount
                node_list.append(copy.copy(oItem))

        except:
            msg = oErr.get_error_message()
            oErr.DoError("do_manu_method")

        return node_list, link_list, author_list, max_value
示例#27
0
    def do_manu_method(self, ssg_corpus, manu_list, min_value):
        """Calculation like Manuscript_Transmission_Of_se172
        
        The @min_value is the minimum link-value the user wants to see
        """
        oErr = ErrHandle()
        node_list = []
        link_list = []
        max_value = 0  # Maximum number of manuscripts in which an SSG occurs
        max_scount = 1  # Maximum number of sermons associated with one SSG
        manu_count = []

        def get_nodes(crp):
            nodes = []
            for idx, item in enumerate(crp.target_ints):
                oNode = dict(group=item,
                             author=crp.target_idx[item],
                             id=crp.titles[idx])
                nodes.append(oNode)
            return nodes

        try:
            # Create a pystyl-corpus object (see above)
            sty_corpus = Corpus(texts=[],
                                titles=[],
                                target_ints=[],
                                target_idx=[])

            ssg_dict = {}
            link_dict = {}
            scount_dict = {}
            node_listT = []
            link_listT = []

            # Initialize manu_count: the number of SSG co-occurring in N manuscripts
            for item in manu_list:
                manu_count.append(0)

            # Walk the ssg_corpus: each SSG is one 'text', having a title and a category (=author code)
            for idx, item in enumerate(
                    EqualGoldCorpusItem.objects.filter(
                        corpus=ssg_corpus).values('words', 'authorname',
                                                  'scount', 'equal__code',
                                                  'equal__id')):
                # Determine the name for this row
                category = item['authorname']
                code = item['equal__code']
                if code == None or code == "" or not " " in code or not "." in code:
                    title = "eqg{}".format(item['equal__id'])
                else:
                    title = code.split(" ")[1]
                # The text = the words
                text = " ".join(json.loads(item['words']))

                # the scount and store it in a dictionary
                scount_dict[title] = item['scount']
                if item['scount'] > max_scount:
                    max_scount = item['scount']

                # Add the text to the corpus
                if title in sty_corpus.titles:
                    ssg_id = -1
                    bFound = False
                    for k, v in ssg_dict.items():
                        if v == title:
                            ssg_id = k
                            bFound = True
                            break
                    oErr.Status(
                        "EqualGoldGraph/do_manu_method: attempt to add same title '{}' for {} and {}"
                        .format(title, ssg_id, item['equal__id']))
                else:
                    # Also make sure to buidl an SSG-dictionary
                    ssg_dict[item['equal__id']] = title

                    sty_corpus.add_text(text, title, category)

            # Get a list of nodes
            node_listT = get_nodes(sty_corpus)

            # Walk the manuscripts
            for manu_item in manu_list:
                manu_id = manu_item["manu_id"]
                # Get a list of all SSGs in this manuscript
                ssg_list = SermonDescrEqual.objects.filter(
                    manu__id=manu_id).order_by('super_id').distinct().values(
                        'super_id')
                ssg_list_id = [x['super_id'] for x in ssg_list]
                # evaluate links between a source and target SSG
                for idx_s, source_id in enumerate(ssg_list_id):
                    # sanity check
                    if source_id in ssg_dict:
                        # Get the title of the source
                        source = ssg_dict[source_id]
                        for idx_t in range(idx_s + 1, len(ssg_list_id) - 1):
                            target_id = ssg_list_id[idx_t]
                            # Double check
                            if target_id in ssg_dict:
                                # Get the title of the target
                                target = ssg_dict[target_id]
                                # Retrieve or create a link from the link_listT
                                link_code = "{}_{}".format(
                                    source_id, target_id)
                                if link_code in link_dict:
                                    oLink = link_listT[link_dict[link_code]]
                                else:
                                    oLink = dict(source=source,
                                                 source_id=source_id,
                                                 target=target,
                                                 target_id=target_id,
                                                 value=0)
                                    link_listT.append(oLink)
                                    link_dict[link_code] = len(link_listT) - 1
                                # Now add to the value
                                oLink['value'] += 1
                                if oLink['value'] > max_value:
                                    max_value = oLink['value']

            # Only accept the links that have a value >= min_value
            node_dict = []
            for oItem in link_listT:
                if oItem['value'] >= min_value:
                    link_list.append(copy.copy(oItem))
                    # Take note of the nodes
                    src = oItem['source']
                    dst = oItem['target']
                    if not src in node_dict: node_dict.append(src)
                    if not dst in node_dict: node_dict.append(dst)
            # Walk the nodes
            for oItem in node_listT:
                if oItem['id'] in node_dict:
                    oItem['scount'] = 100 * scount_dict[
                        oItem['id']] / max_scount
                    node_list.append(copy.copy(oItem))

        except:
            msg = oErr.get_error_message()
            oErr.DoError("do_hier_method1")

        return node_list, link_list, max_value
示例#28
0
    def add_to_context(self, context):

        names_list = [x.lower() for x in COMMON_NAMES]

        def get_author(code):
            """Get the author id from the passim code"""
            author = 10000
            if "PASSIM" in code:
                author = int(code.replace("PASSIM", "").strip().split(".")[0])
            return author

        def add_to_dict(this_dict, item):
            if item != "":
                if not item in this_dict:
                    this_dict[item] = 1
                else:
                    this_dict[item] += 1

        oErr = ErrHandle()
        try:
            # Need to figure out who I am
            profile = Profile.get_user_profile(self.request.user.username)
            instance = self.obj

            # Get the 'manuscript-corpus': the manuscripts in which the same kind of sermon like me is
            manu_list = SermonDescrEqual.objects.filter(
                super=instance).distinct().values("manu_id")
            manu_dict = {}
            for idx, manu in enumerate(manu_list):
                manu_dict[manu['manu_id']] = idx + 1
            author_dict = {}

            lock_status = "new"
            # Set the lock, or return if we are busy with this ssg_corpus
            ssg_corpus = EqualGoldCorpus.objects.filter(profile=profile,
                                                        ssg=instance).last()
            if ssg_corpus != None:
                lock_status = ssg_corpus.status
                # Check the status
                if lock_status == "busy":
                    # Already busy
                    return context
            else:
                # Need to create a lock
                ssg_corpus = EqualGoldCorpus.objects.create(profile=profile,
                                                            ssg=instance)
                lock_status = "busy"

            # Save the status
            ssg_corpus.status = lock_status
            ssg_corpus.save()

            if lock_status == "new":
                # Remove earlier corpora made by me based on this SSG
                EqualGoldCorpus.objects.filter(profile=profile,
                                               ssg=instance).delete()

                # Create a new one
                ssg_corpus = EqualGoldCorpus.objects.create(profile=profile,
                                                            ssg=instance,
                                                            status="busy")

            if lock_status != "ready":

                # Create an EqualGoldCorpus based on the SSGs in these manuscripts
                ssg_list = SermonDescrEqual.objects.filter(
                    manu__id__in=manu_list).order_by(
                        'super_id').distinct().values('super_id')
                ssg_list_id = [x['super_id'] for x in ssg_list]
                all_words = {}
                with transaction.atomic():
                    for ssg in EqualGold.objects.filter(id__in=ssg_list_id):
                        # Add this item to the ssg_corpus
                        latin = {}
                        if ssg.incipit != None:
                            for item in ssg.srchincipit.replace(
                                    ",", "").replace("…", "").split(" "):
                                add_to_dict(latin, item)
                                add_to_dict(all_words, item)
                        if ssg.explicit != None:
                            for item in ssg.srchexplicit.replace(
                                    ",", "").replace("…", "").split(" "):
                                add_to_dict(latin, item)
                                add_to_dict(all_words, item)
                        # Get the name of the author
                        authorname = "empty" if ssg.author == None else ssg.author.name
                        # Create new ssg_corpus item
                        obj = EqualGoldCorpusItem.objects.create(
                            corpus=ssg_corpus,
                            words=json.dumps(latin),
                            equal=ssg,
                            authorname=authorname)

                # What are the 100 MFWs in all_words?
                mfw = [dict(word=k, count=v) for k, v in all_words.items()]
                mfw_sorted = sorted(mfw, key=lambda x: -1 * x['count'])
                mfw_cento = mfw_sorted[:100]
                mfw = []
                for item in mfw_cento:
                    mfw.append(item['word'])
                # Add this list to the ssg_corpus
                ssg_corpus.mfw = json.dumps(mfw)
                ssg_corpus.status = "ready"
                ssg_corpus.save()

            node_list, link_list, max_value = self.do_hier_method3(
                ssg_corpus, names_list)

            # Add the information to the context in data
            context['data'] = dict(node_list=node_list,
                                   link_list=link_list,
                                   watermark=get_watermark(),
                                   max_value=max_value,
                                   legend="SSG network")

            # Can remove the lock
            ssg_corpus.status = "ready"
            ssg_corpus.save()

        except:
            msg = oErr.get_error_message()
            oErr.DoError("EqualGoldPca/add_to_context")

        return context
示例#29
0
    def do_hier_method3(self, ssg_corpus, names_list):
        """Calculate distance matrix with PYSTYL, do the rest myself"""
        oErr = ErrHandle()
        node_list = []
        link_list = []
        max_value = 0

        def dm_to_leo(matrix, crp):
            links = []
            i = 0
            maxv = 0
            for row_id, row in enumerate(matrix):
                # Calculate the link
                minimum = None
                min_id = -1
                for col_idx in range(row_id + 1, len(row) - 1):
                    value = row[col_idx]
                    if minimum == None:
                        if value > 0: minimum = value
                    elif value < minimum:
                        minimum = value
                        min_id = col_idx
                if minimum != None and min_id >= 0:
                    # Create a link
                    oLink = dict(source_id=row_id,
                                 source=crp.titles[row_id],
                                 target_id=min_id,
                                 target=crp.titles[min_id],
                                 value=minimum)
                    links.append(oLink)
                    # Keep track of max_value
                    if minimum > maxv:
                        maxv = minimum

            return links, maxv

        def get_nodes(crp):
            nodes = []
            for idx, item in enumerate(crp.target_ints):
                oNode = dict(group=item,
                             author=crp.target_idx[item],
                             id=crp.titles[idx],
                             scount=5)
                nodes.append(oNode)
            return nodes

        do_example = False
        try:
            # Create a pystyl-corpus object (see above)
            sty_corpus = Corpus(texts=[],
                                titles=[],
                                target_ints=[],
                                target_idx=[])

            ssg_dict = {}

            # Walk the ssg_corpus: each SSG is one 'text', having a title and a category (=author code)
            for idx, item in enumerate(
                    EqualGoldCorpusItem.objects.filter(
                        corpus=ssg_corpus).values('words', 'authorname',
                                                  'equal__code', 'equal__id')):
                # Determine the name for this row
                category = item['authorname']
                code = item['equal__code']
                if code == None or code == "" or not " " in code or not "." in code:
                    title = "eqg{}".format(item['equal__id'])
                else:
                    title = code.split(" ")[1]
                # The text = the words
                text = " ".join(json.loads(item['words']))

                # Add the text to the corpus
                if title in sty_corpus.titles:
                    ssg_id = -1
                    bFound = False
                    for k, v in ssg_dict.items():
                        if v == title:
                            ssg_id = k
                            bFound = True
                            break
                    oErr.Status(
                        "EqualGoldGraph/do_manu_method: attempt to add same title '{}' for {} and {}"
                        .format(title, ssg_id, item['equal__id']))
                else:
                    # Also make sure to buidl an SSG-dictionary
                    ssg_dict[item['equal__id']] = title

                    sty_corpus.add_text(text, title, category)

            # We now 'have' the corpus, so we can work with it...
            sty_corpus.preprocess(alpha_only=True, lowercase=True)
            sty_corpus.tokenize()

            # REmove the common names
            sty_corpus.remove_tokens(rm_tokens=names_list, rm_pronouns=False)

            # Vectorize the corpus
            sty_corpus.vectorize(mfi=200,
                                 ngram_type="word",
                                 ngram_size=1,
                                 vector_space="tf_std")

            # Get a list of nodes
            node_list = get_nodes(sty_corpus)

            # Create a distance matrix
            dm = distance_matrix(sty_corpus, "minmax")

            # Convert the distance matrix into a list of 'nearest links'
            link_list, max_value = dm_to_leo(dm, sty_corpus)

            # Convert the cluster_tree into a node_list and a link_list (i.e. get the list of edges)
            iRead = 1

        except:
            msg = oErr.get_error_message()
            oErr.DoError("do_hier_method1")

        return node_list, link_list, max_value
示例#30
0
def add_codico_to_manuscript(manu):
    """Check if a manuscript has a Codico, and if not create it"""

    def get_number(items, bFirst):
        """Extract the first or last consecutive number from the string"""

        number = -1
        if len(items) > 0:
            if bFirst:
                # Find the first number
                for sInput in items:
                    arNumber = re.findall(r'\d+', sInput)
                    if len(arNumber) > 0:
                        number = int(arNumber[0])
                        break
            else:
                # Find the last number
                for sInput in reversed(items):
                    arNumber = re.findall(r'\d+', sInput)
                    if len(arNumber) > 0:
                        number = int(arNumber[-1])
                        break

        return number
    
    oErr = ErrHandle()
    bResult = False
    msg = ""
    try:
        # Check if the codico exists
        codi = Codico.objects.filter(manuscript=manu).first()
        if codi == None:
            # Get first and last sermons and then their pages
            items = [x['itemsermons__locus'] for x in manu.manuitems.filter(itemsermons__locus__isnull=False).order_by(
                'order').values('itemsermons__locus')]
            if len(items) > 0:
                pagefirst = get_number(items, True)
                pagelast = get_number(items, False)
            else:
                pagefirst = 1
                pagelast = 1
            # Create the codico
            codi = Codico.objects.create(
                name=manu.name, support=manu.support, extent=manu.extent,
                format=manu.format, order=1, pagefirst=pagefirst, pagelast=pagelast,
                origin=manu.origin, manuscript=manu
                )
        else:
            # Possibly copy stuff from manu to codi
            bNeedSaving = False
            if codi.name == "SUPPLY A NAME" and manu.name != "":
                codi.name = manu.name ; bNeedSaving = True
            if codi.support == None and manu.support != None:
                codi.support = manu.support ; bNeedSaving = True
            if codi.extent == None and manu.extent != None:
                codi.extent = manu.extent ; bNeedSaving = True
            if codi.format == None and manu.format != None:
                codi.format = manu.format ; bNeedSaving = True
            if codi.order == 0:
                codi.order = 1 ; bNeedSaving = True
            if codi.origin == None and manu.origin != None:
                codi.origin = manu.origin ; bNeedSaving = True
            # Possibly save changes
            if bNeedSaving:
                codi.save()
        # Copy provenances
        if codi.codico_provenances.count() == 0:
            for mp in manu.manuscripts_provenances.all():
                obj = ProvenanceCod.objects.filter(
                    provenance=mp.provenance, codico=codi, note=mp.note).first()
                if obj == None:
                    obj = ProvenanceCod.objects.create(
                        provenance=mp.provenance, codico=codi, note=mp.note)

        # Copy keywords
        if codi.codico_kw.count() == 0:
            for mk in manu.manuscript_kw.all():
                obj = CodicoKeyword.objects.filter(
                    codico=codi, keyword=mk.keyword).first()
                if obj == None:
                    obj = CodicoKeyword.objects.create(
                        codico=codi, keyword=mk.keyword)

        # Copy date ranges
        if codi.codico_dateranges.count() == 0:
            for md in manu.manuscript_dateranges.all():
                if md.codico_id == None or md.codico_id == 0 or md.codico == None or md.codic.id != codi.id:
                    md.codico = codi
                    md.save()

        # Tie all MsItems that need be to the Codico
        for msitem in manu.manuitems.all().order_by('order'):
            if msitem.codico_id == None or msitem.codico == None or msitem.codico.id != codi.id:
                msitem.codico = codi
                msitem.save()
        bResult = True
    except:
        msg = oErr.get_error_message()
        oErr.DoError("add_codico_to_manuscript")
        bResult = False
    return bResult, msg