def add_to_context(self, context): def add_to_dict(this_dict, item): if item != "": if not item in this_dict: this_dict[item] = 1 else: this_dict[item] += 1 oErr = ErrHandle() try: # Need to figure out who I am profile = Profile.get_user_profile(self.request.user.username) instance = self.obj networkslider = self.qd.get("network_trans_slider", "1") if isinstance(networkslider, str): networkslider = int(networkslider) # Get the 'manuscript-corpus': all manuscripts in which a sermon is that belongs to the same SSG manu_list = SermonDescrEqual.objects.filter( super=instance).distinct().values("manu_id") manu_dict = {} for idx, manu in enumerate(manu_list): manu_dict[manu['manu_id']] = idx + 1 ssg_corpus, lock_status = get_ssg_corpus(profile, instance) if lock_status != "ready": # Create an EqualGoldCorpus based on the SSGs in these manuscripts ssg_list = SermonDescrEqual.objects.filter( manu__id__in=manu_list).order_by( 'super_id').distinct().values('super_id') ssg_list_id = [x['super_id'] for x in ssg_list] with transaction.atomic(): for ssg in EqualGold.objects.filter(id__in=ssg_list_id): # Get the name of the author authorname = "empty" if ssg.author == None else ssg.author.name # Get the scount scount = ssg.scount # Create new ssg_corpus item obj = EqualGoldCorpusItem.objects.create( corpus=ssg_corpus, equal=ssg, authorname=authorname, scount=scount) # Add this list to the ssg_corpus ssg_corpus.status = "ready" ssg_corpus.save() node_list, link_list, author_list, max_value = self.do_manu_method( ssg_corpus, manu_list, networkslider) # Add the information to the context in data context['data'] = dict(node_list=node_list, link_list=link_list, watermark=get_watermark(), author_list=author_list, max_value=max_value, networkslider=networkslider, legend="SSG network") # Can remove the lock ssg_corpus.status = "ready" ssg_corpus.save() except: msg = oErr.get_error_message() oErr.DoError("EqualGoldTrans/add_to_context") return context
def add_to_context(self, context): names_list = [x.lower() for x in COMMON_NAMES] def get_author(code): """Get the author id from the passim code""" author = 10000 if "PASSIM" in code: author = int(code.replace("PASSIM", "").strip().split(".")[0]) return author def add_to_dict(this_dict, item): if item != "": if not item in this_dict: this_dict[item] = 1 else: this_dict[item] += 1 oErr = ErrHandle() try: # Need to figure out who I am profile = Profile.get_user_profile(self.request.user.username) instance = self.obj # Get the 'manuscript-corpus': the manuscripts in which the same kind of sermon like me is manu_list = SermonDescrEqual.objects.filter( super=instance).distinct().values("manu_id") manu_dict = {} for idx, manu in enumerate(manu_list): manu_dict[manu['manu_id']] = idx + 1 author_dict = {} lock_status = "new" # Set the lock, or return if we are busy with this ssg_corpus ssg_corpus = EqualGoldCorpus.objects.filter(profile=profile, ssg=instance).last() if ssg_corpus != None: lock_status = ssg_corpus.status # Check the status if lock_status == "busy": # Already busy return context else: # Need to create a lock ssg_corpus = EqualGoldCorpus.objects.create(profile=profile, ssg=instance) lock_status = "busy" # Save the status ssg_corpus.status = lock_status ssg_corpus.save() if lock_status == "new": # Remove earlier corpora made by me based on this SSG EqualGoldCorpus.objects.filter(profile=profile, ssg=instance).delete() # Create a new one ssg_corpus = EqualGoldCorpus.objects.create(profile=profile, ssg=instance, status="busy") if lock_status != "ready": # Create an EqualGoldCorpus based on the SSGs in these manuscripts ssg_list = SermonDescrEqual.objects.filter( manu__id__in=manu_list).order_by( 'super_id').distinct().values('super_id') ssg_list_id = [x['super_id'] for x in ssg_list] all_words = {} with transaction.atomic(): for ssg in EqualGold.objects.filter(id__in=ssg_list_id): # Add this item to the ssg_corpus latin = {} if ssg.incipit != None: for item in ssg.srchincipit.replace( ",", "").replace("…", "").split(" "): add_to_dict(latin, item) add_to_dict(all_words, item) if ssg.explicit != None: for item in ssg.srchexplicit.replace( ",", "").replace("…", "").split(" "): add_to_dict(latin, item) add_to_dict(all_words, item) # Get the name of the author authorname = "empty" if ssg.author == None else ssg.author.name # Create new ssg_corpus item obj = EqualGoldCorpusItem.objects.create( corpus=ssg_corpus, words=json.dumps(latin), equal=ssg, authorname=authorname) # What are the 100 MFWs in all_words? mfw = [dict(word=k, count=v) for k, v in all_words.items()] mfw_sorted = sorted(mfw, key=lambda x: -1 * x['count']) mfw_cento = mfw_sorted[:100] mfw = [] for item in mfw_cento: mfw.append(item['word']) # Add this list to the ssg_corpus ssg_corpus.mfw = json.dumps(mfw) ssg_corpus.status = "ready" ssg_corpus.save() node_list, link_list, max_value = self.do_hier_method3( ssg_corpus, names_list) # Add the information to the context in data context['data'] = dict(node_list=node_list, link_list=link_list, watermark=get_watermark(), max_value=max_value, legend="SSG network") # Can remove the lock ssg_corpus.status = "ready" ssg_corpus.save() except: msg = oErr.get_error_message() oErr.DoError("EqualGoldPca/add_to_context") return context
def add_to_context(self, context): oErr = ErrHandle() spec_dict = {} link_dict = {} graph_template = 'seeker/super_graph_hist.html' try: # Define the linktype and spectype for obj in FieldChoice.objects.filter(field="seeker.spectype"): spec_dict[obj.abbr] = obj.english_name for obj in FieldChoice.objects.filter(field="seeker.linktype"): link_dict[obj.abbr] = obj.english_name # Need to figure out who I am profile = Profile.get_user_profile(self.request.user.username) instance = self.obj # The networkslider determines whether we are looking for 1st degree, 2nd degree or more networkslider = self.qd.get("network_overlap_slider", "1") if isinstance(networkslider, str): networkslider = int(networkslider) # Initializations ssg_link = {} bUseDirectionality = True # Read the table with links from SSG to SSG eqg_link = EqualGoldLink.objects.all().order_by( 'src', 'dst').values('src', 'dst', 'spectype', 'linktype', 'alternatives', 'note') # Transform into a dictionary with src as key and list-of-dst as value for item in eqg_link: src = item['src'] dst = item['dst'] if bUseDirectionality: spectype = item['spectype'] linktype = item['linktype'] note = item['note'] alternatives = item['alternatives'] if alternatives == None: alternatives = False # Make room for the SRC if needed if not src in ssg_link: ssg_link[src] = [] # Add this link to the list oLink = { 'dst': dst, 'note': note, 'spectype': "", 'linktype': linktype, 'spec': "", 'link': link_dict[linktype], 'alternatives': alternatives } if spectype != None: oLink['spectype'] = spectype oLink['spec'] = spec_dict[spectype] ssg_link[src].append(oLink) else: # Make room for the SRC if needed if not src in ssg_link: ssg_link[src] = [] # Add this link to the list ssg_link[src].append(dst) # Create the overlap network node_list, link_list, hist_set, max_value, max_group = self.do_overlap( ssg_link, networkslider) # Create the buttons for the historical collections hist_list = [{'id': k, 'name': v} for k, v in hist_set.items()] hist_list = sorted(hist_list, key=lambda x: x['name']) context = dict(hist_list=hist_list) hist_buttons = render_to_string(graph_template, context, self.request) # Add the information to the context in data context['data'] = dict(node_list=node_list, link_list=link_list, watermark=get_watermark(), hist_set=hist_set, hist_buttons=hist_buttons, max_value=max_value, max_group=max_group, networkslider=networkslider, legend="SSG overlap network") except: msg = oErr.get_error_message() oErr.DoError("EqualGoldOverlap/add_to_context") return context
def process_files(self, request, source, lResults, lHeader): file_list = [] oErr = ErrHandle() bOkay = True code = "" oStatus = self.oStatus try: # Make sure we have the username username = self.username profile = Profile.get_user_profile(username) team_group = app_editor kwargs = { 'profile': profile, 'username': username, 'team_group': team_group } # Get the contents of the imported file files = request.FILES.getlist('files_field') if files != None: for data_file in files: filename = data_file.name file_list.append(filename) # Set the status oStatus.set("reading", msg="file={}".format(filename)) # Get the source file if data_file == None or data_file == "": self.arErr.append( "No source file specified for the selected project" ) else: # Check the extension arFile = filename.split(".") extension = arFile[len(arFile) - 1] lst_manual = [] lst_read = [] # Further processing depends on the extension oResult = { 'status': 'ok', 'count': 0, 'sermons': 0, 'msg': "", 'user': username } if extension == "xlsx": # This is an Excel file: read the file using openpyxl # Write data temporarily to the WRITABLE dir, but with a temporary filename tmp_path = os.path.abspath( os.path.join(MEDIA_DIR, filename)) with io.open(tmp_path, "wb") as f: sData = data_file.read() f.write(sData) # Read string file wb = openpyxl.load_workbook(tmp_path, read_only=True) sheetnames = wb.sheetnames ws_manu = None ws_sermo = None for sname in sheetnames: if "manu" in sname.lower(): ws_manu = wb[sname] elif "sermo" in sname.lower(): ws_sermo = wb[sname] # Do we have a manuscript worksheet? if ws_manu != None: # Process the manuscript-proper details: columns Name and Value oManu = {} row_num = 1 if ws_manu.cell(row=row_num, column=1).value.lower() == "field" and \ ws_manu.cell(row=row_num, column=2).value.lower() == "value": # we can skip the first row row_num += 1 bStop = False while not bStop: k = ws_manu.cell(row=row_num, column=1).value v = ws_manu.cell(row=row_num, column=2).value if k == "" or k == None: bStop = True else: row_num += 1 k = k.lower() oManu[k] = v # We have an object with key/value pairs: process it manu = Manuscript.custom_add(oManu, **kwargs) # Now get the codicological unit that has been automatically created and adapt it codico = manu.manuscriptcodicounits.first() if codico != None: oManu['manuscript'] = manu codico = Codico.custom_add(oManu, **kwargs) oResult['count'] += 1 oResult['obj'] = manu oResult['name'] = manu.idno # Check if there is a "Sermon" worksheet if ws_sermo != None: # Get the column names row_num = 1 column = 1 header = [] v = ws_sermo.cell(row=row_num, column=column).value while v != None and v != "" and v != "-": header.append(v.lower()) column += 1 v = ws_sermo.cell(row=row_num, column=column).value # Process the sermons in this sheet sermon_list = [] column = 1 row_num += 1 v = ws_sermo.cell(row=row_num, column=column).value while v != "" and v != None: # ==== DEBUG ==== oErr.Status( "Upload excel row_num={}".format( row_num)) # =============== # Create a new sermon object oSermon = {} # Process this row for idx, col_name in enumerate(header): column = idx + 1 oSermon[col_name] = ws_sermo.cell( row=row_num, column=column).value # Process this sermon order = oSermon['order'] sermon = SermonDescr.custom_add( oSermon, manu, order) oResult['sermons'] += 1 # Get parent, firstchild, next parent = oSermon['parent'] firstchild = oSermon['firstchild'] nextone = oSermon['next'] # Add to list sermon_list.append({ 'order': order, 'parent': parent, 'firstchild': firstchild, 'next': nextone, 'sermon': sermon }) # GO to the next row for the next sermon row_num += 1 column = 1 v = ws_sermo.cell(row=row_num, column=column).value # Now process the parent/firstchild/next items with transaction.atomic(): for oSermo in sermon_list: # Get the p/f/n numbers parent_id = oSermo['parent'] firstchild_id = oSermo[ 'firstchild'] next_id = oSermo['next'] # Process parent if parent_id != '' and parent_id != None: # parent_id = str(parent_id) parent = next( (obj['sermon'] for obj in sermon_list if obj['order'] == parent_id), None) oSermo[ 'sermon'].msitem.parent = parent.msitem oSermo['sermon'].msitem.save() # Process firstchild if firstchild_id != '' and firstchild_id != None: # firstchild_id = str(firstchild_id) firstchild = next( (obj['sermon'] for obj in sermon_list if obj['order'] == firstchild_id), None) oSermo[ 'sermon'].msitem.firstchild = firstchild.msitem oSermo['sermon'].msitem.save() # Process next if next_id != '' and next_id != None: # next_id = str(next_id) nextone = next( (obj['sermon'] for obj in sermon_list if obj['order'] == next_id ), None) oSermo[ 'sermon'].msitem.next = nextone.msitem oSermo['sermon'].msitem.save() # Create a report and add it to what we return oContents = { 'headers': lHeader, 'list': lst_manual, 'read': lst_read } oReport = Report.make(username, "ixlsx", json.dumps(oContents)) # Determine a status code statuscode = "error" if oResult == None or oResult[ 'status'] == "error" else "completed" if oResult == None: self.arErr.append( "There was an error. No manuscripts have been added" ) else: lResults.append(oResult) code = "Imported using the [import_excel] function on this filew: {}".format( ", ".join(file_list)) except: bOkay = False code = oErr.get_error_message() return bOkay, code
def process_files(self, request, source, lResults, lHeader): file_list = [] oErr = ErrHandle() bOkay = True code = "" oStatus = self.oStatus # The list of headers to be shown lHeader = [ 'status', 'msg', 'name', 'daterange', 'library', 'idno', 'url' ] def add_manu(lst_manual, lst_read, status="", msg="", user="", name="", url="", daterange="", library="", filename="", sermons="", idno=""): oInfo = {} oInfo['status'] = status oInfo['msg'] = msg oInfo['user'] = user oInfo['name'] = name oInfo['url'] = url oInfo['daterange'] = daterange oInfo['library'] = library oInfo['idno'] = idno oInfo['filename'] = filename oInfo['sermons'] = sermons if status == "error": lst_manual.append(oInfo) else: lst_read.append(oInfo) return True try: # Make sure we have the username username = self.username profile = Profile.get_user_profile(username) team_group = app_editor kwargs = { 'profile': profile, 'username': username, 'team_group': team_group, 'source': source } # Get the contents of the imported file files = request.FILES.getlist('files_field') if files != None: for data_file in files: filename = data_file.name file_list.append(filename) # Set the status oStatus.set("reading", msg="file={}".format(filename)) # Get the source file if data_file == None or data_file == "": self.arErr.append( "No source file specified for the selected project" ) else: # Check the extension arFile = filename.split(".") extension = arFile[len(arFile) - 1] lst_manual = [] lst_read = [] # Further processing depends on the extension oResult = { 'status': 'ok', 'count': 0, 'sermons': 0, 'msg': "", 'user': username, 'filename': filename } if extension == "csv": # This is a CSV file. We expect the catalogue id's to be in the leftmost column # Write data temporarily to the WRITABLE dir, but with a temporary filename tmp_path = os.path.abspath( os.path.join(MEDIA_DIR, filename)) with io.open(tmp_path, "wb") as f: sData = data_file.read() f.write(sData) # Read the CSV file with a reader with open(tmp_path, "r", encoding="utf-8") as f: reader = csv.reader(f, delimiter=",", dialect='excel') # Read the header cells and make a header row in the worksheet headers = next(reader) row_num = 1 column = 1 lCsv = [] for row in reader: # Keep track of the EXCEL row we are in row_num += 1 # Get the ID cell_value = row[0] if cell_value != None and cell_value != "": # Get the catalogue id catalogue_id = int(cell_value) # Set the status oStatus.set( "reading", msg="catalogue id={}".format( catalogue_id)) # Clear the galway and codico objects oGalway = None oCodico = None # Read the manuscript object from the Galway site oGalway = self.get_galway(catalogue_id) # Extract Manuscript information and Codico information from [oGalway] oManu, oCodico = self.get_manucodico( oGalway) if oManu != None and oCodico != None: libname = "{}, {}, {}".format( oManu['country_name'], oManu['city_name'], oManu['library_name']) # Add manuscript (if not yet there) manu = Manuscript.custom_add( oManu, **kwargs) if manu.library == None: # Log that the library is not recognized oErr.Status( "Library not recognized: {}" .format(libname)) # Also add this in the notes notes = "" if manu.notes == None else manu.notes manu.notes = "Library not found: {} \n{}".format( libname, notes) # Make sure to add the source and the RAW data manu.source = source manu.raw = json.dumps(oGalway, indent=2) manu.save() # Now get the codicological unit that has been automatically created and adapt it codico = manu.manuscriptcodicounits.first( ) if codico != None: oCodico['manuscript'] = manu codico = Codico.custom_add( oCodico, **kwargs) # Process results add_manu(lst_manual, lst_read, status=oResult['status'], user=oResult['user'], name=codico.name, daterange=oCodico[ 'date ranges'], library=libname, filename=manu.idno, sermons=0, idno=manu.idno) oResult['count'] += 1 #oResult['obj'] = manu #oResult['name'] = manu.idno oResultManu = dict( name=manu.idno, filename=oManu['url'], sermons=0) lResults.append(oResultManu) # Create a report and add it to what we return oContents = { 'headers': lHeader, 'list': lst_manual, 'read': lst_read } oReport = Report.make(username, "xlsx", json.dumps(oContents)) # Determine a status code statuscode = "error" if oResult == None or oResult[ 'status'] == "error" else "completed" if oResult == None: self.arErr.append( "There was an error. No manuscripts have been added" ) else: lResults.append(oResult) # Make sure we have a success message available code = "Imported using the [import_galway] function on this file: {}".format( ", ".join(file_list)) # Indicate we are ready oStatus.set("ready") except: bOkay = False code = oErr.get_error_message() return bOkay, code
def process_files(self, request, source, lResults, lHeader): file_list = [] oErr = ErrHandle() bOkay = True code = "" oStatus = self.oStatus try: # Make sure we have the username username = self.username profile = Profile.get_user_profile(username) team_group = app_editor kwargs = { 'profile': profile, 'username': username, 'team_group': team_group, 'keyfield': 'path' } # Get the contents of the imported file files = request.FILES.getlist('files_field') if files != None: for data_file in files: filename = data_file.name file_list.append(filename) # Set the status oStatus.set("reading", msg="file={}".format(filename)) # Get the source file if data_file == None or data_file == "": self.arErr.append( "No source file specified for the selected project" ) else: # Check the extension arFile = filename.split(".") extension = arFile[len(arFile) - 1] lst_manual = [] lst_read = [] # Further processing depends on the extension oResult = { 'status': 'ok', 'count': 0, 'sermons': 0, 'msg': "", 'user': username } if extension == "json": # This is a JSON file: Load the file into a variable sData = data_file.read() lst_manu = json.loads( sData.decode(encoding="utf8")) # Walk through the manuscripts for oManu in lst_manu: # Each manuscript has some stuff of its own # We have an object with key/value pairs: process it manu = Manuscript.custom_add(oManu, **kwargs) # Now get the codicological unit that has been automatically created and adapt it codico = manu.manuscriptcodicounits.first() if codico != None: oManu['manuscript'] = manu codico = Codico.custom_add(oManu, **kwargs) oResult['count'] += 1 oResult['obj'] = manu oResult['name'] = manu.idno # Process all the MsItems into a list of sermons sermon_list = [] for oMsItem in oManu['msitems']: # Get the sermon object oSermon = oMsItem['sermon'] order = oMsItem['order'] sermon = SermonDescr.custom_add( oSermon, manu, order, **kwargs) # Keep track of the number of sermons read oResult['sermons'] += 1 # Get parent, firstchild, next parent = oMsItem['parent'] firstchild = oMsItem['firstchild'] nextone = oMsItem['next'] # Add to list sermon_list.append({ 'order': order, 'parent': parent, 'firstchild': firstchild, 'next': nextone, 'sermon': sermon }) # Now process the parent/firstchild/next items with transaction.atomic(): for oSermo in sermon_list: # Get the p/f/n numbers parent_id = oSermo['parent'] firstchild_id = oSermo['firstchild'] next_id = oSermo['next'] # Process parent if parent_id != '' and parent_id != None: # parent_id = str(parent_id) parent = next( (obj['sermon'] for obj in sermon_list if obj['order'] == parent_id), None) oSermo[ 'sermon'].msitem.parent = parent.msitem oSermo['sermon'].msitem.save() # Process firstchild if firstchild_id != '' and firstchild_id != None: # firstchild_id = str(firstchild_id) firstchild = next( (obj['sermon'] for obj in sermon_list if obj['order'] == firstchild_id), None) oSermo[ 'sermon'].msitem.firstchild = firstchild.msitem oSermo['sermon'].msitem.save() # Process next if next_id != '' and next_id != None: # next_id = str(next_id) nextone = next( (obj['sermon'] for obj in sermon_list if obj['order'] == next_id), None) oSermo[ 'sermon'].msitem.next = nextone.msitem oSermo['sermon'].msitem.save() # Create a report and add it to what we return oContents = { 'headers': lHeader, 'list': lst_manual, 'read': lst_read } oReport = Report.make(username, "ixlsx", json.dumps(oContents)) # Determine a status code statuscode = "error" if oResult == None or oResult[ 'status'] == "error" else "completed" if oResult == None: self.arErr.append( "There was an error. No manuscripts have been added" ) else: lResults.append(oResult) code = "Imported using the [import_json] function on this file list: {}".format( ", ".join(file_list)) except: bOkay = False code = oErr.get_error_message() return bOkay, code
def __init__(self, *args, **kwargs): # Obligatory for this type of form!!! self.username = kwargs.pop('username', "") self.team_group = kwargs.pop('team_group', "") self.userplus = kwargs.pop('userplus', "") # Start by executing the standard handling super(ResearchSetForm, self).__init__(*args, **kwargs) oErr = ErrHandle() try: username = self.username profile = Profile.get_user_profile(username) team_group = self.team_group # Some fields are not required self.fields['name'].required = False self.fields['notes'].required = False self.fields['scope'].required = False # Make sure the profile is set correctly self.fields['profileid'].initial = profile.id # Set the widgets correctly self.fields['histlist'].widget = CollOneHistWidget( attrs={ 'username': username, 'team_group': team_group, 'settype': 'hc', 'data-placeholder': 'Select a historical collection...', 'style': 'width: 100%;', 'class': 'searching' }) self.fields['ssgdlist'].widget = CollOneSuperWidget( attrs={ 'username': username, 'team_group': team_group, 'settype': 'pd', 'data-placeholder': 'Select a personal dataset of SSGs...', 'style': 'width: 100%;', 'class': 'searching' }) self.fields['ownlist'].queryset = Profile.objects.all() # Set queryset(s) - for details view self.fields['manulist'].queryset = Manuscript.objects.none() # Note: the collection filters must use the SCOPE of the collection self.fields['histlist'].queryset = Collection.get_scoped_queryset( 'super', username, team_group, settype="hc") self.fields['ssgdlist'].queryset = Collection.get_scoped_queryset( 'super', username, team_group, settype="pd") # Get the instance if 'instance' in kwargs: instance = kwargs['instance'] # Adapt the profile if this is needed self.fields['profileid'].initial = instance.profile.id # Give a choice of manuscripts that are not linked to this researchset manu_ids = [ x['manuscript__id'] for x in SetList.objects.filter( setlisttype='manu', researchset=instance, manuscript__isnull=False).values('manuscript__id') ] qs = Manuscript.objects.filter(mtype='man').exclude( id__in=manu_ids).order_by('library__lcity__name', 'library__name', 'idno') self.fields['manulist'].queryset = qs self.fields['manulist'].widget.queryset = qs except: msg = oErr.get_error_message() oErr.DoError("ResearchSetForm") return None