def test_unify_document_title(self):
title = "Requirements Engineering for Scientific Computing: A Model-Based Approach"
unified, real = unify_document_title(title)
self.assertEqual(real, title)
self.assertEqual(unified, "requirementsengineeringforscientificcomputingamodelbasedapproach")
title = "Comparing state- and operation-based change tracking on models"
unified, real = unify_document_title(title)
self.assertEqual(real, title)
self.assertEqual(unified, "comparingstateandoperationbasedchangetrackingonmodels")
title = "Agile Factory - An Example of an Industry 4.0 Manufacturing Process"
unified, real = unify_document_title(title)
self.assertEqual(real, title)
self.assertEqual(unified, "agilefactoryanexampleofanindustry40manufacturingprocess")
title = "A Domain Specific Requirements Model for Scientific Computing (NIER Track)"
unified, real = unify_document_title(title)
self.assertEqual(real, title)
self.assertEqual(unified, "adomainspecificrequirementsmodelforscientificcomputingniertrack")
title = "SLPC++: Teaching software engineering project courses in industrial application landscapes - A tutorial"
unified, real = unify_document_title(title)
self.assertEqual(real, title)
self.assertEqual(unified, "slpcteachingsoftwareengineeringprojectcoursesinindustrialapplicationlandscapesatutorial")
def test_unify_document_title(self):
title = "Requirements Engineering for Scientific Computing: A Model-Based Approach"
unified, real = unify_document_title(title)
self.assertEqual(real, title)
self.assertEqual(unified, "requirementsengineeringforscientificcomputingamodelbasedapproach")
title = "Comparing state- and operation-based change tracking on models"
unified, real = unify_document_title(title)
self.assertEqual(real, title)
self.assertEqual(unified, "comparingstateandoperationbasedchangetrackingonmodels")
title = "Agile Factory - An Example of an Industry 4.0 Manufacturing Process"
unified, real = unify_document_title(title)
self.assertEqual(real, title)
self.assertEqual(unified, "agilefactoryanexampleofanindustry40manufacturingprocess")
title = "A Domain Specific Requirements Model for Scientific Computing (NIER Track)"
unified, real = unify_document_title(title)
self.assertEqual(real, title)
self.assertEqual(unified, "adomainspecificrequirementsmodelforscientificcomputingniertrack")
title = (
"SLPC++: Teaching software engineering project courses in industrial application landscapes - A tutorial"
)
unified, real = unify_document_title(title)
self.assertEqual(real, title)
self.assertEqual(
unified, "slpcteachingsoftwareengineeringprojectcoursesinindustrialapplicationlandscapesatutorial"
)
def process_profile_documents(self):
"""
Iterates over the profile documents, finds research fields, finds duplicates, finds author profiles
:return:
"""
for profile_unified in self._unified_name_to_profiles:
found_docs = []
profiles = self._unified_name_to_profiles[profile_unified]
if len(profiles) == 0:
log.warning("There were no profiles for the unified name %s" %
profile_unified)
continue
# For each profile linked to that unified name, add the found documents to the list
for profile in profiles:
x = self._profile_docs[profile.identifier]
log.debug(
"Used {len_x} documents from id {mendeley_id} for unified name {name}"
.format(len_x=len(x),
mendeley_id=profile.identifier,
name=unify_profile_name(profile.first_name,
profile.last_name)))
found_docs += x
# Process these documents
for doc in found_docs:
# Add doc to all docs
self._documents.append(doc)
# Create unified document title
doc_unified, doc_real = unify_document_title(doc.core_title)
# Add document to docs
if doc_unified in self._unified_document_title_to_documents:
existing_docs = self._unified_document_title_to_documents[
doc_unified]
existing_docs.append(doc)
else:
self._unified_document_title_to_documents[doc_unified] = [
doc
]
# Append the doc title to the authored_docs of that unified profile name
authored_docs = self._unified_name_to_authored_documents[
profile_unified]
authored_docs.add(doc_unified)
# Process core_authors field of the doc to find participants
for author in doc.core_authors:
self.analyze_author(doc_unified, author)
# Analyze the tags fields of the doc to find research fields
for tag in doc.tags:
self.analyze_field_tag(doc_unified, tag)
log.info("Profile documents have been analyzed")
def process_profile_documents(self):
"""
Iterates over the profile documents, finds research fields, finds duplicates, finds author profiles
:return:
"""
for profile_unified in self._unified_name_to_profiles:
found_docs = []
profiles = self._unified_name_to_profiles[profile_unified]
if len(profiles) == 0:
log.warning("There were no profiles for the unified name %s" % profile_unified)
continue
# For each profile linked to that unified name, add the found documents to the list
for profile in profiles:
x = self._profile_docs[profile.identifier]
log.debug("Used {len_x} documents from id {mendeley_id} for unified name {name}".format(
len_x=len(x),
mendeley_id=profile.identifier,
name=unify_profile_name(profile.first_name, profile.last_name)
))
found_docs += x
# Process these documents
for doc in found_docs:
# Add doc to all docs
self._documents.append(doc)
# Create unified document title
doc_unified, doc_real = unify_document_title(doc.core_title)
# Add document to docs
if doc_unified in self._unified_document_title_to_documents:
existing_docs = self._unified_document_title_to_documents[doc_unified]
existing_docs.append(doc)
else:
self._unified_document_title_to_documents[doc_unified] = [doc]
# Append the doc title to the authored_docs of that unified profile name
authored_docs = self._unified_name_to_authored_documents[profile_unified]
authored_docs.add(doc_unified)
# Process core_authors field of the doc to find participants
for author in doc.core_authors:
self.analyze_author(doc_unified, author)
# Analyze the tags fields of the doc to find research fields
for tag in doc.tags:
self.analyze_field_tag(doc_unified, tag)
log.info("Profile documents have been analyzed")
def process_group_documents(self):
"""
Iterates over the group documents, finds research fields, finds duplicates, finds author profiles
:return:
"""
for doc in self._group_docs:
# Add doc to all docs
self._documents.append(doc)
# Create unified document title
doc_unified, doc_real = unify_document_title(doc.core_title)
# Add document to docs
if doc_unified in self._unified_document_title_to_documents:
existing_docs = self._unified_document_title_to_documents[
doc_unified]
existing_docs.append(doc)
else:
self._unified_document_title_to_documents[doc_unified] = [doc]
# Try to find the main owner of the document through the document profile_id
# If not existent do nothing
# (we can't do much only with the profile_id.
# We could post-fetch the unknown profiles but that is more involved)
profile_id = doc.core_profile_id
if profile_id in self._profiles:
profile = self._profiles[profile_id]
unified_name, real_name = unify_profile_name(
profile.first_name, profile.last_name)
if unified_name in self._unified_name_to_authored_documents:
authored_documents = self._unified_name_to_authored_documents[
unified_name]
authored_documents.add(doc_unified)
# Process core_authors field of the doc to find participants
for author in doc.core_authors:
self.analyze_author(doc_unified, author)
# Analyze the tags fiels of the doc to find research fields
for tag in doc.tags:
self.analyze_field_tag(doc_unified, tag)
log.info("Group documents have been analyzed")
def process_group_documents(self):
"""
Iterates over the group documents, finds research fields, finds duplicates, finds author profiles
:return:
"""
for doc in self._group_docs:
# Add doc to all docs
self._documents.append(doc)
# Create unified document title
doc_unified, doc_real = unify_document_title(doc.core_title)
# Add document to docs
if doc_unified in self._unified_document_title_to_documents:
existing_docs = self._unified_document_title_to_documents[doc_unified]
existing_docs.append(doc)
else:
self._unified_document_title_to_documents[doc_unified] = [doc]
# Try to find the main owner of the document through the document profile_id
# If not existent do nothing
# (we can't do much only with the profile_id.
# We could post-fetch the unknown profiles but that is more involved)
profile_id = doc.core_profile_id
if profile_id in self._profiles:
profile = self._profiles[profile_id]
unified_name, real_name = unify_profile_name(profile.first_name, profile.last_name)
if unified_name in self._unified_name_to_authored_documents:
authored_documents = self._unified_name_to_authored_documents[unified_name]
authored_documents.add(doc_unified)
# Process core_authors field of the doc to find participants
for author in doc.core_authors:
self.analyze_author(doc_unified, author)
# Analyze the tags fiels of the doc to find research fields
for tag in doc.tags:
self.analyze_field_tag(doc_unified, tag)
log.info("Group documents have been analyzed")
