def test_modeling_ar100(self):
output = self.run_pipeline('ar100')
groups = output['model-groups']
germans = groups[
'tag:getty.edu,2019:digital:pipeline:REPLACE-WITH-UUID:shared#GROUP,AUTH,%5BGERMAN%20-%2014TH%20C.%5D']
gallery = groups[
'tag:getty.edu,2019:digital:pipeline:REPLACE-WITH-UUID:shared#PERSON,AUTH,Edinburgh%2C%20Scotland%2C%20UK.%20%20Scottish%20National%20Portrait%20Gallery']
# Assert that a Gallery has a "Corporate Name" and a "Display Title"
self.assertIn('Corporate Name', classified_identifier_sets(gallery))
self.assertIn('Display Title', classified_identifier_sets(gallery))
self.assertEqual(
classified_identifier_sets(gallery), {
'Corporate Name': {
'Edinburgh, Scotland, UK. Scottish National Portrait Gallery',
'National Portrait Gallery',
'Scottish National Portrait Gallery'
},
'Display Title': {'Scottish National Portrait Gallery'},
'Primary Name': {
'Edinburgh, Scotland, UK. Scottish National Portrait Gallery'
},
'STAR Identifier': {'61833'}
})
# and that a generic group representing people also uses "Corporate Name"
self.assertIn('Corporate Name', classified_identifier_sets(germans))
def test_modeling_ar101(self):
output = self.run_pipeline('ar101')
groups = output['model-groups']
people = output['model-person']
group = groups[
'tag:getty.edu,2019:digital:pipeline:REPLACE-WITH-UUID:shared#PERSON,AUTH,Boston%2C%20MA%2C%20USA.%20%20Isabella%20Stewart%20Gardner%20Museum']
self.assertEqual(
classified_identifier_sets(group), {
'Display Title': {'Isabella Stewart Gardner Museum'},
'Corporate Name': {
'Boston, MA, USA. Isabella Stewart Gardner Museum',
'Boston, MA. Isabella Stewart Gardner Museum',
'Isabella Stewart Gardner Museum'
},
'Primary Name':
{'Boston, MA, USA. Isabella Stewart Gardner Museum'},
'STAR Identifier': {'31556'}
})
person = people[
'tag:getty.edu,2019:digital:pipeline:REPLACE-WITH-UUID:shared#PERSON,AUTH,MONOGRAMMIST%20F.Y.']
self.assertEqual(
classified_identifier_sets(person), {
'Personal Name': {'F. Y.', 'Monogrammist F.Y.', 'F.Y.'},
'Primary Name': {'MONOGRAMMIST F.Y.'},
'STAR Identifier': {'13590'}
})
def test_modeling_ar46(self):
output = self.run_pipeline('ar46')
objects = output['model-object']
groups = output['model-groups']
activities = output['model-activity']
sale_act = output['model-sale-activity']
# lot sale date ('+' modified), event begin date, NO end date
event1 = sale_act['tag:getty.edu,2019:digital:pipeline:REPLACE-WITH-UUID:sales#PRIVATE_CONTRACT_SALE-EVENT,Br-279']
sale1 = sale_act['tag:getty.edu,2019:digital:pipeline:REPLACE-WITH-UUID:sales#AUCTION,Br-279,0103,1804-06-11']
self.assertIn('1804-06-11 onwards', classified_identifier_sets(sale1['timespan'])[None])
self.assertEqual(
classified_identifier_sets(sale1['timespan']),
classified_identifier_sets(event1['timespan']),
)
# lot sale date ('+' modified), event begin date, event end date
event2 = sale_act['tag:getty.edu,2019:digital:pipeline:REPLACE-WITH-UUID:sales#PRIVATE_CONTRACT_SALE-EVENT,Br-64']
sale2 = sale_act['tag:getty.edu,2019:digital:pipeline:REPLACE-WITH-UUID:sales#AUCTION,Br-64,0001,1801-11-16']
self.assertIn('1801-11-16 to 1803-01-01', classified_identifier_sets(sale2['timespan'])[None])
self.assertEqual(
classified_identifier_sets(sale2['timespan']),
classified_identifier_sets(event2['timespan']),
)
# lot sale date ('+' modified, zero-day), event begin date (zero-day), NO end date
# in this case, the lot sale date and the event begin date are the same (both with a zero-valued day field).
# as a result, the event keeps the verbatim date string as the Name (with the trailing "-00"), but
# that wasn't asserted on the lot sale, so we use the shortened form to indicate the full month ("1808-07").
event3 = sale_act['tag:getty.edu,2019:digital:pipeline:REPLACE-WITH-UUID:sales#PRIVATE_CONTRACT_SALE-EVENT,Br-608-A']
sale3 = sale_act['tag:getty.edu,2019:digital:pipeline:REPLACE-WITH-UUID:sales#AUCTION,Br-608-A,0449,1808-07']
self.assertIn('1808-07-00', classified_identifier_sets(event3['timespan'])[None])
self.assertIn('1808-07', classified_identifier_sets(sale3['timespan'])[None])
def test_modeling_ar54(self):
output = self.run_pipeline('ar54')
people = output['model-person']
person = people[
'tag:getty.edu,2019:digital:pipeline:REPLACE-WITH-UUID:shared#PERSON,AUTH,Parsons%2C%20Charles%20H.']
self.assertEqual(
classified_identifier_sets(person), {
'STAR Identifier': {'46035'},
'Personal Name': {
'Ch.s H. Parsons', 'Ch.s Parsons', 'Charles H. Parsons',
'Chs Parsons', 'Chs. Parsons', 'Parsons, Chs.',
'c. parsons', 'ch. parsons'
},
'Primary Name': {'Parsons, Charles H.'},
})
star_id = [
i for i in person['identified_by'] if i['content'] == '46035'
][0]
self.assertIn('assigned_by', star_id)
assignments = star_id['assigned_by']
self.assertEqual(len(assignments), 1)
assignors = assignments[0]['carried_out_by']
self.assertEqual(len(assignors), 1)
gpi = assignors[0]
self.assertEqual(gpi['_label'], 'Getty Provenance Index')
def verifyReturnAcquisition(self, tx):
# There was a bug that was causing "Returned" transactions to go through the ETL modeling process twice, resulting in multiple Acquisition identifiers
# This sanity-checks that the return transaction looks right.
acqs = [p for p in tx['part'] if p.get('type') == 'Acquisition']
self.assertEqual(len(acqs), 1)
acq = acqs[0]
self.assertEqual(
classified_identifier_sets(acq),
{None: {'Knoedler return of Stock Number A8960 (1966-02-01)'}})
pass
def test_modeling_ar68(self):
output = self.run_pipeline('ar68')
groups = output['model-groups']
# This is a corporate_body Group, and so should not have any professional activities carried out.
# However, it *should* have a sojourn carried out (and since this is a corporate body, the sojourn is an Establishment activity)
group1 = groups[
'tag:getty.edu,2019:digital:pipeline:REPLACE-WITH-UUID:shared#PERSON,AUTH,Nebraska%20Art%20Association']
activities = group1['carried_out']
self.assertEqual(len(activities), 1)
cls = activities[0]['classified_as']
self.assertEqual(len(cls), 1)
cl = cls[0]
self.assertEqual(cl['_label'], 'Establishment')
# This is a generic group that should have professional activity modeled based on the century active
group2 = groups[
'tag:getty.edu,2019:digital:pipeline:REPLACE-WITH-UUID:shared#GROUP,AUTH,%5BFRENCH%20-%2015TH%20C.%5D']
self.assertIn('carried_out', group2)
activities = group2['carried_out']
self.assertEqual(len(activities), 1)
activity = activities[0]
self.assertEqual(
activity['_label'],
'Professional activity of French persons in the 15th century')
self.assertEqual(classified_identifier_sets(activity['timespan']), {})
# This is a corporate_body Group that has a century active, so should have a professional activity based on that date data
group3 = groups[
'tag:getty.edu,2019:digital:pipeline:REPLACE-WITH-UUID:shared#PERSON,AUTH,Macon%2C%20GA%2C%20USA.%20%20Wesleyan%20College']
self.assertIn('carried_out', group3)
activities = [
a for a in group3['carried_out']
if a['_label'].startswith('Professional activity')
]
self.assertEqual(len(activities), 1)
activity = activities[0]
self.assertEqual(
activity['_label'],
'Professional activity of Macon, GA, USA. Wesleyan College')
# the timespan has a Name which holds the verbatim century_active value
self.assertEqual(classified_identifier_sets(activity['timespan']),
{None: {'19th-20th'}})
def test_modeling_ar37(self):
output = self.run_pipeline('ar37')
activities = output['model-activity']
# Purchase
purchase = activities[
'tag:getty.edu,2019:digital:pipeline:REPLACE-WITH-UUID:knoedler#TX,In,3,110,38']
self.assertDictContainsSubset(
{None: {'Knoedler Purchase of Stock Number 2391 (1880)'}},
classified_identifier_sets(purchase))
self.assertNotIn('K-1966', purchase['_label'])
# Sale
sale = activities[
'tag:getty.edu,2019:digital:pipeline:REPLACE-WITH-UUID:knoedler#TX,Out,3,110,38']
self.assertDictContainsSubset(
{None: {'Knoedler Sale of Stock Number 2391 (1880-05-14)'}},
classified_identifier_sets(sale))
self.assertNotIn('K-1966', sale['_label'])
# Inventorying
inv = activities[
'tag:getty.edu,2019:digital:pipeline:REPLACE-WITH-UUID:knoedler#TX,Out,4,86,15']
self.assertDictContainsSubset(
{
None:
{'Knoedler Inventorying of Stock Number 5323 (1887-04-30)'}
}, classified_identifier_sets(inv))
self.assertNotIn('K-7988', inv['_label'])
# Record without a stock number
missing_sn_act = activities[
'tag:getty.edu,2019:digital:pipeline:REPLACE-WITH-UUID:knoedler#TX,Out,1,75,7']
self.assertDictContainsSubset(
{None: {'Knoedler Sale of [GRI Number K-56] (1883-05-28)'}},
classified_identifier_sets(missing_sn_act))
self.assertDictContainsSubset(
{
'Note': {
'No Knoedler stock number was assigned to the object that is the subject of this provenance activity.'
}
}, classified_identifier_sets(missing_sn_act, 'referred_to_by'))
def test_modeling_ar74(self):
'''
AR-74: Change titling strategy for auction event records for "Auction Event X"
'''
output = self.run_pipeline('ar74')
activities = output['model-sale-activity']
auction = activities[
'tag:getty.edu,2019:digital:pipeline:REPLACE-WITH-UUID:sales#AUCTION-EVENT,B-A136']
self.assertEqual(auction['_label'],
'Auction Event B-A136 (1773-07-20 onwards)')
self.assertEqual(classified_identifier_sets(auction),
{None: {'Auction Event B-A136 (1773-07-20 onwards)'}})
def test_modeling_ar116(self):
'''
AR-116: Add type to linguistic statement to disambiguate if the statement is about buyer or seller
'''
output = self.run_pipeline('ar116')
activities = output['model-activity']
sale = activities['tag:getty.edu,2019:digital:pipeline:REPLACE-WITH-UUID:sales#PROV,B-183,1810-10-26,0050']
acqs = [p for p in sale.get('part', []) if p['type'] == 'Acquisition']
self.assertEqual(len(acqs), 1)
acq = acqs[0]
self.assertEqual(classified_identifier_sets(acq, key='referred_to_by'), {
'Note': {'and', 'or'},
'Qualifier': {'and', 'or'},
'Buyer description': {'and'},
'Seller description': {'or'}})
def test_modeling_ar83(self):
output = self.run_pipeline('ar83')
texts = output['model-lo']
row = texts[
'tag:getty.edu,2019:digital:pipeline:REPLACE-WITH-UUID:knoedler#Text,Book,4,Page,29,Row,14']
self.assertEqual(
classified_identifier_sets(row), {
'Entry Number': {'14'},
'STAR Identifier': {'69'},
'Title': {'Knoedler Stock Book 4, Page 29, Row 14'}
})
star_ids = [i for i in row['identified_by'] if i['content'] == '69']
self.assertEqual(len(star_ids), 1)
star_id = star_ids[0]
self.verifyStarNumber(star_id)
def verifyPrivateField(self, person):
self.assertIn('referred_to_by', person)
refs = [
r for r in person['referred_to_by']
if 'was born in Geneva' in r.get('content', '')
]
self.assertEqual(len(refs), 1)
ref = refs[0]
# expect the PEOPLE 'text' field to be marked as both private and a Biography Statement
expected = "was born in Geneva, briefly active in Paris, and resident in London from 1800 onwards. He put up two of his own paintings for sale in 1801 but both were bought in. He lived in Goldsmith Street, according to Christie's files, and his collection was sold by Christie's in 1850 after his death. He was primarily a painter of animals and landscape."
self.assertEqual(
classified_identifier_sets(person, 'referred_to_by'), {
'Bibliography Statement': {'TB; Thieme-Becker'},
'Source Statement': {'Sales Index 1751-1800; Sales Index 1-5'},
'Biography Statement': {expected},
'private (general concept)': {expected}
})
def test_modeling_ar101(self):
'''
AR-101: Replace Repository Number Type with Star Identifier Type
'''
output = self.run_pipeline('ar101')
lo = output['model-lo']
people = output['model-person']
catalog = lo[
'tag:getty.edu,2019:digital:pipeline:REPLACE-WITH-UUID:sales#CATALOG,SC-A40']
person = people[
'tag:getty.edu,2019:digital:pipeline:REPLACE-WITH-UUID:shared#PERSON,AUTH,MONOGRAMMIST%20F.Y.']
self.assertEqual(
classified_identifier_sets(catalog), {
None: {'Sale Catalog SC-A40'},
'Owner-Assigned Number': {'SC-A40'},
'STAR Identifier': {'13214', 'SCANDICATS-57'}
})
def test_modeling_ar83(self):
'''
AR-83: Fix textual work identifier types
'''
output = self.run_pipeline('ar83')
lo = output['model-lo']
catalog = lo['tag:getty.edu,2019:digital:pipeline:REPLACE-WITH-UUID:sales#CATALOG,SC-A40']
self.assertEqual(classified_identifier_sets(catalog), {
None: {'Sale Catalog SC-A40'},
'Owner-Assigned Number': {'SC-A40'},
'STAR Identifier': {'13214', 'SCANDICATS-57'}
})
for ident in ('13214', 'SCANDICATS-57'):
star_ids = [i for i in catalog['identified_by'] if i['content'] == ident]
self.assertEqual(len(star_ids), 1)
star_id = star_ids[0]
self.verifyStarNumber(star_id)
def test_modeling_ar101(self):
'''
AR-101: Replace Repository Number Type with Star Identifier Type
'''
output = self.run_pipeline('ar101')
lo = output['model-lo']
objects = output['model-object']
vi = output['model-visual-item']
row = lo[
'tag:getty.edu,2019:digital:pipeline:REPLACE-WITH-UUID:knoedler#Text,Book,5,Page,190,Row,38']
self.assertEqual(
classified_identifier_sets(row), {
'Entry Number': {'38'},
'STAR Identifier': {'53051'},
'Title': {'Knoedler Stock Book 5, Page 190, Row 38'}
})
obj = objects[
'tag:getty.edu,2019:digital:pipeline:REPLACE-WITH-UUID:knoedler#Object,5995']
def test_modeling_ar59(self):
output = self.run_pipeline('ar59')
groups = output['model-groups']
group = groups[
'tag:getty.edu,2019:digital:pipeline:REPLACE-WITH-UUID:shared#PERSON,AUTH,Philadelphia%2C%20PA%2C%20USA.%20%20Art%20Association%2C%20Union%20League%20Club']
self.assertEqual(
group["_label"],
'Philadelphia, PA, USA. Art Association, Union League Club')
self.assertEqual(
classified_identifier_sets(group), {
'Corporate Name': {
'Art Ass.n Union League Club',
'Art Association, Union League Club',
'Art Asson Union League Club', 'Art Club Philadelphia',
'Philadelphia, PA, USA. Art Association, '
'Union League Club', 'U.L. Art Assn.', 'Union League',
'Union League Phila', 'art assn phila'
},
'Display Title': {'Art Association, Union League Club'},
'Primary Name':
{'Philadelphia, PA, USA. Art Association, Union League Club'},
'STAR Identifier': {'46175'}
})
def verifyNoteFields(self, person):
# expect there to be four note fields coming from the input fields: text, notes, brief_notes, working_notes
# expect that the brief_notes field is marked public, while the other three are marked private
# expect that the working_notes field is classified as a research statement, while the other three are biography statements
expected = {
'Bibliography Statement': {'Thieme-Becker;'},
'Biography Statement': {
'Martinus Schouman was director of the Dordrecht drawing society Pictura (1800-15)',
'painter and auctioneer in Dordrecht',
'was in Breda after 1839'
},
'Research Statement':
{'There was also an A. Schouman in Dordrecht.'},
'Source Statement': {'Dutch Sales'},
'private (general concept)': {
'Martinus Schouman was director of the Dordrecht drawing society Pictura (1800-15)',
'There was also an A. Schouman in Dordrecht.',
'was in Breda after 1839'
},
'public (general concept)':
{'painter and auctioneer in Dordrecht'}
}
self.assertEqual(classified_identifier_sets(person, 'referred_to_by'),
expected)
def test_modeling_ar78(self):
'''
AR-78: Add translated titles for Scandinavian reords
'''
output = self.run_pipeline('ar78')
objects = output['model-object']
obj1 = objects['tag:getty.edu,2019:digital:pipeline:REPLACE-WITH-UUID:sales#OBJ,SC-A68,0149,1796-04-20']
self.assertEqual(classified_identifier_sets(obj1), {
'Lot Number': {'0149'},
'Primary Name': {'Jagtstykke med en Hiort'},
'Title': {
'2de Jagtstykker, det ene med en Hiort, det andet med en Löve [Malerier i Oliefarve]',
'Hunting piece with a deer',
'Jagtstykke med en Hiort'
},
'Translated Title': {'Hunting piece with a deer'}
})
idents = [i for i in obj1['identified_by'] if i['content'] == 'Hunting piece with a deer']
self.assertEqual(len(idents), 1)
i = idents[0]
self.assertIn('language', i)
self.assertEqual(i['language'][0]['_label'], 'English')
def test_modeling_ar110(self):
'''
AR-110: Carry over Title Modifier in visual items
'''
output = self.run_pipeline('ar110')
vi = output['model-visual-item']
obj1 = vi[
'tag:getty.edu,2019:digital:pipeline:REPLACE-WITH-UUID:sales#OBJ,B-A138,0023,1774-05-30-VisItem']
obj2 = vi[
'tag:getty.edu,2019:digital:pipeline:REPLACE-WITH-UUID:sales#OBJ,B-A139,0119%5Ba%5D,1774-05-31-VisItem']
obj3 = vi[
'tag:getty.edu,2019:digital:pipeline:REPLACE-WITH-UUID:sales#OBJ,B-A139,0194%5Ba%5D,1774-05-31-VisItem']
obj4 = vi[
'tag:getty.edu,2019:digital:pipeline:REPLACE-WITH-UUID:sales#OBJ,D-B1153,0320%5Ba%5D,1908-04-08-VisItem']
# obj1 has a title modifier that DOES NOT match the set of prefixes used for asserting an extra title
# instead, it is asserted as a 'Title Statement' referring to the object
obj1_ids = classified_identifier_sets(obj1)
obj1_refs = classified_identifier_sets(obj1, key='referred_to_by')
obj1_titles = obj1_ids['Title']
self.assertEqual(
obj1_titles, {
"Un très-beau Tableau, peint par David Teniers, d'après un Maître Italien, représentant Jesus-Christ…"
})
# obj2 has a title modifier with the prefix 'THIS LOT:'
obj2_ids = classified_identifier_sets(obj2)
obj2_titles = obj2_ids['Title']
self.assertDictContainsSubset(
{
'Primary Name': {
"Un Tableau, à Fruits &c.; lot 119[b] in the style of Vinckebooms"
}
}, obj2_ids)
self.assertEqual(
obj2_titles, {
"Un Tableau, à Fruits &c.; lot 119[b] in the style of Vinckebooms",
"Un Tableau, à Fruits &c., par Gillemans. Un Païsage dans le goût de Vinckebooms"
})
# obj3 has a title modifier with the prefix 'Sous ce no.:'
obj3_ids = classified_identifier_sets(obj3)
obj3_titles = obj3_ids['Title']
self.assertDictContainsSubset(
{
'Primary Name':
{"Un Paisage ; lot no.194[b] style of Boudewyns"}
}, obj3_ids)
self.assertEqual(
obj3_titles, {
"Un Paisage ; lot no.194[b] style of Boudewyns",
"Deux Païsages, l'un par Artois, & l'autre maniere de Baudewyns"
})
# obj4 has a title modifier with the prefix 'DIESES LOS:'
obj4_ids = classified_identifier_sets(obj4)
obj4_titles = obj4_ids['Title']
self.assertDictContainsSubset(
{'Primary Name': {"Frauenbildnis; H. 17, B.12BD cm"}}, obj4_ids)
self.assertEqual(
obj4_titles, {
"Frauenbildnis; H. 17, B.12BD cm",
"2 Bl. Frauenbildnisse. H. 17 und 18, B. je 12BD cm. Aquarelle."
})
def verifySaleLabel(self, sales, ident, expected_label):
sale = sales[
f'tag:getty.edu,2019:digital:pipeline:REPLACE-WITH-UUID:sales#AUCTION,{ident}']
idents = classified_identifier_sets(sale)
self.assertDictContainsSubset({None: {expected_label}}, idents)
def verifySaleEventLabel(self, sales, ident, expected_label):
sale = sales[
f'tag:getty.edu,2019:digital:pipeline:REPLACE-WITH-UUID:sales#AUCTION-EVENT,{ident}']
idents = classified_identifier_sets(sale)
self.assertEqual(idents, {None: {expected_label}})
