def test_physical_format_from_format_and_type():
"""
Test physical format appending from format and type fields
"""
INPUT = {
"format": [
"76.8 x 104 cm", "Oil on canvas",
"7 1/4 x 6 inches (18.4 x 15.2 cm)",
"Sheet: 9 1/2 x 12 1/8 inches (24.1 x 30.8 cm)"
],
"type": ["Paintings", "Painting"]
}
EXPECTED = {
"format": [
"76.8 x 104 cm", "Oil on canvas",
"7 1/4 x 6 inches (18.4 x 15.2 cm)",
"Sheet: 9 1/2 x 12 1/8 inches (24.1 x 30.8 cm)", "Paintings",
"Painting"
]
}
resp, content = H.request(server() +
"enrich-type?prop=type&format_field=format",
"POST",
body=json.dumps(INPUT))
assert str(resp.status).startswith("2")
FETCHED = json.loads(content)
assert FETCHED == EXPECTED, DictDiffer(EXPECTED, FETCHED).diff()
resp, content = H.request(server() +
"enrich-format?prop=format&type_field=type",
"POST",
body=content)
assert str(resp.status).startswith("2")
FETCHED = json.loads(content)
assert FETCHED == EXPECTED, DictDiffer(EXPECTED, FETCHED).diff()
def test_artstor_cleanup_creator2():
"""
Cleanup the creator field
"""
INPUT = {
"sourceResource": {
"creator": [
" and bananas", " Artist: bananas", "Author: bananas",
"Binder: bananas", "Drawn by bananas", "drawn by bananas",
" illuminator: bananas", "Or bananas ", "Scribe: bananas",
"Resolve bananas", " Apples"
]
}
}
EXPECTED = {
"sourceResource": {
"creator": [
"bananas", "bananas", "bananas", "bananas", "bananas",
"bananas", "bananas", "bananas", "bananas", "bananas", "Apples"
]
}
}
url = server() + "artstor_cleanup_creator"
resp, content = H.request(url, "POST", body=json.dumps(INPUT))
assert str(resp.status).startswith("2")
data = json.loads(content)
assert data == EXPECTED, DictDiffer(data, EXPECTED).diff()
def test_artstor_cleanup_creator3():
"""
Should do nothing since creator field does not exist
"""
INPUT = {"sourceResource": {"subject": "bananas"}}
url = server() + "artstor_cleanup_creator"
resp, content = H.request(url, "POST", body=json.dumps(INPUT))
assert str(resp.status).startswith("2")
data = json.loads(content)
assert data == INPUT, DictDiffer(data, INPUT).diff()
def test_artstor_cleanup_creator1():
"""
Cleanup the creator field
"""
INPUT = {"sourceResource": {"creator": "And bananas"}}
EXPECTED = {"sourceResource": {"creator": "bananas"}}
url = server() + "artstor_cleanup_creator"
resp, content = H.request(url, "POST", body=json.dumps(INPUT))
assert str(resp.status).startswith("2")
data = json.loads(content)
assert data == EXPECTED, DictDiffer(data, EXPECTED).diff()
def test_artstor_cleanup_data_provider():
"""
Remove "Repository:" from Artstor data provider
"""
INPUT = {
"dataProvider": "Repository: Columbia Museum of Art, Columbia, SC"
}
EXPECTED = {"dataProvider": "Columbia Museum of Art, Columbia, SC"}
url = server() + "artstor_cleanup"
resp, content = H.request(url, "POST", body=json.dumps(INPUT))
assert str(resp.status).startswith("2")
data = json.loads(content)
assert data["dataProvider"] == EXPECTED["dataProvider"], DictDiffer(
data, EXPECTED).diff()
def test_shred12():
"""Should shred"""
INPUT = {
"m":
"McBeth, Alexander<br> Greenville (S.C.)<br>South Carolina<br>Account books<br> General stores",
"g": "bananas"
}
EXPECTED = {
"m": [
"McBeth, Alexander", "Greenville (S.C.)", "South Carolina",
"Account books", "General stores"
],
"g":
"bananas"
}
url = server() + "shred?prop=m,g&delim=%3Cbr%3E"
resp, content = H.request(url, "POST", body=json.dumps(INPUT))
assert str(resp.status).startswith("2"), content
FETCHED = json.loads(content)
assert FETCHED == EXPECTED, DictDiffer(EXPECTED, FETCHED).diff()
def test_shred13():
"""Should not shred when multi-char delim within parens"""
INPUT = {
"m":
"McBeth, Alexander<br> Greenville (S.C.<br>S.C.)<br>South Carolina<br>(a(b)) c<br>(a(b<br><br>c(<br>d))efg<br>)(<br>h<br>i<br>) jklmn<br>op",
"g": "bananas"
}
EXPECTED = {
"m": [
"McBeth, Alexander", "Greenville (S.C.<br>S.C.)", "South Carolina",
"(a(b)) c", "(a(b<br><br>c(<br>d))efg<br>)(<br>h<br>i<br>) jklmn",
"op"
],
"g":
"bananas"
}
url = server() + "shred?prop=m,g&delim=%3Cbr%3E"
resp, content = H.request(url, "POST", body=json.dumps(INPUT))
assert str(resp.status).startswith("2"), content
FETCHED = json.loads(content)
assert FETCHED == EXPECTED, DictDiffer(EXPECTED, FETCHED).diff()
def test_shred10():
"""Shred with special chars as delimiter"""
INPUT = {
"m":
"String one\\ Begin of two (String two\\ two and a part of two) String three\\ String four\\ (abc dbf\\ sss\\k)",
"g": "bananas"
}
EXPECTED = {
"m": [
'String one',
'Begin of two (String two\\ two and a part of two) String three',
'String four', '(abc dbf\\ sss\\k)'
],
"g":
"bananas"
}
url = server() + "shred?prop=m,g&delim=%5C"
resp, content = H.request(url, "POST", body=json.dumps(INPUT))
assert str(resp.status).startswith("2"), content
FETCHED = json.loads(content)
assert FETCHED == EXPECTED, DictDiffer(EXPECTED, FETCHED).diff()
def test_shred15():
INPUT = {
"m": [
"Crivelli, Vittorio", "Alberti, Antonio (Antonio da Ferrara)",
"Pietro da Rimini", "Marche school (14th century)",
"Luca di Tomme",
"Gentile da Fabriano (Gentile di Niccolo di Giovanni)",
"Antoniazzo Romano (Antonio di Benedetto Aquilo)", "Rimini School",
"Viviani, Antonio Maria (Il Sordo d'Urbino)", "Ridolfi, Claudio",
"Giusto di Gand (Josse van Wassenhove)", "Berruguete, Pedro",
"Florence school (15th century)",
"Rosselli, Domenico di Giovanni di Bartolomeo",
"School of Umbria, 15th century)", "Foligno School (15th century)",
"Brandani, Federigo"
]
}
url = server() + "shred?prop=m"
resp, content = H.request(url, "POST", body=json.dumps(INPUT))
assert str(resp.status).startswith("2"), content
FETCHED = json.loads(content)
assert FETCHED == INPUT, DictDiffer(INPUT, FETCHED).diff()
def test_shred14():
"""Should shred and disregard mismatched parens since count of open or
closed parens is 0
"""
INPUT = {
"m": "H: 35 in.A) 11 1/2 x 13 x 13 in.; " + \
"B) 4 1/4 x 12 1/2 x 3 1/4 in.; C) 10 x 4 x 4 in.; " + \
"glass; Furnishings;"
}
EXPECTED = {
"m": [
"H: 35 in.A) 11 1/2 x 13 x 13 in.",
"B) 4 1/4 x 12 1/2 x 3 1/4 in.", "C) 10 x 4 x 4 in.", "glass",
"Furnishings"
]
}
url = server() + "shred?prop=m"
resp, content = H.request(url, "POST", body=json.dumps(INPUT))
assert str(resp.status).startswith("2"), content
FETCHED = json.loads(content)
assert FETCHED == EXPECTED, DictDiffer(EXPECTED, FETCHED).diff()
def test_capitalize_value_exclude():
"""Should capitalize first letter of each property"""
INPUT = {
"id": "123",
"sourceResource": {
"format": ["format1", "format2"],
"subject": ["subject", "hi there", "hello"]
}
}
EXPECTED = {
"id": "123",
"sourceResource": {
"format": ["Format1", "Format2"],
"subject": ["subject", "hi there", "hello"]
}
}
resp, content = H.request(url + "?exclude=sourceResource/subject", "POST",
json.dumps(INPUT))
assert resp.status == 200
FETCHED = json.loads(content)
assert FETCHED == EXPECTED, DictDiffer(EXPECTED, FETCHED).diff()
def test_shred11():
"""Shred in two steps"""
INPUT = {
"a": [
"Sheet: 9 1/2 x 12 1/8 inches (24.1 x 30.8 cm)",
"Gray, green,and brown washes with black chalk over graphite on medium, slightly textured, brown wove paper"
]
}
EXPECTED = {
"a": [
"Sheet", "9 1/2 x 12 1/8 inches (24.1 x 30.8 cm)",
"Gray, green,and brown washes with black chalk over graphite on medium, slightly textured, brown wove paper"
]
}
url = server() + "shred?prop=a"
resp, content = H.request(url, "POST", body=json.dumps(INPUT))
assert str(resp.status).startswith("2"), content
url = server() + "shred?prop=a&delim=%3A"
resp, content = H.request(url, "POST", body=content)
assert str(resp.status).startswith("2"), content
FETCHED = json.loads(content)
assert FETCHED == EXPECTED, DictDiffer(EXPECTED, FETCHED).diff()
def test_shred9():
"""Do not shred on values within parenthesis"""
INPUT = {
"p":
"String one; (String two; two and a part of two); String three; String four; (abc dbf; sss;k)",
"q": "d;e;f",
"h":
"String one; (String two; two and a part of two) String three; String four; (abc dbf; sss;k)",
"m":
"String one; Begin of two (String two; two and a part of two) String three; String four; (abc dbf; sss;k)",
"g": "bananas",
"a": "Sheet: 9 1/2 x 12 1/8 inches (24.1 x 30.8 cm)"
}
EXPECTED = {
"p": [
"String one", "(String two; two and a part of two)",
"String three", "String four", "(abc dbf; sss;k)"
],
"q": ["d", "e", "f"],
"h": [
'String one', '(String two; two and a part of two) String three',
'String four', '(abc dbf; sss;k)'
],
"m": [
'String one',
'Begin of two (String two; two and a part of two) String three',
'String four', '(abc dbf; sss;k)'
],
"a":
"Sheet: 9 1/2 x 12 1/8 inches (24.1 x 30.8 cm)",
"g":
"bananas"
}
url = server() + "shred?prop=p,q,h,m,g,a"
resp, content = H.request(url, "POST", body=json.dumps(INPUT))
assert str(resp.status).startswith("2")
FETCHED = json.loads(content)
assert FETCHED == EXPECTED, DictDiffer(EXPECTED, FETCHED).diff()
def test_capitalize_value():
"""Should capitalize first letter of each property"""
INPUT = {
"id": "123",
"spatial": {
"key1": "asheville",
"key2": "north Carolina"
},
"subject": ["subject", "hi there", "hello"]
}
EXPECTED = {
"id": "123",
"spatial": {
"key1": "Asheville",
"key2": "North Carolina"
},
"subject": ["Subject", "Hi there", "Hello"]
}
resp, content = _get_server_response(
json.dumps(INPUT), prop="spatial/key1,spatial/key2,subject")
assert resp.status == 200
FETCHED = json.loads(content)
assert FETCHED == EXPECTED, DictDiffer(EXPECTED, FETCHED).diff()
print("")
print("ERROR: missing configuration")
print("")
sys.exit(1)
print("\033[0m")
print("")
print("=======================")
print("= Deploy connectors =")
print("=======================")
print("")
ref_configs = fetchers.get_configs_from_repos()
actual_configs, inverted_actual_configs, crawler_ids = fetchers.get_configs_from_website()
differ = DictDiffer(ref_configs, actual_configs)
added = differ.added()
removed = differ.removed()
changed, changed_attributes = differ.changed()
if len(added) > 0:
print("")
print("Will be added :")
for config in added:
print(" - " + config)
if len(removed) > 0:
print("")
print("Will be delete :")
for config in removed:
