def add_person_albums(actors, person_album_ids, ns):
for actor in actors:
term = actor.find('lido:roleActor/lido:term', ns)
if term is not None and term.text == 'kujutatu':
muis_actor_id = int(actor.find("lido:actor/lido:actorID", ns).text)
names = actor.findall(
"lido:actor/lido:nameActorSet/lido:appellationValue", ns)
all_names = ''
main_name = ''
for name in names:
if name.attrib['{' + ns['lido'] + '}pref'] == 'preferred':
main_name_parts = name.text.split(',')
main_name = main_name_parts[-1].replace(' ', '')
if len(main_name_parts) > 1:
for part in main_name_parts[0:len(main_name_parts) -
1]:
main_name += ' ' + part
all_names += (name.attrib['{' + ns['lido'] + '}label'] + ': ' +
name.text + '. ').capitalize()
existing_album = Album.objects.filter(name=main_name,
atype=Album.PERSON).first()
if existing_album is None:
person_album = Album(name=main_name,
description=all_names,
muis_person_ids=[muis_actor_id],
atype=Album.PERSON)
person_album.save()
person_album_ids.append(person_album.id)
else:
person_album_ids.append(existing_album.id)
return person_album_ids
def handle(self, *args, **options):
with io.open(os.path.dirname(os.path.abspath(__file__)) +
'/kreutzwaldi_sajand_people.csv',
encoding='utf-8') as csv_file:
csv_reader = csv.reader(csv_file)
line_count = 0
for row in csv_reader:
if line_count == 0:
print(f'Column names are {", ".join(row)}')
line_count += 1
else:
if row[0]:
# Previously confirmed already on Ajapaik
print('Skipped %s' % row[1])
continue
if row[3] or row[4]:
# Skip partial names or non-names
print('Skipped partial or non-name %s' % row[1])
continue
row[1] = row[1].strip()
existing_album = Album.objects.filter(name=row[1]).first()
if existing_album:
# Found existing person on Ajapaik, skip
print('Already on Ajapaik: %s' % row[1])
continue
new_person_album = Album(name=row[1],
atype=Album.PERSON,
open=True,
gender=row[2],
wikidata_qid=row[6])
new_person_album.save()
print('Created %s' % new_person_album.name)
line_count += 1
print(f'Processed {line_count} lines.')
def handle(self, *args, **options):
translation.activate('fi')
url = args[0]
target_album = args[1]
try:
self.album = Album.objects.get(name=target_album)
except ObjectDoesNotExist:
new_album = Album(name=target_album,
atype=Album.CURATED,
is_public=True)
new_album.save()
self.album = new_album
items_per_page = 20
page = 1
parser = etree.XMLParser()
request = Request(url)
response = urlopen(request)
data = etree.fromstring(response.read(), parser=parser)
# For testing
#f = open(ABSOLUTE_PROJECT_ROOT + '/ajapaik/home/management/commands/finna_import_test_xml.xml', 'r')
#data = etree.fromstring(f.read(), parser=parser)
xml_response = data.find("response")
number_of_items = int(xml_response.find("numFound").text)
pages_to_get = int(ceil(number_of_items / items_per_page))
self._create_photos_from_xml_response(xml_response)
if pages_to_get > 1:
while page < pages_to_get:
page += 1
url = self._set_query_parameter(url, 'page', page)
request = Request(url)
response = urlopen(request)
data = etree.fromstring(response.read(), parser=parser)
xml_response = data.find("response")
self._create_photos_from_xml_response(xml_response)
def map_single_person(person_album: Album) -> None:
print('Collecting rectangles for person %s' % person_album.pk)
rectangles_with_this_person = FaceRecognitionRectangle.objects.filter(face_encoding__isnull=False).filter(
Q(subject_consensus_id=person_album.pk) | Q(subject_ai_suggestion_id=person_album.pk)).distinct('pk').all()
print('%s rectangles found' % rectangles_with_this_person.count())
encodings_for_this_person = []
for rectangle in rectangles_with_this_person:
encodings_for_this_person.append(loads(rectangle.face_encoding))
try:
if encodings_for_this_person:
person_album.face_encodings = dumps(encodings_for_this_person)
person_album.light_save()
except: # noqa
return
def test_albums_tartunlp_translation():
for each in [
'Stereofotos no Parīzes (Francija)',
'Stereofotos iš Paryžiaus (Prancūzija)',
'Stereophotos from Paris (France)',
'Стереофоты из Парижа (Франция)', 'Stereofotot Pariisista (Ranska)'
]:
responses.add(responses.GET,
'https://api.neurotolge.ee/v1.1/translate',
json={'tgt': each})
test_instance = Album(name_et='Stereofotosid Pariisist (Prantsusmaa)',
name_de='Stereophotos aus Paris (Frankreich)',
atype=Album.CURATED)
test_instance.fill_untranslated_fields()
assert 'Pariisist' in getattr(test_instance, 'name_et')
assert 'Parīzes' in getattr(test_instance, 'name_lv')
assert 'Paryžiaus' in getattr(test_instance, 'name_lt')
assert 'Paris' in getattr(test_instance, 'name_en')
assert 'Парижа' in getattr(test_instance, 'name_ru')
assert 'Frankreich' in getattr(test_instance, 'name_de')
assert 'Pariisista' in getattr(test_instance, 'name_fi')
assert 'et,de' == getattr(test_instance, 'name_original_language')
def finna_add_to_album(photo, target_album):
if target_album and target_album != '':
album = Album.objects.filter(name_en=target_album).first()
if not album:
album = Album(name_en=target_album,
atype=Album.CURATED,
is_public=True,
cover_photo=photo)
album.save()
ap_found = AlbumPhoto.objects.filter(album=album, photo=photo).first()
if not ap_found:
ap = AlbumPhoto(album=album, photo=photo)
ap.save()
# update counts
album.save()
def handle(self, *args, **options):
self.query_url = "http://europeana.eu/api/v2/search.json"
self.resource_url = "http://europeana.eu/api/v2/record"
query = args[0]
geoname = args[1]
album_name = args[2]
text_language = args[3]
translation.activate('en')
try:
area = Area.objects.get(name=geoname)
except ObjectDoesNotExist:
new_area = Area(name=geoname)
new_area.save()
area = new_area
try:
album = Album.objects.get(name=album_name)
except ObjectDoesNotExist:
new_album = Album(name=album_name,
atype=Album.COLLECTION,
is_public=True)
new_album.save()
album = new_album
translation.activate(text_language)
query_result = self.query_europeana(query)
item_count = int(query_result["itemsCount"])
ret = []
for i in xrange(0, item_count):
if "dataProvider" in query_result["items"][
i] and "id" in query_result["items"][i]:
if not self._resource_already_exists(
query_result["items"][i]["dataProvider"][0],
query_result["items"][i]["id"]):
new_photo = Photo(
area=area,
source=Source.objects.get(
description=query_result["items"][i]
["dataProvider"][0]),
source_key=query_result["items"][i]["id"],
licence="Public domain")
if "edmIsShownAt" in query_result["items"][i]:
new_photo.source_url = query_result["items"][i][
"edmIsShownAt"][0]
if "edmAgentLabel" in query_result["items"][i]:
new_photo.author = query_result["items"][i][
"edmAgentLabel"][0]["def"]
if "title" in query_result["items"][i]:
new_photo.description = query_result["items"][i][
"title"][0]
opener = build_opener()
opener.addheaders = [(
"User-Agent",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/537.36"
)]
try:
img_response = opener.open(
query_result["items"][i]["edmIsShownBy"][0])
new_photo.image.save("europeana.jpg",
ContentFile(img_response.read()))
new_photo.save()
ap = AlbumPhoto(album=album, photo=new_photo)
ap.save()
except:
pass
def handle(self, *args, **options):
urls = [
"http://www.europeana.eu/portal/record/08535/local__default__F_7595.html?start=12&query=*%3A*&startPage=1&qf=ansicht&qf=where%3Avienna&qf=REUSABILITY%3Arestricted&qf=photograph&qt=false&rows=96&format=labels",
"http://www.europeana.eu/portal/record/08535/local__default__F_7598.html?start=13&query=*%3A*&startPage=1&qf=ansicht&qf=where%3Avienna&qf=REUSABILITY%3Arestricted&qf=photograph&qt=false&rows=96&format=labels",
"http://www.europeana.eu/portal/record/08535/local__default__F_7599.html?start=14&query=*%3A*&startPage=1&qf=ansicht&qf=where%3Avienna&qf=REUSABILITY%3Arestricted&qf=photograph&qt=false&rows=96&format=labels",
"http://www.europeana.eu/portal/record/08535/local__default__F_7600.html?start=15&query=*%3A*&startPage=1&qf=ansicht&qf=where%3Avienna&qf=REUSABILITY%3Arestricted&qf=photograph&qt=false&rows=96&format=labels",
"http://www.europeana.eu/portal/record/08535/local__default__F_3450.html?start=17&query=*%3A*&startPage=1&qf=ansicht&qf=where%3Avienna&qf=REUSABILITY%3Arestricted&qf=photograph&qt=false&rows=96&format=labels",
"http://www.europeana.eu/portal/record/08535/local__default__F_3451.html?start=18&query=*%3A*&startPage=1&qf=ansicht&qf=where%3Avienna&qf=REUSABILITY%3Arestricted&qf=photograph&qt=false&rows=96&format=labels",
"http://www.europeana.eu/portal/record/08535/local__default__F_3453.html?start=19&query=*%3A*&startPage=1&qf=ansicht&qf=where%3Avienna&qf=REUSABILITY%3Arestricted&qf=photograph&qt=false&rows=96&format=labels",
"http://www.europeana.eu/portal/record/08535/local__default__F_3449.html?start=20&query=*%3A*&startPage=1&qf=ansicht&qf=where%3Avienna&qf=REUSABILITY%3Arestricted&qf=photograph&qt=false&rows=96&format=labels",
"http://www.europeana.eu/portal/record/08535/local__default__F_3452.html?start=21&query=*%3A*&startPage=1&qf=ansicht&qf=where%3Avienna&qf=REUSABILITY%3Arestricted&qf=photograph&qt=false&rows=96&format=labels",
]
area = Area.objects.filter(name='Vienna').first()
if not area:
new_area = Area(name='Vienna')
new_area.lat = '48.2000'
new_area.lon = '16.3667'
new_area.save()
area = new_area
new_album = Album.objects.filter(name='Vienna, Austria').first()
if not new_album:
new_album = Album(name='Vienna, Austria', atype=Album.CURATED)
new_album.save()
new_album.save()
translation.activate('de')
for url in urls:
result = requests.get(url)
tree = html.fromstring(result.text)
image_url = tree.xpath(
'//td[@property="isShownBy http://www.europeana.eu/schemas/edm/isShownBy"]/text()'
)[0]
source_url = tree.xpath(
'//td[@property="isShownAt http://www.europeana.eu/schemas/edm/isShownAt"]/text()'
)[0]
author = tree.xpath(
'//span[@property="creator http://purl.org/dc/elements/1.1/creator"]/text()'
)[0]
source = Source.objects.filter(
name=
"Architekturmuseum der Technischen Universität Berlin in der Universitätsbibliothek"
).first()
if not source:
source = Source(
name=
"Architekturmuseum der Technischen Universität Berlin in der Universitätsbibliothek",
description=
"Architekturmuseum der Technischen Universität Berlin in der Universitätsbibliothek"
)
source.save()
description = tree.xpath(
'//span[@property="subject http://purl.org/dc/elements/1.1/subject"]/text()'
)[0]
source_key = tree.xpath(
'//span[@property="identifier http://purl.org/dc/elements/1.1/identifier"]/text()'
)[0]
if not description:
description = tree.xpath(
'//span[@property="title http://purl.org/dc/terms/title"]/text()'
)[0]
new_photo = Photo(
description=description,
source_key=source_key,
source=source,
source_url=source_url,
area=area,
author=author,
licence=Licence.objects.get(
url="http://creativecommons.org/licenses/by-nc-sa/4.0/"),
)
new_photo.save()
opener = build_opener()
opener.addheaders = [(
"User-Agent",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/537.36"
)]
img_response = opener.open(image_url)
new_photo.image.save("europeana.jpg",
ContentFile(img_response.read()))
new_photo.width = new_photo.image.width
new_photo.height = new_photo.image.height
AlbumPhoto(album=new_album, photo=new_photo).save()