def add_person_albums(actors, person_album_ids, ns): for actor in actors: term = actor.find('lido:roleActor/lido:term', ns) if term is not None and term.text == 'kujutatu': muis_actor_id = int(actor.find("lido:actor/lido:actorID", ns).text) names = actor.findall( "lido:actor/lido:nameActorSet/lido:appellationValue", ns) all_names = '' main_name = '' for name in names: if name.attrib['{' + ns['lido'] + '}pref'] == 'preferred': main_name_parts = name.text.split(',') main_name = main_name_parts[-1].replace(' ', '') if len(main_name_parts) > 1: for part in main_name_parts[0:len(main_name_parts) - 1]: main_name += ' ' + part all_names += (name.attrib['{' + ns['lido'] + '}label'] + ': ' + name.text + '. ').capitalize() existing_album = Album.objects.filter(name=main_name, atype=Album.PERSON).first() if existing_album is None: person_album = Album(name=main_name, description=all_names, muis_person_ids=[muis_actor_id], atype=Album.PERSON) person_album.save() person_album_ids.append(person_album.id) else: person_album_ids.append(existing_album.id) return person_album_ids
def handle(self, *args, **options): with io.open(os.path.dirname(os.path.abspath(__file__)) + '/kreutzwaldi_sajand_people.csv', encoding='utf-8') as csv_file: csv_reader = csv.reader(csv_file) line_count = 0 for row in csv_reader: if line_count == 0: print(f'Column names are {", ".join(row)}') line_count += 1 else: if row[0]: # Previously confirmed already on Ajapaik print('Skipped %s' % row[1]) continue if row[3] or row[4]: # Skip partial names or non-names print('Skipped partial or non-name %s' % row[1]) continue row[1] = row[1].strip() existing_album = Album.objects.filter(name=row[1]).first() if existing_album: # Found existing person on Ajapaik, skip print('Already on Ajapaik: %s' % row[1]) continue new_person_album = Album(name=row[1], atype=Album.PERSON, open=True, gender=row[2], wikidata_qid=row[6]) new_person_album.save() print('Created %s' % new_person_album.name) line_count += 1 print(f'Processed {line_count} lines.')
def handle(self, *args, **options): translation.activate('fi') url = args[0] target_album = args[1] try: self.album = Album.objects.get(name=target_album) except ObjectDoesNotExist: new_album = Album(name=target_album, atype=Album.CURATED, is_public=True) new_album.save() self.album = new_album items_per_page = 20 page = 1 parser = etree.XMLParser() request = Request(url) response = urlopen(request) data = etree.fromstring(response.read(), parser=parser) # For testing #f = open(ABSOLUTE_PROJECT_ROOT + '/ajapaik/home/management/commands/finna_import_test_xml.xml', 'r') #data = etree.fromstring(f.read(), parser=parser) xml_response = data.find("response") number_of_items = int(xml_response.find("numFound").text) pages_to_get = int(ceil(number_of_items / items_per_page)) self._create_photos_from_xml_response(xml_response) if pages_to_get > 1: while page < pages_to_get: page += 1 url = self._set_query_parameter(url, 'page', page) request = Request(url) response = urlopen(request) data = etree.fromstring(response.read(), parser=parser) xml_response = data.find("response") self._create_photos_from_xml_response(xml_response)
def map_single_person(person_album: Album) -> None: print('Collecting rectangles for person %s' % person_album.pk) rectangles_with_this_person = FaceRecognitionRectangle.objects.filter(face_encoding__isnull=False).filter( Q(subject_consensus_id=person_album.pk) | Q(subject_ai_suggestion_id=person_album.pk)).distinct('pk').all() print('%s rectangles found' % rectangles_with_this_person.count()) encodings_for_this_person = [] for rectangle in rectangles_with_this_person: encodings_for_this_person.append(loads(rectangle.face_encoding)) try: if encodings_for_this_person: person_album.face_encodings = dumps(encodings_for_this_person) person_album.light_save() except: # noqa return
def test_albums_tartunlp_translation(): for each in [ 'Stereofotos no Parīzes (Francija)', 'Stereofotos iš Paryžiaus (Prancūzija)', 'Stereophotos from Paris (France)', 'Стереофоты из Парижа (Франция)', 'Stereofotot Pariisista (Ranska)' ]: responses.add(responses.GET, 'https://api.neurotolge.ee/v1.1/translate', json={'tgt': each}) test_instance = Album(name_et='Stereofotosid Pariisist (Prantsusmaa)', name_de='Stereophotos aus Paris (Frankreich)', atype=Album.CURATED) test_instance.fill_untranslated_fields() assert 'Pariisist' in getattr(test_instance, 'name_et') assert 'Parīzes' in getattr(test_instance, 'name_lv') assert 'Paryžiaus' in getattr(test_instance, 'name_lt') assert 'Paris' in getattr(test_instance, 'name_en') assert 'Парижа' in getattr(test_instance, 'name_ru') assert 'Frankreich' in getattr(test_instance, 'name_de') assert 'Pariisista' in getattr(test_instance, 'name_fi') assert 'et,de' == getattr(test_instance, 'name_original_language')
def finna_add_to_album(photo, target_album): if target_album and target_album != '': album = Album.objects.filter(name_en=target_album).first() if not album: album = Album(name_en=target_album, atype=Album.CURATED, is_public=True, cover_photo=photo) album.save() ap_found = AlbumPhoto.objects.filter(album=album, photo=photo).first() if not ap_found: ap = AlbumPhoto(album=album, photo=photo) ap.save() # update counts album.save()
def handle(self, *args, **options): self.query_url = "http://europeana.eu/api/v2/search.json" self.resource_url = "http://europeana.eu/api/v2/record" query = args[0] geoname = args[1] album_name = args[2] text_language = args[3] translation.activate('en') try: area = Area.objects.get(name=geoname) except ObjectDoesNotExist: new_area = Area(name=geoname) new_area.save() area = new_area try: album = Album.objects.get(name=album_name) except ObjectDoesNotExist: new_album = Album(name=album_name, atype=Album.COLLECTION, is_public=True) new_album.save() album = new_album translation.activate(text_language) query_result = self.query_europeana(query) item_count = int(query_result["itemsCount"]) ret = [] for i in xrange(0, item_count): if "dataProvider" in query_result["items"][ i] and "id" in query_result["items"][i]: if not self._resource_already_exists( query_result["items"][i]["dataProvider"][0], query_result["items"][i]["id"]): new_photo = Photo( area=area, source=Source.objects.get( description=query_result["items"][i] ["dataProvider"][0]), source_key=query_result["items"][i]["id"], licence="Public domain") if "edmIsShownAt" in query_result["items"][i]: new_photo.source_url = query_result["items"][i][ "edmIsShownAt"][0] if "edmAgentLabel" in query_result["items"][i]: new_photo.author = query_result["items"][i][ "edmAgentLabel"][0]["def"] if "title" in query_result["items"][i]: new_photo.description = query_result["items"][i][ "title"][0] opener = build_opener() opener.addheaders = [( "User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/537.36" )] try: img_response = opener.open( query_result["items"][i]["edmIsShownBy"][0]) new_photo.image.save("europeana.jpg", ContentFile(img_response.read())) new_photo.save() ap = AlbumPhoto(album=album, photo=new_photo) ap.save() except: pass
def handle(self, *args, **options): urls = [ "http://www.europeana.eu/portal/record/08535/local__default__F_7595.html?start=12&query=*%3A*&startPage=1&qf=ansicht&qf=where%3Avienna&qf=REUSABILITY%3Arestricted&qf=photograph&qt=false&rows=96&format=labels", "http://www.europeana.eu/portal/record/08535/local__default__F_7598.html?start=13&query=*%3A*&startPage=1&qf=ansicht&qf=where%3Avienna&qf=REUSABILITY%3Arestricted&qf=photograph&qt=false&rows=96&format=labels", "http://www.europeana.eu/portal/record/08535/local__default__F_7599.html?start=14&query=*%3A*&startPage=1&qf=ansicht&qf=where%3Avienna&qf=REUSABILITY%3Arestricted&qf=photograph&qt=false&rows=96&format=labels", "http://www.europeana.eu/portal/record/08535/local__default__F_7600.html?start=15&query=*%3A*&startPage=1&qf=ansicht&qf=where%3Avienna&qf=REUSABILITY%3Arestricted&qf=photograph&qt=false&rows=96&format=labels", "http://www.europeana.eu/portal/record/08535/local__default__F_3450.html?start=17&query=*%3A*&startPage=1&qf=ansicht&qf=where%3Avienna&qf=REUSABILITY%3Arestricted&qf=photograph&qt=false&rows=96&format=labels", "http://www.europeana.eu/portal/record/08535/local__default__F_3451.html?start=18&query=*%3A*&startPage=1&qf=ansicht&qf=where%3Avienna&qf=REUSABILITY%3Arestricted&qf=photograph&qt=false&rows=96&format=labels", "http://www.europeana.eu/portal/record/08535/local__default__F_3453.html?start=19&query=*%3A*&startPage=1&qf=ansicht&qf=where%3Avienna&qf=REUSABILITY%3Arestricted&qf=photograph&qt=false&rows=96&format=labels", "http://www.europeana.eu/portal/record/08535/local__default__F_3449.html?start=20&query=*%3A*&startPage=1&qf=ansicht&qf=where%3Avienna&qf=REUSABILITY%3Arestricted&qf=photograph&qt=false&rows=96&format=labels", "http://www.europeana.eu/portal/record/08535/local__default__F_3452.html?start=21&query=*%3A*&startPage=1&qf=ansicht&qf=where%3Avienna&qf=REUSABILITY%3Arestricted&qf=photograph&qt=false&rows=96&format=labels", ] area = Area.objects.filter(name='Vienna').first() if not area: new_area = Area(name='Vienna') new_area.lat = '48.2000' new_area.lon = '16.3667' new_area.save() area = new_area new_album = Album.objects.filter(name='Vienna, Austria').first() if not new_album: new_album = Album(name='Vienna, Austria', atype=Album.CURATED) new_album.save() new_album.save() translation.activate('de') for url in urls: result = requests.get(url) tree = html.fromstring(result.text) image_url = tree.xpath( '//td[@property="isShownBy http://www.europeana.eu/schemas/edm/isShownBy"]/text()' )[0] source_url = tree.xpath( '//td[@property="isShownAt http://www.europeana.eu/schemas/edm/isShownAt"]/text()' )[0] author = tree.xpath( '//span[@property="creator http://purl.org/dc/elements/1.1/creator"]/text()' )[0] source = Source.objects.filter( name= "Architekturmuseum der Technischen Universität Berlin in der Universitätsbibliothek" ).first() if not source: source = Source( name= "Architekturmuseum der Technischen Universität Berlin in der Universitätsbibliothek", description= "Architekturmuseum der Technischen Universität Berlin in der Universitätsbibliothek" ) source.save() description = tree.xpath( '//span[@property="subject http://purl.org/dc/elements/1.1/subject"]/text()' )[0] source_key = tree.xpath( '//span[@property="identifier http://purl.org/dc/elements/1.1/identifier"]/text()' )[0] if not description: description = tree.xpath( '//span[@property="title http://purl.org/dc/terms/title"]/text()' )[0] new_photo = Photo( description=description, source_key=source_key, source=source, source_url=source_url, area=area, author=author, licence=Licence.objects.get( url="http://creativecommons.org/licenses/by-nc-sa/4.0/"), ) new_photo.save() opener = build_opener() opener.addheaders = [( "User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/537.36" )] img_response = opener.open(image_url) new_photo.image.save("europeana.jpg", ContentFile(img_response.read())) new_photo.width = new_photo.image.width new_photo.height = new_photo.image.height AlbumPhoto(album=new_album, photo=new_photo).save()