def test_photos_tartunlp_translation():
for each in [
'Igaunijas teātrimaja, skatā Igaunija puiesteelt',
'Estijos teatro namai, vaizdas Estija puiesteelt',
'Estonia Theatre House, view Estonia puiesteelt',
'Эстония театрический дом, вид Эстония пуiesteelt',
'Estlands Theaterhaus, Sicht auf Estland puiesteelt'
]:
responses.add(responses.POST,
'https://api.tartunlp.ai/translation/v2',
json={'result': each})
# Make sure _fi isn't overridden - et is picked as the source because it is at the start of TARTUNLP_LANGUAGES
test_instance = Photo(
description_et='Estonia teatrimaja, vaade Estonia puiesteelt',
description_fi='Viro teatrimaja, vaade Viro puiesteelt')
test_instance.fill_untranslated_fields()
assert 'teatrimaja' in getattr(test_instance, 'description_et')
assert 'Igaunijas' in getattr(test_instance, 'description_lv')
assert 'Estijos' in getattr(test_instance, 'description_lt')
assert 'Theatre' in getattr(test_instance, 'description_en')
assert 'театрический' in getattr(test_instance, 'description_ru')
assert 'Theaterhaus' in getattr(test_instance, 'description_de')
assert 'Viro' in getattr(test_instance, 'description_fi')
assert 'et,fi' == getattr(test_instance, 'description_original_language')
def handle(self, *args, **options):
album = Album.objects.filter(pk=22614).get()
with io.open(os.path.dirname(os.path.abspath(__file__)) +
'/kreutzwaldi_sajand.csv',
encoding='utf-8') as csv_file:
csv_reader = csv.reader(csv_file)
line_count = 0
for row in csv_reader:
if line_count == 0:
print(f'Column names are {", ".join(row)}')
line_count += 1
else:
response = requests.get(row[4], stream=True)
lf = tempfile.NamedTemporaryFile()
for block in response.iter_content(1024 * 8):
if not block:
break
lf.write(block)
new_picture = Photo(external_id=row[0],
source_id=152,
source_url=row[1],
source_key=row[3],
description=row[2])
new_picture.image.save(row[5], files.File(lf))
new_picture.save()
AlbumPhoto(photo=new_picture, album=album).save()
line_count += 1
print(f'Processed {line_count} lines.')
album.save()
def _create_photos_from_xml_response(self, xml_response):
for elem in xml_response:
if elem.tag == "docs":
if not self._resource_already_exists(elem):
new_photo = Photo(
title=elem.find("title").text,
description=elem.find("title_sort").text,
source=Source.objects.get(
description=elem.find('institution').text),
source_key=elem.find("identifier").text,
external_id=elem.find("identifier").text,
date_text=getattr(elem.find("main_date_str"), 'text',
None),
author=elem.find("author").text,
source_url=elem.find("record_link").text,
licence=Licence.objects.filter(
url='https://creativecommons.org/about/pdm').first(
))
opener = build_opener()
opener.addheaders = [(
"User-Agent",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/537.36"
)]
img_response = opener.open(elem.find("image_links").text)
new_photo.image.save("finna.jpg",
ContentFile(img_response.read()))
new_photo.save()
ap = AlbumPhoto(album=self.album, photo=new_photo)
ap.save()
def handle(self, *args, **options):
translation.activate('en')
tag = args[0]
page = args[1]
search_url = 'https://api.flickr.com/services/rest/?method=flickr.photos.search&api_key=' + settings.FLICKR_API_KEY + '&tags=' + tag + '&is_commons=1&content_type=6&extras=license,original_format&format=json&nojsoncallback=1&page=' + page
# https://farm{farm-id}.staticflickr.com/{server-id}/{id}_{o-secret}_o.(jpg|gif|png)
image_url_template = 'https://farm%s.staticflickr.com/%s/%s_%s_b.jpg'
# https://www.flickr.com/photos/{user-id}/{photo-id}
reference_url_template = 'https://www.flickr.com/photos/%s/%s'
request = Request(search_url)
response = urlopen(request)
data = response.read()
# Testing
# data = open(ABSOLUTE_PROJECT_ROOT + '/ajapaik/home/management/commands/flickr_import_test.json', 'r').read()
data = json.loads(data)
source = Source.objects.get(description='The British Library')
licence = Licence.objects.get(name='No known copyright restrictions')
album = Album.objects.get(
name='The British Library Metropolitan Improvements')
area = Area.objects.get(name='London')
for photo in data['photos']['photo']:
if photo['license'] == '7' and not self._resource_already_exists(
photo['id']):
new_photo = Photo(source=source,
source_url=reference_url_template %
(photo['owner'], photo['id']),
source_key=photo['id'],
date_text='1830',
licence=licence,
description=photo['title'],
area=area,
author='Shepherd, Thomas Hosmer')
try:
image_url = image_url_template % (
photo['farm'], photo['server'], photo['id'],
photo['secret'])
opener = build_opener()
opener.addheaders = [(
"User-Agent",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/537.36"
)]
img_response = opener.open(image_url)
new_photo.image.save("tbl.jpg",
ContentFile(img_response.read()))
new_photo.save()
ap = AlbumPhoto(album=album, photo=new_photo)
ap.save()
except:
# print "Problem loading image"
continue
def analyse_single_photo(photo: Photo) -> None:
print('Processing photo %s' % photo.pk)
if photo.width > 5000 or photo.height > 5000:
print('Skipping too big photo %s' % photo.pk)
return
try:
image = face_recognition.load_image_file(photo.image)
except: # noqa
return
try:
detected_faces = face_recognition.face_locations(image)
except: # noqa
return
for detected_face in detected_faces:
new_rectangle = FaceRecognitionRectangle(
photo=photo, coordinates=dumps(detected_face))
new_rectangle.save()
photo.face_detection_attempted_at = datetime.datetime.now()
photo.light_save()
def test_can_action_be_done():
source = settings.MEDIA_ROOT + '/uploads/test_image.png'
white = Image.new('RGBA', (32, 32), (255, 255, 255, 0))
white.save(source, "PNG")
photo = Photo(image=source, title='Title', description='Description')
photo.save()
photo = Photo.objects.get(id=photo.id)
photo.image.name = 'uploads/test_image.png'
photo.light_save()
user = User.objects.create_user('user', '*****@*****.**', 'user')
profile = Profile(user=user)
profile.save()
profile = Profile.objects.filter(pk=profile.id).first()
assert can_action_be_done(PhotoFlipSuggestion, photo, profile, 'flip',
True) is True
assert can_action_be_done(PhotoRotationSuggestion, photo, profile,
'rotated', 90) is True
_, flip_suggestions, _, _ = suggest_photo_edit([], 'flip', True, Points,
40, Points.FLIP_PHOTO,
PhotoFlipSuggestion, photo,
profile, '', 'do_flip')
PhotoFlipSuggestion.objects.bulk_create(flip_suggestions)
_, rotation_suggestions, _, _ = suggest_photo_edit(
[], 'rotated', 90, Points, 20, Points.ROTATE_PHOTO,
PhotoRotationSuggestion, photo, profile, '', 'do_rotate')
photo = Photo.objects.filter(id=photo.id).first()
PhotoRotationSuggestion.objects.bulk_create(rotation_suggestions)
assert can_action_be_done(PhotoFlipSuggestion, photo, profile, 'flip',
False) is True
assert can_action_be_done(PhotoRotationSuggestion, photo, profile,
'rotated', 180) is True
user_2 = User.objects.create_user('user2', '*****@*****.**', 'user2')
profile_2 = Profile(user=user_2)
profile_2.save()
profile_2 = Profile.objects.filter(pk=profile_2.id).first()
_, flip_suggestions, _, _ = suggest_photo_edit([], 'flip', True, Points,
40, Points.FLIP_PHOTO,
PhotoFlipSuggestion, photo,
profile_2, '', 'do_flip')
PhotoFlipSuggestion.objects.bulk_create(flip_suggestions)
_, rotation_suggestions, _, _ = suggest_photo_edit(
[], 'rotated', 90, Points, 20, Points.ROTATE_PHOTO,
PhotoRotationSuggestion, photo, profile_2, '', 'do_rotate')
PhotoRotationSuggestion.objects.bulk_create(rotation_suggestions)
user_3 = User.objects.create_user('user3', '*****@*****.**', 'user3')
profile_3 = Profile(user=user_3)
profile_3.save()
profile_3 = Profile.objects.filter(pk=profile_3.id).first()
assert can_action_be_done(PhotoFlipSuggestion, photo, profile_3, 'flip',
False) is False
assert can_action_be_done(PhotoRotationSuggestion, photo, profile_3,
'rotated', 180) is False
os.remove(source)
def handle(self, *args, **options):
self.query_url = "http://europeana.eu/api/v2/search.json"
self.resource_url = "http://europeana.eu/api/v2/record"
query = args[0]
geoname = args[1]
album_name = args[2]
text_language = args[3]
translation.activate('en')
try:
area = Area.objects.get(name=geoname)
except ObjectDoesNotExist:
new_area = Area(name=geoname)
new_area.save()
area = new_area
try:
album = Album.objects.get(name=album_name)
except ObjectDoesNotExist:
new_album = Album(name=album_name,
atype=Album.COLLECTION,
is_public=True)
new_album.save()
album = new_album
translation.activate(text_language)
query_result = self.query_europeana(query)
item_count = int(query_result["itemsCount"])
ret = []
for i in xrange(0, item_count):
if "dataProvider" in query_result["items"][
i] and "id" in query_result["items"][i]:
if not self._resource_already_exists(
query_result["items"][i]["dataProvider"][0],
query_result["items"][i]["id"]):
new_photo = Photo(
area=area,
source=Source.objects.get(
description=query_result["items"][i]
["dataProvider"][0]),
source_key=query_result["items"][i]["id"],
licence="Public domain")
if "edmIsShownAt" in query_result["items"][i]:
new_photo.source_url = query_result["items"][i][
"edmIsShownAt"][0]
if "edmAgentLabel" in query_result["items"][i]:
new_photo.author = query_result["items"][i][
"edmAgentLabel"][0]["def"]
if "title" in query_result["items"][i]:
new_photo.description = query_result["items"][i][
"title"][0]
opener = build_opener()
opener.addheaders = [(
"User-Agent",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/537.36"
)]
try:
img_response = opener.open(
query_result["items"][i]["edmIsShownBy"][0])
new_photo.image.save("europeana.jpg",
ContentFile(img_response.read()))
new_photo.save()
ap = AlbumPhoto(album=album, photo=new_photo)
ap.save()
except:
pass
def handle(self, *args, **options):
translation.activate('en')
set_id = '72157652352869904'
page = 1
set_url = 'https://api.flickr.com/services/rest/?method=flickr.photosets.getPhotos&api_key=' + settings.FLICKR_API_KEY + '&photoset_id=' + set_id + '&extras=license,owner_name,geo,tags&format=json&nojsoncallback=1&page=' + str(
page)
# https://farm{farm-id}.staticflickr.com/{server-id}/{id}_{o-secret}_o.(jpg|gif|png)
image_url_template = 'https://farm%s.staticflickr.com/%s/%s_%s_b.jpg'
# https://www.flickr.com/photos/{user-id}/{photo-id}
reference_url_template = 'https://www.flickr.com/photos/%s/%s'
request = Request(set_url)
response = urlopen(request)
data = response.read()
data = json.loads(data)
source = Source.objects.filter(description='Flickr').first()
if not source:
source = Source(name='Flickr', description='Flickr')
source.save()
licence = Licence.objects.get(
url='https://creativecommons.org/licenses/by/2.0/')
album = Album.objects.get(pk=1089)
for photo in data['photoset']['photo']:
if not self._resource_already_exists(photo['id']):
new_photo = Photo(
source=source,
source_url=(reference_url_template %
(photo['ownername'], photo['id'])).replace(
' ', '_'),
source_key=photo['id'],
keywords=photo['tags'],
licence=licence,
description=photo['title'],
author='CeriC')
try:
image_url = image_url_template % (
photo['farm'], photo['server'], photo['id'],
photo['secret'])
opener = build_opener()
opener.addheaders = [(
"User-Agent",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/537.36"
)]
img_response = opener.open(image_url)
new_photo.image.save("ceric.jpg",
ContentFile(img_response.read()))
new_photo.save()
if photo['latitude'] and photo['longitude']:
source_geotag = GeoTag(lat=photo['latitude'],
lon=photo['longitude'],
origin=GeoTag.SOURCE,
type=GeoTag.SOURCE_GEOTAG,
map_type=GeoTag.NO_MAP,
photo=new_photo,
is_correct=True,
trustworthiness=0.07)
source_geotag.save()
new_photo.latest_geotag = source_geotag.created
new_photo.set_calculated_fields()
new_photo.save()
ap = AlbumPhoto(album=album, photo=new_photo)
ap.save()
if not album.cover_photo:
album.cover_photo = new_photo
album.light_save()
except:
# print "Problem loading image %s" % photo['id']
continue
album.save()
def handle(self, *args, **options):
urls = [
"http://www.europeana.eu/portal/record/08535/local__default__F_7595.html?start=12&query=*%3A*&startPage=1&qf=ansicht&qf=where%3Avienna&qf=REUSABILITY%3Arestricted&qf=photograph&qt=false&rows=96&format=labels",
"http://www.europeana.eu/portal/record/08535/local__default__F_7598.html?start=13&query=*%3A*&startPage=1&qf=ansicht&qf=where%3Avienna&qf=REUSABILITY%3Arestricted&qf=photograph&qt=false&rows=96&format=labels",
"http://www.europeana.eu/portal/record/08535/local__default__F_7599.html?start=14&query=*%3A*&startPage=1&qf=ansicht&qf=where%3Avienna&qf=REUSABILITY%3Arestricted&qf=photograph&qt=false&rows=96&format=labels",
"http://www.europeana.eu/portal/record/08535/local__default__F_7600.html?start=15&query=*%3A*&startPage=1&qf=ansicht&qf=where%3Avienna&qf=REUSABILITY%3Arestricted&qf=photograph&qt=false&rows=96&format=labels",
"http://www.europeana.eu/portal/record/08535/local__default__F_3450.html?start=17&query=*%3A*&startPage=1&qf=ansicht&qf=where%3Avienna&qf=REUSABILITY%3Arestricted&qf=photograph&qt=false&rows=96&format=labels",
"http://www.europeana.eu/portal/record/08535/local__default__F_3451.html?start=18&query=*%3A*&startPage=1&qf=ansicht&qf=where%3Avienna&qf=REUSABILITY%3Arestricted&qf=photograph&qt=false&rows=96&format=labels",
"http://www.europeana.eu/portal/record/08535/local__default__F_3453.html?start=19&query=*%3A*&startPage=1&qf=ansicht&qf=where%3Avienna&qf=REUSABILITY%3Arestricted&qf=photograph&qt=false&rows=96&format=labels",
"http://www.europeana.eu/portal/record/08535/local__default__F_3449.html?start=20&query=*%3A*&startPage=1&qf=ansicht&qf=where%3Avienna&qf=REUSABILITY%3Arestricted&qf=photograph&qt=false&rows=96&format=labels",
"http://www.europeana.eu/portal/record/08535/local__default__F_3452.html?start=21&query=*%3A*&startPage=1&qf=ansicht&qf=where%3Avienna&qf=REUSABILITY%3Arestricted&qf=photograph&qt=false&rows=96&format=labels",
]
area = Area.objects.filter(name='Vienna').first()
if not area:
new_area = Area(name='Vienna')
new_area.lat = '48.2000'
new_area.lon = '16.3667'
new_area.save()
area = new_area
new_album = Album.objects.filter(name='Vienna, Austria').first()
if not new_album:
new_album = Album(name='Vienna, Austria', atype=Album.CURATED)
new_album.save()
new_album.save()
translation.activate('de')
for url in urls:
result = requests.get(url)
tree = html.fromstring(result.text)
image_url = tree.xpath(
'//td[@property="isShownBy http://www.europeana.eu/schemas/edm/isShownBy"]/text()'
)[0]
source_url = tree.xpath(
'//td[@property="isShownAt http://www.europeana.eu/schemas/edm/isShownAt"]/text()'
)[0]
author = tree.xpath(
'//span[@property="creator http://purl.org/dc/elements/1.1/creator"]/text()'
)[0]
source = Source.objects.filter(
name=
"Architekturmuseum der Technischen Universität Berlin in der Universitätsbibliothek"
).first()
if not source:
source = Source(
name=
"Architekturmuseum der Technischen Universität Berlin in der Universitätsbibliothek",
description=
"Architekturmuseum der Technischen Universität Berlin in der Universitätsbibliothek"
)
source.save()
description = tree.xpath(
'//span[@property="subject http://purl.org/dc/elements/1.1/subject"]/text()'
)[0]
source_key = tree.xpath(
'//span[@property="identifier http://purl.org/dc/elements/1.1/identifier"]/text()'
)[0]
if not description:
description = tree.xpath(
'//span[@property="title http://purl.org/dc/terms/title"]/text()'
)[0]
new_photo = Photo(
description=description,
source_key=source_key,
source=source,
source_url=source_url,
area=area,
author=author,
licence=Licence.objects.get(
url="http://creativecommons.org/licenses/by-nc-sa/4.0/"),
)
new_photo.save()
opener = build_opener()
opener.addheaders = [(
"User-Agent",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/537.36"
)]
img_response = opener.open(image_url)
new_photo.image.save("europeana.jpg",
ContentFile(img_response.read()))
new_photo.width = new_photo.image.width
new_photo.height = new_photo.image.height
AlbumPhoto(album=new_album, photo=new_photo).save()
def handle(self, *args, **options):
logger = logging.getLogger(__name__)
# Import sets
muis_url = 'https://www.muis.ee/OAIService/OAIService'
set_name = (options['set_name'])[0]
museum_name = set_name.split(':')[0]
source = Source.objects.filter(name=museum_name).first()
if source is None:
sets_url = muis_url + '?verb=ListSets'
url_response = urllib.request.urlopen(sets_url)
parser = ET.XMLParser(encoding="utf-8")
tree = ET.fromstring(url_response.read(), parser=parser)
ns = {'d': 'http://www.openarchives.org/OAI/2.0/'}
sets = tree.findall('d:ListSets/d:set', ns)
for s in sets:
if s.find('d:setSpec', ns).text == museum_name:
source_description = s.find('d:setName', ns).text
source = Source(name=museum_name, description=source_description)
source.save()
source = Source.objects.filter(name=museum_name).first()
album_ids = (options['album_ids'])
albums = Album.objects.filter(id__in=album_ids)
all_person_album_ids_set = set()
list_identifiers_url = muis_url + '?verb=ListRecords&set=' + set_name \
+ '&metadataPrefix=lido'
url_response = urllib.request.urlopen(list_identifiers_url)
parser = ET.XMLParser(encoding="utf-8")
tree = ET.fromstring(url_response.read(), parser=parser)
ns = {'d': 'http://www.openarchives.org/OAI/2.0/', 'lido': 'http://www.lido-schema.org'}
header = 'd:header/'
records = tree.findall('d:ListRecords/d:record', ns)
record = 'd:metadata/lido:lidoWrap/lido:lido/'
object_identification_wrap = record + 'lido:descriptiveMetadata/lido:objectIdentificationWrap/'
object_description_wraps = \
object_identification_wrap + 'lido:objectDescriptionWrap/lido:objectDescriptionSet'
title_wrap = object_identification_wrap + 'lido:titleWrap/'
repository_wrap = object_identification_wrap + 'lido:repositoryWrap/'
event_wrap = record + 'lido:descriptiveMetadata/lido:eventWrap/'
record_wrap = record + 'lido:administrativeMetadata/lido:recordWrap/'
resource_wrap = record + 'lido:administrativeMetadata/lido:resourceWrap/'
actor_wrap = event_wrap + 'lido:eventSet/lido:event/lido:eventActor/'
for rec in records:
try:
locations = []
person_album_ids = []
creation_date_earliest = None
creation_date_latest = None
external_id = rec.find(header + 'd:identifier', ns).text \
if rec.find(header + 'd:identifier', ns) is not None \
else None
existing_photo = Photo.objects.filter(source=source, external_id=external_id).first()
if existing_photo is not None:
continue
image_url = rec.find(resource_wrap + 'lido:resourceSet/lido:'
+ 'resourceRepresentation/lido:linkResource', ns).text \
if rec.find(resource_wrap + 'lido:resourceSet/lido:'
+ 'resourceRepresentation/lido:linkResource', ns) is not None\
else None
image_extension = rec.find(resource_wrap + 'lido:resourceSet/lido:'
+ 'resourceRepresentation/lido:linkResource',
ns).attrib['{' + ns['lido'] + '}formatResource'] \
if rec.find(resource_wrap + 'lido:resourceSet/lido:'
+ 'resourceRepresentation/lido:linkResource', ns) is not None\
else None
source_url_find = rec.find(record_wrap + 'lido:recordInfoSet/lido:recordInfoLink', ns)
source_url = source_url_find.text \
if source_url_find is not None \
else None
identifier_find = rec.find(repository_wrap + 'lido:repositorySet/lido:workID', ns)
identifier = identifier_find.text \
if identifier_find is not None \
else None
if image_url is None:
continue
img_data = requests.get(image_url).content
image_id = external_id.split(':')[-1]
file_name = set_name + '_' + image_id + '.' + image_extension
file_name = file_name.replace(':', '_')
path = settings.MEDIA_ROOT + '/uploads/' + file_name
with open(path, 'wb') as handler:
handler.write(img_data)
photo = Photo(
image=path,
source_key=identifier,
source_url=source_url,
external_id=external_id,
source=source
)
dt = datetime.utcnow()
dt.replace(tzinfo=timezone.utc)
photo.muis_update_time = dt.replace(tzinfo=timezone.utc).isoformat()
photo.light_save()
photo = Photo.objects.get(id=photo.id)
photo.image.name = 'uploads/' + file_name
title_find = rec.find(title_wrap + 'lido:titleSet/lido:appellationValue', ns)
title = title_find.text \
if title_find is not None \
else None
if title:
photo = reset_modeltranslated_field(photo, 'title', title)
dating = None
photo, dating = set_text_fields_from_muis(photo, dating, rec, object_description_wraps, ns)
photo.light_save()
creation_date_earliest = None
creation_date_latest = None
date_prefix_earliest = None
date_prefix_latest = None
events = rec.findall(event_wrap + 'lido:eventSet/lido:event', ns)
if events is not None and len(events) > 0:
locations, \
creation_date_earliest, \
creation_date_latest, \
date_prefix_earliest, \
date_prefix_latest, \
date_earliest_has_suffix, \
date_latest_has_suffix, \
= extract_dating_from_event(
events,
locations,
creation_date_earliest,
creation_date_latest,
photo.latest_dating is not None or dating is not None,
ns
)
if dating is not None:
creation_date_earliest, date_prefix_earliest, date_earliest_has_suffix = \
get_muis_date_and_prefix(dating, False)
creation_date_latest, date_prefix_latest, date_latest_has_suffix = \
get_muis_date_and_prefix(dating, True)
actors = rec.findall(actor_wrap + 'lido:actorInRole', ns)
person_album_ids = add_person_albums(actors, person_album_ids, ns)
photo.add_to_source_album()
if locations != []:
photo = add_geotag_from_address_to_photo(photo, locations)
photo = add_dating_to_photo(
photo,
creation_date_earliest,
creation_date_latest,
date_prefix_earliest,
date_prefix_latest,
Dating,
date_earliest_has_suffix,
date_latest_has_suffix,
)
photo.light_save()
for album in albums:
if not album.cover_photo:
album.cover_photo = photo
ap = AlbumPhoto(photo=photo, album=album, type=AlbumPhoto.CURATED)
ap.save()
person_albums = Album.objects.filter(id__in=person_album_ids)
if person_albums is not None:
for album in person_albums:
if not album.cover_photo:
album.cover_photo = photo
ap = AlbumPhoto(photo=photo, album=album, type=AlbumPhoto.FACE_TAGGED)
ap.save()
all_person_album_ids_set.add(album.id)
photo.set_calculated_fields()
except Exception as e:
logger.exception(e)
exception = ApplicationException(exception=traceback.format_exc(), photo=photo)
exception.save()
for album in albums:
album.set_calculated_fields()
album.save()
all_person_album_ids = list(all_person_album_ids_set)
all_person_albums = Album.objects.filter(id__in=all_person_album_ids)
if all_person_albums is not None:
for person_album in all_person_albums:
person_album.set_calculated_fields()
person_album.save()
def finna_import_photo(id, profile):
record_url = 'https://api.finna.fi/v1/record'
finna_result = get(
record_url, {
'id':
id,
'field[]': [
'id', 'title', 'shortTitle', 'images', 'imageRights',
'authors', 'source', 'geoLocations', 'recordPage', 'year',
'summary', 'rawData'
],
})
results = finna_result.json()
p = results.get('records', None)
# print >> sys.stderr, ('import_finna_photo: %s' % 'https://api.finna.fi/v1/record?id=' + id)
if p and p[0]:
p = p[0]
comma = ', '
if not len(p['images']):
return None
institution = None
if p['source']:
if 'translated' in p['source'][0]:
institution = p['source'][0]['translated']
elif 'value' in p['source'][0]:
institution = p['source'][0]['value']
geography = None
if 'geoLocations' in p:
for each in p['geoLocations']:
if 'POINT' in each:
point_parts = each.split(' ')
lon = point_parts[0][6:]
lat = point_parts[1][:-1]
p['longitude'] = lon
p['latitude'] = lat
geography = Point(x=float(lon), y=float(lat), srid=4326)
break
title = p.get('title').rstrip()
summary = ''
if 'summary' in p:
for each in p['summary']:
if len(each.rstrip()) > len(summary):
summary = re.sub(
'--[^-]*?(filmi|paperi|negatiivi|digitaalinen|dng|dia|lasi|väri|tif|jpg|, mv)[^-]*?$',
'', each).rstrip()
if title in summary:
description = summary
elif len(title) < 20:
description = title + '; ' + summary
else:
description = title
address = ''
if 'rawData' in p and 'geographic' in p['rawData']:
for each in p['rawData']['geographic']:
if len(each.rstrip()) > len(address):
address = each.rstrip()
licence = Licence.objects.get(pk=15) # 15 = unknown licence
if 'imageRights' in p and 'copyright' in p['imageRights']:
licence_str = p['imageRights']['copyright']
if licence_str:
licence = Licence.objects.filter(name=licence_str).first()
if not licence:
licence = Licence(name=licence_str)
licence.save()
authors = []
if 'authors' in p:
if p['authors']['primary']:
for k, each in p['authors']['primary'].items():
authors.append(k)
source = Source.objects.filter(description=institution).first()
if not source:
# print >> sys.stderr, ('Saving missing source '%s'' % institution)
source = Source(name=institution, description=institution)
source.save()
source = Source.objects.filter(description=institution).first()
external_id = p['id'][:99] # muis_id
external_sub_id = None # muis_media_id
if '_' in p['id']:
external_id = p['id'].split('_')[0]
external_sub_id = p['id'].split('_')[1]
new_photo = Photo(user=profile,
author=comma.join(authors),
title=finna_cut_title(p.get('title', None),
p.get('title_short', None)),
description=description,
address=address[:255],
source=source,
types=None,
keywords=None,
date_text=p.get('year').encode('utf-8') if p.get(
'year', None) else None,
licence=licence,
external_id=external_id,
external_sub_id=external_sub_id,
source_key=p.get('id', None),
source_url='https://www.finna.fi' +
p.get('recordPage'),
geography=geography)
opener = build_opener()
opener.addheaders = [('User-Agent', settings.UA)]
img_url = 'https://www.finna.fi' + p['images'][0].replace(
'size=large', 'size=master')
img_response = opener.open(img_url)
new_photo.image.save('finna.jpg', ContentFile(img_response.read()))
# print >> sys.stderr, ('import_finna_photo saving file')
new_photo.save()
if p.get('latitude') and p.get('longitude') and not \
GeoTag.objects.filter(type=GeoTag.SOURCE_GEOTAG, photo__source_key=new_photo.source_key).exists():
source_geotag = GeoTag(lat=p.get('latitude'),
lon=p.get('longitude'),
origin=GeoTag.SOURCE,
type=GeoTag.SOURCE_GEOTAG,
map_type=GeoTag.NO_MAP,
photo=new_photo,
is_correct=True,
trustworthiness=0.07,
geography=geography)
source_geotag.save()
new_photo.latest_geotag = source_geotag.created
new_photo.set_calculated_fields()
new_photo.save()
new_photo.add_to_source_album()
id = int(new_photo.id)
photo = Photo.objects.filter(pk=id).first()
# print >> sys.stderr, ('import_finna_photo saved photo id: %d' % new_photo.id)
return photo
def handle(self, *args, **options):
photos = os.listdir(settings.MEDIA_ROOT +
"/uploads/Digar_postkaartide_tagakyljed/")
for p in photos:
first = Photo.objects.filter(external_id=p.split("_")[1]).first()
if first is None or (first.source_url
and not 'digar' in first.source_url):
print(p.split("_")[1])
else:
new_photo = Photo(
image=settings.MEDIA_ROOT +
"/uploads/Digar_postkaartide_tagakyljed/DIGAR_" +
p.split("_")[1] + "_2.jpg",
source=first.source,
source_url=first.source_url,
source_key=first.source_key,
licence=first.licence,
description=first.description)
if first.title:
new_photo.title = first.title
if first.description:
new_photo.description = first.description
if first.author:
new_photo.author = first.author
if first.licence_id:
new_photo.licence_id = first.licence_id
if first.source_id:
new_photo.source_id = first.source_id
if first.source_url:
new_photo.source_url = first.source_url
if first.source_key:
new_photo.source_key = first.source_key
if first.date_text:
new_photo.date_text = first.date_text
if first.date:
new_photo.date = first.date
try:
new_photo.save()
first.set_postcard(new_photo)
except Exception as e:
print(e)