def import_images(logo_dir):
report_data = dict(failed=0, succeeded=0)
for logo_name in os.listdir(logo_dir):
error_message = ""
identifier, extension = splitext(logo_name)
if extension.lower() in (".png", ".jpg", ".jpeg", ".gif"):
try:
internal_org_id = InternalOrganisationID.objects.get(
recording_org=Organisation.objects.get(id=CORDAID_ORG_ID),
identifier=identifier)
org = internal_org_id.referenced_org
filename = model_and_instance_based_filename(
"Organisation", org.pk, "logo", logo_name)
with open(os.path.join(logo_dir, logo_name), "rb") as f:
logo_data = f.read()
logo_tmp = NamedTemporaryFile(delete=True)
logo_tmp.write(logo_data)
logo_tmp.flush()
org.logo.save(filename, File(logo_tmp), save=True)
action = "succeeded"
except Exception, e:
action = "failed"
error_message = "with the following error message: {error_message}".format(
error_message=e.message)
report_data[action] += 1
log_and_alert(
u"Upload of image to organisation {org_id} {action} {error_message}"
.format(org_id=org.id, action=action, error_message=error_message))
def import_images(logo_dir):
report_data = dict(failed=0, succeeded=0)
for logo_name in os.listdir(logo_dir):
error_message = ""
identifier, extension = splitext(logo_name)
if extension.lower() in (".png", ".jpg", ".jpeg", ".gif"):
try:
internal_org_id = InternalOrganisationID.objects.get(
recording_org=Organisation.objects.get(id=CORDAID_ORG_ID), identifier=identifier
)
org = internal_org_id.referenced_org
filename = model_and_instance_based_filename("Organisation", org.pk, "logo", logo_name)
with open(os.path.join(logo_dir, logo_name), "rb") as f:
logo_data = f.read()
logo_tmp = NamedTemporaryFile(delete=True)
logo_tmp.write(logo_data)
logo_tmp.flush()
org.logo.save(filename, File(logo_tmp), save=True)
action = "succeeded"
except Exception, e:
action = "failed"
error_message = "with the following error message: {error_message}".format(error_message=e.message)
report_data[action] += 1
log_and_alert(
u"Upload of image to organisation {org_id} {action} {error_message}".format(
org_id=org.id, action=action, error_message=error_message
)
)
class ProjectSaver():
""" Perform the post import saving steps for the project, all related to the project image
"""
def __init__(self, activity, importing_org):
self.activity = activity
if activity.iati_id():
try:
self.project = Project.objects.get(
iati_activity_id=activity.iati_id()
)
return
except:
msg = "Could not find project or multiple projects found with IATI ID: {iati_id}"
log(
msg,
dict(
iati_id=self.activity.iati_id(),
)
)
raise Project.DoesNotExist
def _current_image(self):
image = ImageImporter(RVO_DEFAULT_IMAGE)
try:
image.get_image()
except Exception, e:
log(
"Error trying to fetch image to project. Image URL: {extra}",
dict(
iati_id=self.activity.iati_id(),
event=ERROR_IMAGE_UPLOAD,
extra=self.activity.current_image()
)
)
if image.image:
filename = model_and_instance_based_filename(
'Project', self.project.pk, 'current_image', image.filename
)
image_temp = NamedTemporaryFile(delete=True)
image_temp.write(image.image)
image_temp.flush()
self.project.current_image.save(filename, File(image_temp), save=True)
log(
"Save project image: {extra}",
dict(
iati_id=self.activity.iati_id(),
event=ACTION_SET_IMAGE,
extra=filename
)
)
else:
log(
"No image found for project: {rsr_id}",
dict(
iati_id=self.activity.iati_id(),
event=ERROR_IMAGE_NOT_FOUND,
)
)
def cleanup_images(queryset):
"""Clean out the images file tree of old images.
The method for cleaning out old images is as follows:
for every object of the relevant model
for each image field
save the image to a temp directory
remove all images in the "real" directory for the object
for each image field
save the image back to the "real" dir
"""
for obj in queryset:
opts = obj._meta
for f in opts.fields:
if type(f).__name__ == 'ImageField':
model_field = getattr(obj, f.name)
if hasattr(model_field, 'file'):
print("Temp saving:", model_field.name)
name = os.path.split(model_field.name)[1]
# save the func used for upload_to. model_field.field.generate_filename points
# to the model's image_path() function, but the function isn't a true model method
# leading to problems when trying to get it from the model.
# There's probably some introspection way to get to the func
orig_image_path = model_field.field.generate_filename
model_field.field.generate_filename = temp_image
model_field.save(name, model_field.file)
path = obj.image_path('')
full_path = os.path.join(settings.MEDIA_ROOT, path)
try: # path might not exist, for updates without images for instance
for file_name in os.listdir(full_path):
try:
os.remove(os.path.join(full_path, file_name))
except OSError:
pass
except OSError:
pass
# time to reverse
for f in opts.fields:
if type(f).__name__ == 'ImageField':
model_field = getattr(obj, f.name)
if hasattr(model_field, 'file'):
current_name = os.path.split(model_field.name)[1]
name_parts = current_name.split('_')
# check if file name fits pattern
if (name_parts[0] == opts.object_name
and name_parts[1] == str(obj.pk)
and name_parts[2] == f.name.split('_')[0]):
# remove any trailing '_' that may occur if cleanup was run more than once
# without emptying the temp dir
current_name = os.path.splitext(current_name)[0].strip(
'_') + os.path.splitext(current_name)[1]
name = current_name
else:
# files with non-conformant names get fixed.
name = model_and_instance_based_filename(
opts.object_name, obj.pk, f.name, model_field.name)
model_field.field.generate_filename = orig_image_path
model_field.save(name, model_field.file)
print("Putting back:", model_field.name)
def cleanup_images(queryset):
"""Clean out the images file tree of old images.
The method for cleaning out old images is as follows:
for every object of the relevant model
for each image field
save the image to a temp directory
remove all images in the "real" directory for the object
for each image field
save the image back to the "real" dir
"""
for obj in queryset:
opts = obj._meta
for f in opts.fields:
if type(f).__name__ == 'ImageField':
model_field = getattr(obj, f.name)
if hasattr(model_field, 'file'):
print "Temp saving:", model_field.name
name = os.path.split(model_field.name)[1]
# save the func used for upload_to. model_field.field.generate_filename points
# to the model's image_path() function, but the function isn't a true model method
# leading to problems when trying to get it from the model.
# There's probably some introspection way to get to the func
orig_image_path = model_field.field.generate_filename
model_field.field.generate_filename = temp_image
model_field.save(name, model_field.file)
path = obj.image_path('')
full_path = os.path.join(settings.MEDIA_ROOT, path)
try: # path might not exist, for updates without images for instance
for file_name in os.listdir(full_path):
try:
os.remove(os.path.join(full_path, file_name))
except OSError:
pass
except OSError:
pass
# time to reverse
for f in opts.fields:
if type(f).__name__ == 'ImageField':
model_field = getattr(obj, f.name)
if hasattr(model_field, 'file'):
current_name = os.path.split(model_field.name)[1]
name_parts = current_name.split('_')
# check if file name fits pattern
if (name_parts[0] == opts.object_name and name_parts[1] == unicode(obj.pk) and name_parts[2] == f.name.split('_')[0]):
# remove any trailing '_' that may occur if cleanup was run more than once
# without emptying the temp dir
current_name = os.path.splitext(current_name)[0].strip('_') + os.path.splitext(current_name)[1]
name = current_name
else:
# files with non-conformant names get fixed.
name = model_and_instance_based_filename(opts.object_name, obj.pk, f.name, model_field.name)
model_field.field.generate_filename = orig_image_path
model_field.save(name, model_field.file)
print "Putting back:", model_field.name
def test_create_filename(self):
"""
Create a file name for an image based on the model name, the current object's pk, the
field name of the model and the current date and time.
"""
filename = model_and_instance_based_filename('objectname', 1, 'fieldname', 'imgname.jpg')
filename_list = filename.split('_')
# Test that the filename contains the correct parts
self.assertIn('objectname', filename_list)
self.assertIn('1', filename_list)
self.assertIn('fieldname', filename_list)
self.assertEqual('.jpg', filename[-4:])
def test_create_filename(self):
"""
Create a file name for an image based on the model name, the current object's pk, the
field name of the model and the current date and time.
"""
filename = model_and_instance_based_filename('objectname', 1, 'fieldname', 'imgname.jpg')
filename_list = filename.split('_')
# Test that the filename contains the correct parts
self.assertIn('objectname', filename_list)
self.assertIn('1', filename_list)
self.assertIn('fieldname', filename_list)
self.assertEqual('.jpg', filename[-4:])
def do_import(self):
"""
:return: List; contains fields that have changed
"""
from . import same_data
changes = []
photo_id = self.get_attrib(self.parent_elem, akvo_ns('photo-id'),
'current_image')
current_image = file_from_zip_archive(
self.iati_import_job.iati_xml_file,
"out_proj/{}.jpg".format(photo_id))
if current_image:
tmp_file = NamedTemporaryFile()
for line in current_image.readlines():
tmp_file.write(line)
tmp_file.flush()
# update current image if it's different from the existing one
try:
old_file = self.project.current_image.file
except (IOError, ValueError):
old_file = None
new_file = File(tmp_file)
if not same_data(old_file, new_file):
filename = model_and_instance_based_filename(
'Project', self.project.pk, 'current_image', 'image.jpg')
new_file.seek(0)
self.project.current_image.save(filename, new_file)
changes += ['current_image']
current_image_caption = self.get_attrib(self.parent_elem,
akvo_ns('image-caption'),
'current_image_caption')
if current_image_caption:
changes += self.update_project_field('current_image_caption',
current_image_caption)
current_image_credit = self.get_attrib(self.parent_elem,
akvo_ns('photo-credit'),
'current_image_credit')
if current_image_credit:
changes += self.update_project_field('current_image_credit',
current_image_credit)
return changes
def import_images(image_dir, photos):
outsys("\nRunning {}() ".format(who_am_i()))
for image_name in os.listdir(image_dir):
photo_id, ext = splitext(image_name)
if ext.lower() in ['.png', '.jpg', '.jpeg', '.gif']:
try:
internal_id = photos.get(
photo_id,
{'internal_project_id': None})['internal_project_id']
project = Project.objects.get(
partnerships__internal_id=internal_id)
filename = model_and_instance_based_filename(
'Project', project.pk, 'current_image', image_name)
with open(os.path.join(image_dir, image_name), 'rb') as f:
image_data = f.read()
image_temp = NamedTemporaryFile(delete=True)
image_temp.write(image_data)
image_temp.flush()
project.current_image.save(filename,
File(image_temp),
save=True)
f.close()
project.current_image_caption = photos.get(
photo_id, {'image_caption': ''})['image_caption']
project.current_image_credit = photos.get(
photo_id, {'image_credit': ''})['image_credit']
project.save()
log(
u"Uploaded image to project {pk}",
dict(internal_id=internal_id,
pk=project.pk,
event=ACTION_SET_IMAGE))
outsys(".")
except Exception, e:
log(
u"Upload failed. internal_id: {internal_id} Exception class: {extra}",
dict(internal_id=internal_id,
event=ERROR_IMAGE_UPLOAD,
extra=e.__class__),
)
outsys("*")
def do_import(self):
"""
:return: List; contains fields that have changed
"""
from . import same_data
changes = []
photo_id = self.get_attrib(self.parent_elem, akvo_ns('photo-id'), 'current_image')
current_image = file_from_zip_archive(
self.iati_import_job.iati_xml_file, "out_proj/{}.jpg".format(photo_id))
if current_image:
tmp_file = NamedTemporaryFile()
for line in current_image.readlines():
tmp_file.write(line)
tmp_file.flush()
# update current image if it's different from the existing one
try:
old_file = self.project.current_image.file
except (IOError, ValueError):
old_file = None
new_file = File(tmp_file)
if not same_data(old_file, new_file):
filename = model_and_instance_based_filename(
'Project', self.project.pk, 'current_image', 'image.jpg')
new_file.seek(0)
self.project.current_image.save(filename, new_file)
changes += ['current_image']
current_image_caption = self.get_attrib(
self.parent_elem, akvo_ns('image-caption'), 'current_image_caption')
if current_image_caption:
changes += self.update_project_field('current_image_caption', current_image_caption)
current_image_credit = self.get_attrib(
self.parent_elem, akvo_ns('photo-credit'), 'current_image_credit')
if current_image_credit:
changes += self.update_project_field('current_image_credit', current_image_credit)
return changes
def import_images(image_dir, photos):
outsys("\nRunning {}() ".format(who_am_i()))
for image_name in os.listdir(image_dir):
photo_id, ext = splitext(image_name)
if ext.lower() in ['.png', '.jpg', '.jpeg', '.gif']:
try:
internal_id = photos.get(
photo_id, {'internal_project_id': None}
)['internal_project_id']
project = Project.objects.get(
partnerships__internal_id=internal_id
)
filename = model_and_instance_based_filename(
'Project', project.pk, 'current_image', image_name
)
with open(os.path.join(image_dir, image_name), 'rb') as f:
image_data = f.read()
image_temp = NamedTemporaryFile(delete=True)
image_temp.write(image_data)
image_temp.flush()
project.current_image.save(filename, File(image_temp), save=True)
f.close()
project.current_image_caption = photos.get(
photo_id, {'image_caption': ''}
)['image_caption']
project.current_image_credit = photos.get(
photo_id, {'image_credit': ''}
)['image_credit']
project.save()
log(
u"Uploaded image to project {pk}",
dict(internal_id=internal_id, pk=project.pk, event=ACTION_SET_IMAGE))
outsys(".")
except Exception, e:
log(
u"Upload failed. internal_id: {internal_id} Exception class: {extra}",
dict(internal_id=internal_id, event=ERROR_IMAGE_UPLOAD, extra=e.__class__),
)
outsys("*")
def set_logo(self, organisation, identifier):
""" Get the organisation's logo from the Cordaid zip archive """
logo = file_from_zip_archive(
self.iati_import_job.iati_xml_file, "out_rltn/{}.jpg".format(identifier))
if logo:
tmp_file = NamedTemporaryFile()
for line in logo.readlines():
tmp_file.write(line)
tmp_file.flush()
# update the logo if it's different from the existing one
try:
old_file = organisation.logo.file
except (IOError, ValueError):
old_file = None
new_file = File(tmp_file)
if not same_data(old_file, new_file):
filename = model_and_instance_based_filename(
'Organisation', organisation.pk, 'logo', 'image.jpg')
new_file.seek(0)
organisation.logo.save(filename, new_file)
return 'logo'
return None
def set_logo(self, organisation, identifier):
""" Get the organisation's logo from the Cordaid zip archive """
logo = file_from_zip_archive(
self.iati_import_job.iati_xml_file, "out_rltn/{}.jpg".format(identifier))
if logo:
tmp_file = NamedTemporaryFile()
for line in logo.readlines():
tmp_file.write(line)
tmp_file.flush()
# update the logo if it's different from the existing one
try:
old_file = organisation.logo.file
except (IOError, ValueError):
old_file = None
new_file = File(tmp_file)
if not same_data(old_file, new_file):
filename = model_and_instance_based_filename(
'Organisation', organisation.pk, 'logo', 'image.jpg')
new_file.seek(0)
organisation.logo.save(filename, new_file)
return 'logo'
return None
def organisation_logo(org_etree, internal_id, org):
logo_file = glob.glob(
os.path.join(
CORDAID_LOGOS_DIR,
u"{logo_id}.*".format(logo_id=text_from_xpath(org_etree, 'logo_id'))
)
)
if len(logo_file) == 1:
logo_filename = basename(logo_file[0])
_, extension = splitext(logo_filename)
if extension.lower() in (".png", ".jpg", ".jpeg", ".gif"):
filename = model_and_instance_based_filename(
"Organisation",
org.pk,
"logo",
logo_filename
)
with open(os.path.join(CORDAID_LOGOS_DIR, logo_filename), "rb") as f:
logo_data = f.read()
logo_tmp = NamedTemporaryFile(delete=True)
logo_tmp.write(logo_data)
logo_tmp.flush()
org.logo.save(
filename, File(logo_tmp), save=True
)
log(
u" Added logo {extra} to org {pk}, ",
dict(
log_type=LOG_ORGANISATIONS,
internal_id=internal_id,
pk=org.pk,
label=org.name,
event=ACTION_SET_IMAGE,
extra= filename,
)
)
def _current_image(self):
image = ImageImporter(self.activity.current_image())
try:
image.get_image()
except Exception, e:
log(
"Error trying to fetch image to project. Image URL: {extra}",
dict(rsr_id=self.activity.rsr_id(),
internal_id=self.activity.internal_id(),
iati_id=self.activity.iati_id(),
event=ERROR_IMAGE_UPLOAD,
extra=self.activity.current_image()))
if image.image:
filename = model_and_instance_based_filename(
'Project', self.project.pk, 'current_image', image.filename)
image_temp = NamedTemporaryFile(delete=True)
image_temp.write(image.image)
image_temp.flush()
self.project.current_image.save(filename,
File(image_temp),
save=True)
log(
"Save project image: {extra}",
dict(rsr_id=self.activity.rsr_id(),
internal_id=self.activity.internal_id(),
iati_id=self.activity.iati_id(),
event=ACTION_SET_IMAGE,
extra=filename))
else:
log(
image.get_image()
except Exception, e:
log(
"Error trying to fetch image to project. Image URL: {extra}",
dict(
rsr_id=self.activity.rsr_id(),
internal_id=self.activity.internal_id(),
iati_id=self.activity.iati_id(),
event=ERROR_IMAGE_UPLOAD,
extra=self.activity.current_image()
)
)
if image.image:
filename = model_and_instance_based_filename(
'Project', self.project.pk, 'current_image', image.filename
)
image_temp = NamedTemporaryFile(delete=True)
image_temp.write(image.image)
image_temp.flush()
self.project.current_image.save(filename, File(image_temp), save=True)
log(
"Save project image: {extra}",
dict(
rsr_id=self.activity.rsr_id(),
internal_id=self.activity.internal_id(),
iati_id=self.activity.iati_id(),
event=ACTION_SET_IMAGE,
extra=filename
)
)