def import_images(logo_dir): report_data = dict(failed=0, succeeded=0) for logo_name in os.listdir(logo_dir): error_message = "" identifier, extension = splitext(logo_name) if extension.lower() in (".png", ".jpg", ".jpeg", ".gif"): try: internal_org_id = InternalOrganisationID.objects.get( recording_org=Organisation.objects.get(id=CORDAID_ORG_ID), identifier=identifier) org = internal_org_id.referenced_org filename = model_and_instance_based_filename( "Organisation", org.pk, "logo", logo_name) with open(os.path.join(logo_dir, logo_name), "rb") as f: logo_data = f.read() logo_tmp = NamedTemporaryFile(delete=True) logo_tmp.write(logo_data) logo_tmp.flush() org.logo.save(filename, File(logo_tmp), save=True) action = "succeeded" except Exception, e: action = "failed" error_message = "with the following error message: {error_message}".format( error_message=e.message) report_data[action] += 1 log_and_alert( u"Upload of image to organisation {org_id} {action} {error_message}" .format(org_id=org.id, action=action, error_message=error_message))
def import_images(logo_dir): report_data = dict(failed=0, succeeded=0) for logo_name in os.listdir(logo_dir): error_message = "" identifier, extension = splitext(logo_name) if extension.lower() in (".png", ".jpg", ".jpeg", ".gif"): try: internal_org_id = InternalOrganisationID.objects.get( recording_org=Organisation.objects.get(id=CORDAID_ORG_ID), identifier=identifier ) org = internal_org_id.referenced_org filename = model_and_instance_based_filename("Organisation", org.pk, "logo", logo_name) with open(os.path.join(logo_dir, logo_name), "rb") as f: logo_data = f.read() logo_tmp = NamedTemporaryFile(delete=True) logo_tmp.write(logo_data) logo_tmp.flush() org.logo.save(filename, File(logo_tmp), save=True) action = "succeeded" except Exception, e: action = "failed" error_message = "with the following error message: {error_message}".format(error_message=e.message) report_data[action] += 1 log_and_alert( u"Upload of image to organisation {org_id} {action} {error_message}".format( org_id=org.id, action=action, error_message=error_message ) )
class ProjectSaver(): """ Perform the post import saving steps for the project, all related to the project image """ def __init__(self, activity, importing_org): self.activity = activity if activity.iati_id(): try: self.project = Project.objects.get( iati_activity_id=activity.iati_id() ) return except: msg = "Could not find project or multiple projects found with IATI ID: {iati_id}" log( msg, dict( iati_id=self.activity.iati_id(), ) ) raise Project.DoesNotExist def _current_image(self): image = ImageImporter(RVO_DEFAULT_IMAGE) try: image.get_image() except Exception, e: log( "Error trying to fetch image to project. Image URL: {extra}", dict( iati_id=self.activity.iati_id(), event=ERROR_IMAGE_UPLOAD, extra=self.activity.current_image() ) ) if image.image: filename = model_and_instance_based_filename( 'Project', self.project.pk, 'current_image', image.filename ) image_temp = NamedTemporaryFile(delete=True) image_temp.write(image.image) image_temp.flush() self.project.current_image.save(filename, File(image_temp), save=True) log( "Save project image: {extra}", dict( iati_id=self.activity.iati_id(), event=ACTION_SET_IMAGE, extra=filename ) ) else: log( "No image found for project: {rsr_id}", dict( iati_id=self.activity.iati_id(), event=ERROR_IMAGE_NOT_FOUND, ) )
def cleanup_images(queryset): """Clean out the images file tree of old images. The method for cleaning out old images is as follows: for every object of the relevant model for each image field save the image to a temp directory remove all images in the "real" directory for the object for each image field save the image back to the "real" dir """ for obj in queryset: opts = obj._meta for f in opts.fields: if type(f).__name__ == 'ImageField': model_field = getattr(obj, f.name) if hasattr(model_field, 'file'): print("Temp saving:", model_field.name) name = os.path.split(model_field.name)[1] # save the func used for upload_to. model_field.field.generate_filename points # to the model's image_path() function, but the function isn't a true model method # leading to problems when trying to get it from the model. # There's probably some introspection way to get to the func orig_image_path = model_field.field.generate_filename model_field.field.generate_filename = temp_image model_field.save(name, model_field.file) path = obj.image_path('') full_path = os.path.join(settings.MEDIA_ROOT, path) try: # path might not exist, for updates without images for instance for file_name in os.listdir(full_path): try: os.remove(os.path.join(full_path, file_name)) except OSError: pass except OSError: pass # time to reverse for f in opts.fields: if type(f).__name__ == 'ImageField': model_field = getattr(obj, f.name) if hasattr(model_field, 'file'): current_name = os.path.split(model_field.name)[1] name_parts = current_name.split('_') # check if file name fits pattern if (name_parts[0] == opts.object_name and name_parts[1] == str(obj.pk) and name_parts[2] == f.name.split('_')[0]): # remove any trailing '_' that may occur if cleanup was run more than once # without emptying the temp dir current_name = os.path.splitext(current_name)[0].strip( '_') + os.path.splitext(current_name)[1] name = current_name else: # files with non-conformant names get fixed. name = model_and_instance_based_filename( opts.object_name, obj.pk, f.name, model_field.name) model_field.field.generate_filename = orig_image_path model_field.save(name, model_field.file) print("Putting back:", model_field.name)
def cleanup_images(queryset): """Clean out the images file tree of old images. The method for cleaning out old images is as follows: for every object of the relevant model for each image field save the image to a temp directory remove all images in the "real" directory for the object for each image field save the image back to the "real" dir """ for obj in queryset: opts = obj._meta for f in opts.fields: if type(f).__name__ == 'ImageField': model_field = getattr(obj, f.name) if hasattr(model_field, 'file'): print "Temp saving:", model_field.name name = os.path.split(model_field.name)[1] # save the func used for upload_to. model_field.field.generate_filename points # to the model's image_path() function, but the function isn't a true model method # leading to problems when trying to get it from the model. # There's probably some introspection way to get to the func orig_image_path = model_field.field.generate_filename model_field.field.generate_filename = temp_image model_field.save(name, model_field.file) path = obj.image_path('') full_path = os.path.join(settings.MEDIA_ROOT, path) try: # path might not exist, for updates without images for instance for file_name in os.listdir(full_path): try: os.remove(os.path.join(full_path, file_name)) except OSError: pass except OSError: pass # time to reverse for f in opts.fields: if type(f).__name__ == 'ImageField': model_field = getattr(obj, f.name) if hasattr(model_field, 'file'): current_name = os.path.split(model_field.name)[1] name_parts = current_name.split('_') # check if file name fits pattern if (name_parts[0] == opts.object_name and name_parts[1] == unicode(obj.pk) and name_parts[2] == f.name.split('_')[0]): # remove any trailing '_' that may occur if cleanup was run more than once # without emptying the temp dir current_name = os.path.splitext(current_name)[0].strip('_') + os.path.splitext(current_name)[1] name = current_name else: # files with non-conformant names get fixed. name = model_and_instance_based_filename(opts.object_name, obj.pk, f.name, model_field.name) model_field.field.generate_filename = orig_image_path model_field.save(name, model_field.file) print "Putting back:", model_field.name
def test_create_filename(self): """ Create a file name for an image based on the model name, the current object's pk, the field name of the model and the current date and time. """ filename = model_and_instance_based_filename('objectname', 1, 'fieldname', 'imgname.jpg') filename_list = filename.split('_') # Test that the filename contains the correct parts self.assertIn('objectname', filename_list) self.assertIn('1', filename_list) self.assertIn('fieldname', filename_list) self.assertEqual('.jpg', filename[-4:])
def do_import(self): """ :return: List; contains fields that have changed """ from . import same_data changes = [] photo_id = self.get_attrib(self.parent_elem, akvo_ns('photo-id'), 'current_image') current_image = file_from_zip_archive( self.iati_import_job.iati_xml_file, "out_proj/{}.jpg".format(photo_id)) if current_image: tmp_file = NamedTemporaryFile() for line in current_image.readlines(): tmp_file.write(line) tmp_file.flush() # update current image if it's different from the existing one try: old_file = self.project.current_image.file except (IOError, ValueError): old_file = None new_file = File(tmp_file) if not same_data(old_file, new_file): filename = model_and_instance_based_filename( 'Project', self.project.pk, 'current_image', 'image.jpg') new_file.seek(0) self.project.current_image.save(filename, new_file) changes += ['current_image'] current_image_caption = self.get_attrib(self.parent_elem, akvo_ns('image-caption'), 'current_image_caption') if current_image_caption: changes += self.update_project_field('current_image_caption', current_image_caption) current_image_credit = self.get_attrib(self.parent_elem, akvo_ns('photo-credit'), 'current_image_credit') if current_image_credit: changes += self.update_project_field('current_image_credit', current_image_credit) return changes
def import_images(image_dir, photos): outsys("\nRunning {}() ".format(who_am_i())) for image_name in os.listdir(image_dir): photo_id, ext = splitext(image_name) if ext.lower() in ['.png', '.jpg', '.jpeg', '.gif']: try: internal_id = photos.get( photo_id, {'internal_project_id': None})['internal_project_id'] project = Project.objects.get( partnerships__internal_id=internal_id) filename = model_and_instance_based_filename( 'Project', project.pk, 'current_image', image_name) with open(os.path.join(image_dir, image_name), 'rb') as f: image_data = f.read() image_temp = NamedTemporaryFile(delete=True) image_temp.write(image_data) image_temp.flush() project.current_image.save(filename, File(image_temp), save=True) f.close() project.current_image_caption = photos.get( photo_id, {'image_caption': ''})['image_caption'] project.current_image_credit = photos.get( photo_id, {'image_credit': ''})['image_credit'] project.save() log( u"Uploaded image to project {pk}", dict(internal_id=internal_id, pk=project.pk, event=ACTION_SET_IMAGE)) outsys(".") except Exception, e: log( u"Upload failed. internal_id: {internal_id} Exception class: {extra}", dict(internal_id=internal_id, event=ERROR_IMAGE_UPLOAD, extra=e.__class__), ) outsys("*")
def do_import(self): """ :return: List; contains fields that have changed """ from . import same_data changes = [] photo_id = self.get_attrib(self.parent_elem, akvo_ns('photo-id'), 'current_image') current_image = file_from_zip_archive( self.iati_import_job.iati_xml_file, "out_proj/{}.jpg".format(photo_id)) if current_image: tmp_file = NamedTemporaryFile() for line in current_image.readlines(): tmp_file.write(line) tmp_file.flush() # update current image if it's different from the existing one try: old_file = self.project.current_image.file except (IOError, ValueError): old_file = None new_file = File(tmp_file) if not same_data(old_file, new_file): filename = model_and_instance_based_filename( 'Project', self.project.pk, 'current_image', 'image.jpg') new_file.seek(0) self.project.current_image.save(filename, new_file) changes += ['current_image'] current_image_caption = self.get_attrib( self.parent_elem, akvo_ns('image-caption'), 'current_image_caption') if current_image_caption: changes += self.update_project_field('current_image_caption', current_image_caption) current_image_credit = self.get_attrib( self.parent_elem, akvo_ns('photo-credit'), 'current_image_credit') if current_image_credit: changes += self.update_project_field('current_image_credit', current_image_credit) return changes
def import_images(image_dir, photos): outsys("\nRunning {}() ".format(who_am_i())) for image_name in os.listdir(image_dir): photo_id, ext = splitext(image_name) if ext.lower() in ['.png', '.jpg', '.jpeg', '.gif']: try: internal_id = photos.get( photo_id, {'internal_project_id': None} )['internal_project_id'] project = Project.objects.get( partnerships__internal_id=internal_id ) filename = model_and_instance_based_filename( 'Project', project.pk, 'current_image', image_name ) with open(os.path.join(image_dir, image_name), 'rb') as f: image_data = f.read() image_temp = NamedTemporaryFile(delete=True) image_temp.write(image_data) image_temp.flush() project.current_image.save(filename, File(image_temp), save=True) f.close() project.current_image_caption = photos.get( photo_id, {'image_caption': ''} )['image_caption'] project.current_image_credit = photos.get( photo_id, {'image_credit': ''} )['image_credit'] project.save() log( u"Uploaded image to project {pk}", dict(internal_id=internal_id, pk=project.pk, event=ACTION_SET_IMAGE)) outsys(".") except Exception, e: log( u"Upload failed. internal_id: {internal_id} Exception class: {extra}", dict(internal_id=internal_id, event=ERROR_IMAGE_UPLOAD, extra=e.__class__), ) outsys("*")
def set_logo(self, organisation, identifier): """ Get the organisation's logo from the Cordaid zip archive """ logo = file_from_zip_archive( self.iati_import_job.iati_xml_file, "out_rltn/{}.jpg".format(identifier)) if logo: tmp_file = NamedTemporaryFile() for line in logo.readlines(): tmp_file.write(line) tmp_file.flush() # update the logo if it's different from the existing one try: old_file = organisation.logo.file except (IOError, ValueError): old_file = None new_file = File(tmp_file) if not same_data(old_file, new_file): filename = model_and_instance_based_filename( 'Organisation', organisation.pk, 'logo', 'image.jpg') new_file.seek(0) organisation.logo.save(filename, new_file) return 'logo' return None
def organisation_logo(org_etree, internal_id, org): logo_file = glob.glob( os.path.join( CORDAID_LOGOS_DIR, u"{logo_id}.*".format(logo_id=text_from_xpath(org_etree, 'logo_id')) ) ) if len(logo_file) == 1: logo_filename = basename(logo_file[0]) _, extension = splitext(logo_filename) if extension.lower() in (".png", ".jpg", ".jpeg", ".gif"): filename = model_and_instance_based_filename( "Organisation", org.pk, "logo", logo_filename ) with open(os.path.join(CORDAID_LOGOS_DIR, logo_filename), "rb") as f: logo_data = f.read() logo_tmp = NamedTemporaryFile(delete=True) logo_tmp.write(logo_data) logo_tmp.flush() org.logo.save( filename, File(logo_tmp), save=True ) log( u" Added logo {extra} to org {pk}, ", dict( log_type=LOG_ORGANISATIONS, internal_id=internal_id, pk=org.pk, label=org.name, event=ACTION_SET_IMAGE, extra= filename, ) )
def _current_image(self): image = ImageImporter(self.activity.current_image()) try: image.get_image() except Exception, e: log( "Error trying to fetch image to project. Image URL: {extra}", dict(rsr_id=self.activity.rsr_id(), internal_id=self.activity.internal_id(), iati_id=self.activity.iati_id(), event=ERROR_IMAGE_UPLOAD, extra=self.activity.current_image())) if image.image: filename = model_and_instance_based_filename( 'Project', self.project.pk, 'current_image', image.filename) image_temp = NamedTemporaryFile(delete=True) image_temp.write(image.image) image_temp.flush() self.project.current_image.save(filename, File(image_temp), save=True) log( "Save project image: {extra}", dict(rsr_id=self.activity.rsr_id(), internal_id=self.activity.internal_id(), iati_id=self.activity.iati_id(), event=ACTION_SET_IMAGE, extra=filename)) else: log(
image.get_image() except Exception, e: log( "Error trying to fetch image to project. Image URL: {extra}", dict( rsr_id=self.activity.rsr_id(), internal_id=self.activity.internal_id(), iati_id=self.activity.iati_id(), event=ERROR_IMAGE_UPLOAD, extra=self.activity.current_image() ) ) if image.image: filename = model_and_instance_based_filename( 'Project', self.project.pk, 'current_image', image.filename ) image_temp = NamedTemporaryFile(delete=True) image_temp.write(image.image) image_temp.flush() self.project.current_image.save(filename, File(image_temp), save=True) log( "Save project image: {extra}", dict( rsr_id=self.activity.rsr_id(), internal_id=self.activity.internal_id(), iati_id=self.activity.iati_id(), event=ACTION_SET_IMAGE, extra=filename ) )