def fetch_and_create_image(self, url, image_title):
'''
fetches, creates image object
returns tuple with Image object and context dictionary containing
request URL
'''
context = {
"file_url": url,
"foreign_title": image_title,
}
try:
image_file = requests.get(url)
local_image = Image(
title=image_title,
file=ImageFile(
BytesIO(image_file.content),
name=image_title
)
)
local_image.save()
return (local_image, context)
except Exception as e:
context.update({
"exception": e,
})
raise ImageCreationFailed(context, None)
def get_body(message):
if message["attachment_media_type"] == "image":
im = ""
title = message["attachment_media_object"]["filename"]
try:
im = Image.objects.get(title=title).id
except Exception:
http_res = requests.get(message["attachment_uri"])
image_file = ImageFile(BytesIO(http_res.content),
name=title)
image = Image(title=title, file=image_file)
image.save()
im = image.id
block = blocks.StructBlock([("message", blocks.TextBlock()),
("image", ImageChooserBlock())])
block_value = block.to_python({
"message": message["answer"],
"image": im
})
return block_value
else:
block = blocks.StructBlock([
("message", blocks.TextBlock()),
])
block_value = block.to_python({"message": message["answer"]})
return block_value
def process_content_image(self, content):
self.stdout.write('\tGenerate and replace entry content images....')
if content:
root = lxml.html.fromstring(content)
for img_node in root.iter('img'):
parent_node = img_node.getparent()
if 'wp-content' in img_node.attrib[
'src'] or 'files' in img_node.attrib['src']:
image = self._import_image(img_node.attrib['src'])
if image:
title = img_node.attrib.get(
'title') or img_node.attrib.get('alt')
new_image = WagtailImage(file=File(file=image),
title=title)
new_image.save()
if parent_node.tag == 'a':
parent_node.addnext(
ET.XML(self._image_to_embed(new_image)))
parent_node.drop_tree()
else:
parent_node.append(
ET.XML(self._image_to_embed(new_image)))
img_node.drop_tag()
content = ET.tostring(root)
return content
def handle(self, *args, **options):
# Get the only instance of Magazine Index Page
magazine_index_page = MagazineIndexPage.objects.get()
with open(options["file"]) as import_file:
issues = csv.DictReader(import_file)
for issue in issues:
response = requests.get(issue["cover_image_url"])
image_file = BytesIO(response.content)
image = Image(
title=issue["title"] + " cover image",
file=ImageFile(image_file,
name=issue["cover_image_file_name"]),
)
image.save()
import_issue = MagazineIssue(
title=issue["title"],
publication_date=issue["publication_date"],
first_published_at=issue["publication_date"],
issue_number=issue["issue_number"],
cover_image=image,
)
# Add issue to site page hiererchy
magazine_index_page.add_child(instance=import_issue)
magazine_index_page.save()
self.stdout.write("All done!")
def create_images_from_urls_in_content(self, body):
"""create Image objects and transfer image files to media root"""
from bs4 import BeautifulSoup
soup = BeautifulSoup(body, "html5lib")
for img in soup.findAll("img"):
if "width" in img:
width = img["width"]
if "height" in img:
height = img["height"]
else:
width = 100
height = 100
_, file_name = os.path.split(img["src"])
if not img["src"]:
continue # Blank image
if img["src"].startswith("data:"):
continue # Embedded image
resp = requests.get(self.prepare_url(img["src"]), stream=True)
if resp.status_code != requests.codes.ok:
print("Unable to import " + img["src"])
continue
fp = BytesIO()
fp.write(resp.content)
image = Image(title=file_name, width=width, height=height)
image.file.save(file_name, File(fp))
image.save()
if img.has_attr("srcset"):
img["srcset"] = ""
new_url = image.get_rendition("original").url
img["src"] = new_url
soup.body.hidden = True
return soup.body
def set_featured_media(self, page: BlogPage, post):
featured_media_id: int = post.get("featured_media")
if not featured_media_id:
return
featured_medias: list = post["_embedded"].get("wp:featuredmedia")
if featured_medias is None:
return
for feature_media in featured_medias:
if feature_media.get("id") == featured_media_id:
source_url = feature_media["source_url"]
try: # Wordpress 5.3 API nests title in "rendered"
title = feature_media["title"]["rendered"]
except TypeError: # Fallback for older (or newer?) wordpress
title = feature_media["title"]
details = feature_media["media_details"]
resp = requests.get(source_url, stream=True)
if resp.status_code != requests.codes.ok:
print("Unable to import " + source_url)
continue
fp = BytesIO()
fp.write(resp.content)
image = Image(title=title,
width=details["width"],
height=details["height"])
image.file.save(details["file"], File(fp))
image.save()
page.header_image = image
def get_image(base_url, image_id):
# TODO: guard against non-existent images
image_attributes = get_image_attributes(base_url, image_id)
image_file = requests.get(image_attributes["file"])
image = Image(title=image_attributes["title"],
file=ImageFile(BytesIO(image_file.content),
name=image_attributes["title"]))
image.save()
return image
def get_image(base_url, image_id):
# TODO: guard against non-existent images
image_attributes = get_image_attributes(base_url, image_id)
image_file = requests.get(image_attributes["file"])
image = Image(
title=image_attributes["title"],
file=ImageFile(
BytesIO(image_file.content), name=image_attributes["title"]
)
)
image.save()
return image
def create_image(self, text):
# create a Wagtail image with a random coloured background
# and a text overlay
colours = tuple(random.sample(range(255), 3))
image = Image.new("RGB", (600, 400), color=colours)
d = ImageDraw.Draw(image)
d.text((10, 10), text, fill=(255, 255, 255))
f = BytesIO()
image.save(f, format="png")
filename = text.replace(" ", "-").lower() + "-%s-%s-%s.fake" % colours
wagtail_image = WagtailImage(title=text, file=ImageFile(f, name=filename))
wagtail_image.save()
return wagtail_image
def parse_media_blocks(media_urls):
media_blocks = []
for url in media_urls.split(", "):
domain = urlparse(url).netloc
if domain in ["vimeo.com", "www.youtube.com"]:
embed = get_embed(url)
embed_tuple = ("embed", embed)
media_blocks.append(embed_tuple)
else:
# The default should be to fetch a PDF or image file (i.e. from westernfriend.org)
response = requests.get(url)
content_type = response.headers["content-type"]
file_name = url.split("/")[-1]
file_bytes = BytesIO(response.content)
if content_type == "application/pdf":
# Create file
document_file = File(file_bytes, name=file_name)
document = Document(
title=file_name,
file=document_file,
)
document.save()
document_link_block = ("document", document)
media_blocks.append(document_link_block)
elif content_type in ["image/jpeg", "image/png"]:
# create image
image_file = ImageFile(file_bytes, name=file_name)
image = Image(
title=file_name,
file=image_file,
)
image.save()
image_block = ("image", image)
media_blocks.append(image_block)
else:
print(url)
print(content_type)
print("-----")
return media_blocks
def handle(self, *args, **options):
self.stdout.write(self.style.SUCCESS("Importing images"))
if not options["from_dir"].endswith("/"):
options["from_dir"] = options["from_dir"] + "/"
for filepath in sorted(glob(options["from_dir"] + "*-90x90.png")):
with open(filepath, "rb") as image_file:
name = filepath.split("/")[-1][:-10]
image = Image(title=name)
image.file = ImageFile(file=image_file, name=name + ".png")
image.file_size = image.file.size
image.file.seek(0)
image._set_file_hash(image.file.read())
image.file.seek(0)
# Reindex the image to make sure all tags are indexed
search_index.insert_or_update_object(image)
image.save()
image.tags.add("illustration")
self.stdout.write(
self.style.SUCCESS(f"{image.pk},{image.title}"))
self.stdout.write(self.style.SUCCESS("Importing images finished"))
def _set_image(self, obj, attr_name, folder_path, img_path):
"""helper to set images for objects"""
img_path = folder_path.joinpath(img_path)
# create and set the file if it does not yet exist
qs = Image.objects.filter(title=img_path.name)
if not qs.exists():
with open(img_path, "rb") as f:
# setting name= is important. otherwise it uses the entire file path as
# name, which leaks server filesystem structure to the outside.
image_file = File(f, name=img_path.stem)
image = Image(title=img_path.name, file=image_file.open())
image.save()
else:
image = qs[0]
setattr(obj, attr_name, image)
obj.save()
def import_header_image(self, entry, items, image_id):
self.stdout.write('\tImport header images....')
for item in items:
post_type = item.find(u'{{{0:s}}}post_type'.format(
self.WP_NS)).text
if post_type == 'attachment' and item.find(
u'{{{0:s}}}post_id'.format(self.WP_NS)).text == image_id:
title = item.find('title').text
image_url = item.find(u'{{{0:s}}}attachment_url'.format(
self.WP_NS)).text
image = self._import_image(image_url)
if image:
new_image = WagtailImage(file=File(file=image),
title=title)
new_image.save()
entry.header_image = new_image
entry.save()
def mock(title):
"""
Mock a ImageBlock
:param title: String
:return: Stream content
"""
url = str.strip(constants.URL_IMAGE_MOCK_1)
filename = "%s.png" % title
try:
ret = Image.objects.get(title=title)
except Image.DoesNotExist:
response = requests.get(url)
file = ImageFile(BytesIO(response.content), name=filename)
ret = Image(title=title, file=file)
ret.save()
return {'type': 'image', 'value': ret.id}
def create_image(self, image_file, title, filename, tag):
image = Image(title=title)
image.file = ImageFile(file=image_file, name=filename)
image.file_size = image.file.size
image.file.seek(0)
image._set_file_hash(image.file.read())
image.file.seek(0)
# Reindex the image to make sure all tags are indexed
search_index.insert_or_update_object(image)
image.save()
image.tags.add(tag)
return image
def update_thumbnail(self):
generated_file = False
if self.video.thumbnail:
file = open(self.video.thumbnail.path, 'rb')
file = File(file)
else:
clip = cv2.VideoCapture(self.video.file.path)
ret, frame = clip.read()
generated_file = 'thumbnail.jpeg'
cv2.imwrite(generated_file, frame)
file = open(generated_file, 'rb')
file = File(file)
thumbnail = Image(title=text_processing.html_to_str(
self.english_title),
file=file)
thumbnail.save()
self.video_thumbnail = thumbnail
self.save()
if generated_file:
os.remove(generated_file)
return thumbnail
def test_norun(self):
image = Image(**image_kwargs)
# Make operation
operation = self.operation_class(*filter_spec.split('-'))
# Make operation recorder
context = DummyImageTransform((image.width, image.height))
# Attempt (and hopefully fail) to run
with self.assertRaises(ValueError):
operation.run(context, image)
def test_cache_key_fill_filter_with_focal_point(self):
image = Image(
width=1000,
height=1000,
focal_point_width=100,
focal_point_height=100,
focal_point_x=500,
focal_point_y=500,
)
fil = Filter(spec='fill-100x100')
cache_key = fil.get_cache_key(image)
self.assertEqual(cache_key, '0bbe3b2f')
def test_norun(self):
image = Image(**image_kwargs)
# Make operation
operation = self.operation_class(*filter_spec.split('-'))
# Make operation recorder
operation_recorder = WillowOperationRecorder(
(image.width, image.height))
# Attempt (and hopefully fail) to run
with self.assertRaises(ValueError):
operation.run(operation_recorder, image, {})
def get_or_create_image(self, row):
image = None
if self.image_path and row['filename']:
path = os.path.join(self.image_path, row['filename'])
if os.path.exists(path):
image_title = re.sub(r'[^\w\.]', '-',
row['filename'].strip()).lower()
hash = get_hash_from_file(path)
image = Image.objects.filter(file__contains=hash).first()
if not image:
new_name = re.sub(r'.*\.', hash + '.', row['filename'])
image = Image(file=ImageFile(File(open(path, 'rb')),
name=new_name),
title=image_title)
image.save()
_print_operation(image, True, 'title')
return image
def test_run(self):
image = Image(**image_kwargs)
# Make operation
operation = self.operation_class(*filter_spec.split('-'))
# Make context
context = DummyImageTransform((image.width, image.height))
# Run
context = operation.run(context, image)
# Check
self.assertEqual(context.operations, expected_output)
def test_run(self):
image = Image(**image_kwargs)
# Make operation
operation = self.operation_class(*filter_spec.split('-'))
# Make operation recorder
operation_recorder = WillowOperationRecorder((image.width, image.height))
# Run
operation.run(operation_recorder, image, {})
# Check
self.assertEqual(operation_recorder.ran_operations, expected_output)
def form_valid(self, form):
image_file = form.cleaned_data['image_file']
# print(image_file.name)
photographer = Photographer(
age_range=form.cleaned_data['age_range'],
gender_category=form.cleaned_data['gender'],
gender_other=form.cleaned_data['gender_other'],
)
photographer.save()
image = Image(title=image_file.name, file=image_file)
image.save()
photo = form.save()
photo.image = image
photo.photographer = photographer
photo.save()
self.request.session['reference_number'] = photo.reference_number
return super().form_valid(form)
def is_media(path: str):
"""
:param path: string path to the file to be checked
:return: media mimetype string / format or None
"""
mimetype = mimetypes.guess_type(path)
if mimetype[0] and mimetype[0].startswith(('image', 'video')):
return mimetype[0]
else:
try:
with Image.open(path) as image:
return image.format
except IOError:
logger.exception("Path was not an image")
return None
def fetch_and_create_image(self, url, image_title):
'''
fetches, creates image object
returns tuple with Image object and context dictionary containing
request URL
'''
context = {
"file_url": url,
"foreign_title": image_title,
}
try:
image_file = requests.get(url)
local_image = Image(title=image_title,
file=ImageFile(BytesIO(image_file.content),
name=image_title))
local_image.save()
return (local_image, context)
except Exception as e:
context.update({
"exception": e,
})
raise ImageCreationFailed(context, None)
def handle(self, *args, **options):
# Get the only instance of Magazine Index Page
magazine_index_page = MagazineIndexPage.objects.get()
with open(options["file"]) as import_file:
issues = csv.DictReader(import_file)
issues_list = list(issues)
for issue in tqdm(issues_list, desc="Issues", unit="row"):
response = requests.get(issue["cover_image_url"])
image_file = BytesIO(response.content)
image = Image(
title=issue["title"] + " cover image",
file=ImageFile(image_file,
name=issue["cover_image_file_name"]),
)
image.save()
publication_date_tz_aware = make_aware(
datetime.strptime(issue["publication_date"], "%Y-%m-%d"))
import_issue = MagazineIssue(
title=issue["title"],
publication_date=publication_date_tz_aware,
first_published_at=publication_date_tz_aware,
issue_number=issue["issue_number"],
cover_image=image,
)
# Add issue to site page hiererchy
magazine_index_page.add_child(instance=import_issue)
magazine_index_page.save()
self.stdout.write("All done!")
def parse_results(self):
media_files = self.results
for r in media_files:
sub_site = r.get('source')
collection_name = SOURCES[sub_site]
collection = Collection.objects.get(name=collection_name)
source_url = r.get('source_url')
media_type = r.get('media_type')
media_name = source_url.split('/')[-1]
response = requests.get(source_url)
title = r.get('title') # if the title id blank it causes an error
if not title:
title = 'No title was available'
if response:
if media_type == 'file': # save to documents
media_file = File(BytesIO(response.content),
name=media_name)
file = Document(title=title,
file=media_file,
collection=collection)
file.save()
file.created_at = r.get('date')
file.save()
elif media_type == 'image': # save to images
image_file = ImageFile(BytesIO(response.content),
name=media_name)
image = Image(title=title,
file=image_file,
collection=collection)
image.save()
image.created_at = r.get('date')
image.save()
else:
sys.stdout.write(
'⚠️ Got no response. Error has been logged importer/log/import_media_files.txt\n'
)
with open('importer/log/import_media_files.txt',
'a') as the_file:
the_file.write('{}\n'.format(r))
if self.next:
time.sleep(self.sleep_between_fetches)
self.fetch_url(self.next)
self.parse_results()
return Document.objects.count() + Image.objects.count(), 0
def create_wagtail_image_from_remote(image_url=None, images_folder='original_images', collection=None):
basename = os.path.basename(image_url)
db_file_field = os.path.join(images_folder, basename).replace('\\', '/')
destination_image = os.path.join(
settings.MEDIA_ROOT,
images_folder,
os.path.basename(image_url)
)
if collection is None:
collection = get_behance_collection()
r = requests.get(image_url)
if Image.objects.filter(file=db_file_field).count() == 0:
if r.status_code == 200:
with open(destination_image, 'wb') as f:
f.write(r.content)
local_image = PILImage.open(destination_image)
width, height = local_image.size
img = Image()
img.file = db_file_field
img.title = basename
img.width = width
img.height = height
img.collection = collection
img.save()
return img
else:
return Image.objects.get(file=db_file_field)
return None
def handle(self, *args, **options):
self.stdout.write(self.style.SUCCESS("Importing images"))
with open(options["speakers_csv"], "r") as file:
for row in DictReader(file):
if not row["filename"]:
self.stdout.write(
self.style.WARNING(
f"====> skipping {row['post_name']}, {row['post_title']}"
))
continue
image_path = os.path.join(options["from_dir"], row["filename"])
with open(image_path, "rb") as image_file:
image = Image(title=row["post_title"])
if row["filename"].lower().endswith(".jpg") or row[
"filename"].lower().endswith(".jpeg"):
image_filename = f"{row['first_name']} {row['last_name']}.jpg"
elif row["filename"].lower().endswith(".png"):
image_filename = f"{row['first_name']} {row['last_name']}.png"
else:
raise ValueError("Unknown file format")
image.file = ImageFile(file=image_file,
name=image_filename)
image.file_size = image.file.size
image.file.seek(0)
image._set_file_hash(image.file.read())
image.file.seek(0)
# Reindex the image to make sure all tags are indexed
search_index.insert_or_update_object(image)
image.save()
image.tags.add("speaker")
speaker = Speaker.objects.get(wordpress_id=row["ID"])
speaker.photo = image
speaker.save()
self.stdout.write(
self.style.SUCCESS(f"{image.pk},{image.title}"))
self.stdout.write(self.style.SUCCESS("Importing images finished"))
def create_image(self, instance):
data = self.collect_data(instance)
image_description = self.get_image_description(instance)
tags = image_description.get('tags') and image_description.pop('tags')
image = Image(**image_description)
snippet_image = self.create_snippet_image(data)
file_name = self.get_file_name()
image.file.save(file_name, snippet_image, save=False)
image.save()
if tags:
image.tags.set(*tags, clear=True)
image.save()
return image
def test_cache_key(self):
image = Image(width=1000, height=1000)
fil = Filter(spec='max-100x100')
cache_key = fil.get_cache_key(image)
self.assertEqual(cache_key, '')
def test_cache_key_fill_filter(self):
image = Image(width=1000, height=1000)
fil = Filter(spec='fill-100x100')
cache_key = fil.get_cache_key(image)
self.assertEqual(cache_key, '2e16d0ba')
