def test_url():
domain = 'http://via.placeholder.com'
urls = {
domain + '/500x500.png': {'type': 'png', 'width': 500, 'height': 500},
domain + '/500x500.jpg': {'type': 'jpg', 'width': 500, 'height': 500},}
for url, expected in urls.items():
assert imgspy.info(urllib.request.urlopen(url)) == expected
def img():
image = imgspy.info(IMG_URL)
return {
'type': image['type'],
'x_start': 0,
'y_start': 0,
'x_end': image['width'],
'y_end': image['height'],
}
def save_image(self, img_src):
image_info = imgspy.info(img_src)
new_name = self.random_str() + '.' + image_info['type']
cur_date = time.strftime('%Y%m%d', time.localtime())
new_path = './download/image/' + cur_date + '/'
self.createPath(new_path)
# ms = MSSQL(host="localhost", user="******", pwd="123456", db="spider")
# ms.ExecNonQuery("""INSERT INTO image(name,source) VALUES ("%s","%s")""" % (new_name, img_src))
img_response = requests.get(img_src)
with open(new_path + new_name, 'wb') as file:
file.write(img_response.content)
print(img_src, ' --保存成功！')
def test_samples():
for filepath in glob.glob(os.path.join(BASEDIR, 'fixtures/sample*')):
filename = os.path.basename(filepath)
match = re.match(FORMAT, filename)
expected = {
'type': match.group('format'),
'width': int(match.group('width')),
'height': int(match.group('height'))}
actual = imgspy.info(open(filepath, 'rb'))
assert isinstance(actual, dict), filename
actual_subset = {k: v for k, v in actual.items() if k in expected}
assert actual_subset == expected, filename
def get_image(submission):
# image media
submission_image = None
submission_images = []
lazy_load_start_time = time.time()
if is_valid_thumbnail(submission.thumbnail):
# print('{0} thumbnail "{1}"'.format(submission.title, submission.thumbnail))
submission_image = submission.thumbnail
# print('Using thumbnail for image preview: {0}'.format(image_preview))
if hasattr(submission, 'preview'):
# print('Time to lazy load: {0}'.format(time.time() - lazy_load_start_time))
# submission has preview image
preview_resolutions = submission.preview['images'][0]['resolutions']
# print(len(preview_resolutions), preview_resolutions)
if len(preview_resolutions) > 0:
# cannot trust reddit to report accurate resolutions
# images are often larger than reported
# network request is the main bottleneck
# request + PIL is slower than headers
submission_image = preview_resolutions[0]['url']
# go from largest to smallest because most preview images are small
for preview_image in reversed(preview_resolutions):
response = None
tries = 0
while response is None and tries < 10:
try:
response = imgspy.info(preview_image['url'])
except Exception as e:
print(
'Exception retrieving image information for submission {0} "{1}" with image preview {2}: {3}'.format(
submission.id, submission.title, preview_image, e))
tries += 1
if response is not None:
width, height = response['width'], response['height']
dimensions_ratio = width / height
if dimensions_ratio <= config['media'].getfloat('max_width_to_height_ratio') and width <= config[
'media'].getfloat('max_width') and height <= config['media'].getfloat('max_height'):
# print('preview images stopped at ({0}, {1}) with ratio {2}'.format(width, height, dimensions_ratio))
submission_image = preview_image['url']
break
if submission_image is None:
submission_image = submission.preview['images'][0]['source']['url']
else:
submission_image = submission.preview['images'][0]['source']['url']
# dimensions_ratio = image['width'] / image['height']
# if dimensions_ratio > MAX_WIDTH_TO_HEIGHT_RATIO or image['width'] > MAX_WIDTH or image[
# 'height'] > MAX_HEIGHT:
# print('preview images stopped at ({0}, {1}) with ratio {2}'.format(image['width'], image['height'],
# dimensions_ratio))
# break
# else:
# print(image)
# image_preview = image['url']
# else:
# source_image = submission.preview['images'][0]['source']
# if (source_image['width'] > MAX_WIDTH or source_image['height'] > MAX_HEIGHT) and is_valid_thumbnail(submission.thumbnail):
# image_preview = submission.thumbnail
# print('Using thumbnail for image preview: {0}'.format(image_preview))
# else:
# print('preview image (width, height): ({0}, {1}), url: {2}'.format(source_image['width'], source_image['height'], source_image['url']))
# image_preview = source_image['url']
if hasattr(submission, 'is_gallery') and submission.is_gallery:
# Reddit now has image galleries
if hasattr(submission, 'gallery_data') and submission.gallery_data is not None: # check if gallery is deleted
gallery_images = submission.gallery_data['items']
for gallery_image in gallery_images:
media_id = gallery_image['media_id']
# use media_metadata attribute
metadata = submission.media_metadata[media_id]
# e = media type, m = extension, s = preview image, p = preview images, id = id
# just use the first image's preview for now
gallery_previews = metadata['p']
# print(gallery_previews)
preview_index = min(1, len(gallery_previews) - 1) # sometimes the images are too small
submission_images.append(gallery_previews[preview_index]['u'])
else:
print(f'Submission {submission.id} gallery is deleted')
else:
submission_images.append(submission_image)
return submission_images
def user_profile_uploadphoto(token, img_url, x_start, y_start, x_end, y_end):
'''
user_profile_uploadphoto
Args:
token: authorises user.
img_url: The url of the source image is provided.
x_start, y_start, x_end, y_end: Image dimensions are specified by the user.
Returns:
returns {}
Raises:
AccessError when the token is invalid
InputError when img_url returns an HTTP status other than 200.
InputError when any of the parameters are not within the dimensions of the image at the URL.
InputError when the image uploaded is not a JPG
'''
user = token_validator(token)
# Verify that the img_url is valid
r = requests.head(img_url)
valid_url = r.status_code == 200
if not valid_url:
raise InputError(
"The image url returned a HTTP status other than 200.")
# Verify that the image has JPG format
image = imgspy.info(img_url)
if image['type'] == None or image['type'] != 'jpg':
raise InputError("Image uploaded is not a JPG.")
# Verify that the parameters are within image dimensions
# {'type': 'jpg', 'width': 2370, 'height': 1927}
img_width = range(0, image['width'] + 1)
img_height = range(0, image['height'] + 1)
if (x_start in img_width and x_end in img_width and y_start in img_height
and y_end in img_height):
print("valid")
else:
raise InputError("x_start, y_start, x_end, y_end are not within \
the dimensions of the image at the URL.")
# Generate unique image_id using uuid
profile_img_url = str(uuid.uuid4())
# Download and retrieve the image
profile_image = f"{IMG_LOCATION}/{profile_img_url}.jpg"
# Move the image into the appropriate directory
urllib.request.urlretrieve(img_url, profile_image)
# Crop the image
imageObject = Image.open(profile_image)
cropped = imageObject.crop((x_start, y_start, x_end, y_end))
cropped.save(profile_image)
# Add image url to the user's data
for users in data['users']:
if users['u_id'] == user['u_id']:
users['profile_img_url'] = profile_img_url + '.jpg'
return {}
round(float(weight_calc[0]) * 453.592, 0)
if weight_calc[1] == 'g':
prod_dict[row[0].value]['weigth'] = round(float(weight_calc[0]), 0)
if weight_calc[1] == 'kg' or weight_calc[1] == 'Kg':
prod_dict[row[0].value]['weigth'] = \
round(float(weight_calc[0]) * 1000, 0)
photos = str(row[5].value).split(',')
prod_dict[row[0].value]['photos'] = []
counter = 0
for i in photos:
url, sep, tail = i.partition('%')
if counter < 10:
# image url resist and min size test
try:
w = imgspy.info(url)['width']
h = imgspy.info(url)['height']
print(w, 'x', h)
if int(w) >= 300 and int(h) >= 100:
prod_dict[row[0].value]['photos'].append(url)
counter += 1
print('!!! ', url)
except:
print('bad url', url)
for i, v in prod_dict.items():
print(i)
Product(**v).save()
text = dict(
title='Greetings',
def write_manifests(context, item):
identifier = item["identifier"]
if not file_exists("manifest",
identifier) or context.mode_def.name == "test":
mapping = item["mapping"]
item = item["row"]
context.log.info("Processing: {0}".format(str(identifier)))
# Get image sizes
img_info = imgspy.info(
BUCKET + "/iiif/{0}/full/max/0/default.jpg".format(identifier))
imgwidth, imgheight = img_info["width"], img_info["height"]
# Logo
logo = iiifpapi3.logo()
logo.set_id(
"https://aws1.discourse-cdn.com/free1/uploads/imaginerio/original/1X/8c4f71106b4c8191ffdcafb4edeedb6f6f58b482.png"
)
logo.set_format("image/png")
logo.set_hightwidth(164, 708)
# Header
manifest = iiifpapi3.Manifest()
manifest.set_id(extendbase_url="{0}/manifest.json".format(
str(identifier).replace(" ", "_")))
manifest.add_label("pt-BR", item["Title"])
# Metadata
def map_wikidata(values_en, label_en, label_pt):
en = []
pt = []
if not values_en:
return None
else:
for value_en in values_en.split("||"):
try:
url = "http://wikidata.org/wiki/{0}".format(
mapping.loc[value_en, "Wiki ID"])
value_pt = mapping.loc[value_en, "Label:pt"]
en.append(
'<a class="uri-value-link" target="_blank" href="{0}">{1}</a>'
.format(url, value_en))
pt.append(
'<a class="uri-value-link" target="_blank" href="{0}">{1}</a>'
.format(url, value_pt))
except:
en.append(value_en)
pt.append(value_en)
d = {"en": en, "pt": pt}
return {
"label_en": [label_en],
"label_pt": [label_pt],
"value_en": d["en"],
"value_pt": d["pt"],
}
def set_metadata_field(manifest, field):
if field:
if "value" in field:
value = {
"none": field["value"]
} if any(field["value"]) else None
else:
value = ({
"en": field["value_en"],
"pt-BR": field["value_pt"]
} if any(field["value_pt"]) else None)
if value:
manifest.add_metadata(
entry={
"label": {
"en": field["label_en"],
"pt-BR": field["label_pt"],
},
"value": value,
})
else:
pass
# Values
title = {
"label_en": ["Title"],
"label_pt": ["Título"],
"value": [item["Title"]],
}
description = {
"label_en": ["Description"],
"label_pt": ["Descrição"],
"value_en":
[item["Description (English)"]] if item["Description (English)"]
else [item["Description (Portuguese)"]],
"value_pt": [item["Description (Portuguese)"]]
if item["Description (Portuguese)"] else
[item["Description (English)"]],
}
creator = {
"label_en": ["Creator"],
"label_pt": ["Autor"],
"value": [item["Creator"]],
}
date = {
"label_en": ["Date"],
"label_pt": ["Data"],
"value": [str(item["Date"])],
}
try:
depicts = {
"label_en": ["Depicts"],
"label_pt": ["Retrata"],
"value": [
'<a class="uri-value-link" target="_blank" href="{0}">{1}</a>'
.format(*depicts.split(" ", maxsplit=1))
for depicts in item["Depicts"].split("||")
],
}
except:
depicts = None
type = map_wikidata(item["Type"], "Type", "Tipo")
materials = map_wikidata(item["Materials"], "Materials", "Materiais")
fabrication_method = map_wikidata(item["Fabrication Method"],
"Fabrication Method",
"Método de Fabricação")
width = {
"label_en": ["Width (mm)"],
"label_pt": ["Largura (mm)"],
"value": [str(item["Width (mm)"])],
}
height = {
"label_en": ["Height (mm)"],
"label_pt": ["Altura (mm)"],
"value": [str(item["Height (mm)"])],
}
fields = [
title,
description,
creator,
date,
depicts,
type,
materials,
fabrication_method,
width,
height,
]
for field in fields:
set_metadata_field(manifest, field)
# Rights & Attribution
if description["value_en"]:
manifest.add_summary(language="en",
text=description["value_en"][0])
manifest.add_summary(language="pt-BR",
text=description["value_pt"][0])
manifest.add_requiredStatement(
label="Hosting",
value="Hosted by imagineRio",
language_l="en",
language_v="en",
)
manifest.add_requiredStatement(
label="Hospedagem",
value="Hospedado por imagineRio",
language_l="pt-BR",
language_v="pt-BR",
)
if item["Attribution"]:
manifest.add_requiredStatement(
label="Attribution",
value=item["Attribution"],
language_v="en",
language_l="en",
)
manifest.add_requiredStatement(
label="Atribuição",
value=item["Attribution"],
language_v="pt-BR",
language_l="pt-BR",
)
if item["License"] and item["License"].startswith("http"):
manifest.set_rights(item["License"])
elif item["Rights"] and item["Rights"].startswith("http"):
manifest.set_rights(item["Rights"])
else:
manifest.set_rights("http://rightsstatements.org/vocab/CNE/1.0/")
# Thumbnail
thumbnail = iiifpapi3.thumbnail()
thumb_width = int(imgwidth / 16)
thumb_height = int(imgheight / 16)
thumbnail.set_id(
extendbase_url="{0}/full/{1},{2}/0/default.jpg".format(
str(identifier).replace(" ", "_"), thumb_width, thumb_height))
thumbnail.set_hightwidth(thumb_height, thumb_width)
manifest.add_thumbnail(thumbnailobj=thumbnail)
# Homepage
item_homepage = iiifpapi3.homepage()
homepage_id = item["Source URL"] if item[
"Source URL"] else "https://null"
try:
item_homepage.set_id(objid=homepage_id)
except:
url_parts = list(urlsplit(homepage_id))
url_parts[2:5] = ["", "", ""]
new_url = urlunsplit(url_parts)
item_homepage.set_id(objid=new_url)
homepage_label = item["Source"] if item["Source"] else "imagineRio"
item_homepage.add_label(language="none", text=homepage_label)
item_homepage.set_type("Text")
item_homepage.set_format("text/html")
manifest.add_homepage(item_homepage)
# See Also
imaginerio_seealso = iiifpapi3.seeAlso()
imaginerio_seealso.set_id(
objid="https://www.imaginerio.org/map#{0}".format(identifier))
imaginerio_seealso.add_label(language="none", text="imagineRio")
imaginerio_seealso.set_type("Text")
manifest.add_seeAlso(imaginerio_seealso)
if item["Wikidata ID"]:
wikidata_seealso = iiifpapi3.seeAlso()
wikidata_seealso.set_id(objid="https://www.wikidata.org/wiki/{0}".
format(item["Wikidata ID"]))
wikidata_seealso.add_label(language="none", text="Wikidata")
wikidata_seealso.set_type("Text")
# wikidata_seealso.set_format("text/html")
manifest.add_seeAlso(wikidata_seealso)
if item["Smapshot ID"]:
smapshot_seealso = iiifpapi3.seeAlso()
smapshot_seealso.set_id(
objid="https://smapshot.heig-vd.ch/visit/{0}".format(
item["Smapshot ID"]))
smapshot_seealso.add_label(language="none", text="Smapshot")
smapshot_seealso.set_type("Text")
# smapshot_seealso.set_format("text/html")
manifest.add_seeAlso(smapshot_seealso)
# Provider
item_provider = iiifpapi3.provider()
item_provider.set_id("https://imaginerio.org/")
item_provider.add_label("en", "imagineRio")
item_provider.add_label("pt-BR", "imagineRio")
item_provider.add_logo(logo)
item_provider.add_homepage(item_homepage)
manifest.add_provider(item_provider)
# Canvas
canvas = manifest.add_canvas_to_items()
canvas.set_id(extendbase_url="canvas/p1")
canvas.set_height(imgheight)
canvas.set_width(imgwidth)
canvas.add_label(language="none", text=identifier)
annopage = canvas.add_annotationpage_to_items()
annopage.set_id(extendbase_url="annotation-page/p1")
annotation = annopage.add_annotation_to_items(target=canvas.id)
annotation.set_id(extendbase_url="annotation/p1")
annotation.set_motivation("painting")
annotation.body.set_id(
extendbase_url="{0}/full/max/0/default.jpg".format(identifier))
annotation.body.set_type("Image")
annotation.body.set_format("image/jpeg")
annotation.body.set_width(imgwidth)
annotation.body.set_height(imgheight)
s = annotation.body.add_service()
s.set_id(extendbase_url="{0}/".format(identifier))
s.set_type("ImageService3")
s.set_profile("level0")
# setup items to send to io_managers
manifest_path = "iiif/{0}/manifest.json".format((identifier))
manifest_obj = manifest.json_dumps(
dumps_errors=False,
ensure_ascii=False,
context="http://iiif.io/api/presentation/3/context.json",
)
data = [{"data": manifest_obj, "key": manifest_path, "type": "json"}]
# Logo
logo = iiifpapi3.logo()
logo.set_id(
"https://aws1.discourse-cdn.com/free1/uploads/imaginerio/original/1X/8c4f71106b4c8191ffdcafb4edeedb6f6f58b482.png"
)
logo.set_format("image/png")
logo.set_hightwidth(164, 708)
for collection_name in item["Collections"].lower().split("||"):
collection_path = "iiif/collection/{0}.json".format(
collection_name)
try:
if context.mode_def.name == "test":
collection = read_API3_json(collection_path)
else:
endpoint = BUCKET + "/" + collection_path
retry_strategy = Retry(
total=3,
status_forcelist=[429, 500, 502, 503, 504],
method_whitelist=["HEAD", "GET", "OPTIONS"],
)
adapter = HTTPAdapter(max_retries=retry_strategy)
http = requests.Session()
http.mount("https://", adapter)
http.mount("http://", adapter)
collection_data = http.get(endpoint).json()
collection = read_API3_json_dict(collection_data)
# Deal with duplicates
for item in collection.items:
if "{0}/manifest".format(identifier) in item.id:
collection.items.remove(item)
except Exception as e:
print(e)
print("Couldn't find collection: {0}. Creating...".format(
collection_name))
# Homepage
collection_homepage = iiifpapi3.homepage()
collection_homepage.set_id("https://imaginerio.org")
collection_homepage.set_type("Text")
collection_homepage.add_label("none", "imagineRio")
collection_homepage.set_format("text/html")
# Provider
collection_provider = iiifpapi3.provider()
collection_provider.set_id("https://imaginerio.org")
collection_provider.add_label("en", "imagineRio")
collection_provider.add_label("pt-BR", "imagineRio")
collection_provider.add_logo(logo)
collection_provider.add_homepage(collection_homepage)
# Collection manifest
collection = iiifpapi3.Collection()
collection.set_id(extendbase_url="collection/{0}.json".format(
collection_name))
collection.add_label("en", collection_name)
collection.add_label("pt-BR", collection_name)
collection.add_requiredStatement(
label="Attribution",
value="Hosted by imagineRio",
language_l="en",
language_v="en",
)
collection.add_requiredStatement(
label="Attribution",
value="Hospedado por imagineRio",
language_l="pt-BR",
language_v="pt-BR",
)
collection.set_rights(
"http://rightsstatements.org/vocab/CNE/1.0/")
thumbnailobj = iiifpapi3.thumbnail()
if collection_name == "all" or collection_name == "views":
thumb_id = "0071824cx001-01/full/295,221/0/default.jpg"
h, w = 221, 295
elif collection_name == "plans":
thumb_id = "10639297/full/259,356/0/default.jpg"
h, w = 356, 259
elif collection_name == "maps":
thumb_id = "10643717/full/512,259/0/default.jpg"
h, w = 259, 512
elif collection_name == "aerials":
thumb_id = "24879867/full/394,260/0/default.jpg"
h, w = 260, 394
elif collection_name == "mare":
thumb_id = "31770323/full/188,125/0/default.jpg"
h, w = 125, 188
thumbnailobj.set_id(extendbase_url=thumb_id)
thumbnailobj.set_hightwidth(h, w)
collection.add_thumbnail(thumbnailobj=thumbnailobj)
collection.add_provider(collection_provider)
collection.add_manifest_to_items(manifest)
# setup items to send to io_managers
collection = collection.json_dumps(
dumps_errors=False,
ensure_ascii=False,
context="http://iiif.io/api/presentation/3/context.json",
)
collections = {}
collections["data"] = collection
collections["key"] = collection_path
collections["type"] = "json"
data.append(collections)
print("Collection updated: {0}".format(collection_name))
else:
pass
return data
def test_datastr():
data = textwrap.dedent('''data:image/png;base64,
iVBORw0KGgoAAAANSUhEUgAAAAIAAAABCAYAAAD0In+
KAAAAD0lEQVR42mNk+M9QzwAEAAmGAYCF+yOnAAAAAElFTkSuQmCC''')
assert imgspy.info(data) == {'type': 'png', 'width': 2, 'height': 1}
