def creatSet(photoSet, setName):
setName = setName.replace('\\',' ')
setName = setName.replace('/',' ')
setName = string.strip(setName)
photos = [] #real photo objects
for p in photoSet:
photos.append(flickr.Photo(id = p))
fset = None
#check if set with the name exists already
for s in existingSets:
if(s.title == setName):
fset= s
logging.debug('tags2set: Found existing set %s' % setName)
# return
break
try:
if(fset == None):
#print photos[0]
#print setName
fset = flickr.Photoset.create(photos[0], setName, 'auto generated by folders2flickr')
logging.debug('tags2set: Creating new set %s' % setName)
except:
logging.error('tags2set: Cannot create set %s' % setName)
logging.error(sys.exc_info()[0])
try:
fset.editPhotos(photos)
except:
logging.error('tags2set: Cannot edit set %s' % setName)
logging.error(sys.exc_info()[0])
logging.debug('tags2set: ...added %d photos' % len(photos) )
return fset
def process(pid, uid):
photo = flickr.Photo(pid)
photo._load_properties()
if photo._Photo__license in ["0", "7", "8"]:
print " *** Bad license:", photo._Photo__license
badImages.append(uid)
return None
author = photo._Photo__owner.realname
if not author:
author = photo._Photo__owner.username
url = max(photo.getSizes(), key=lambda item: item["width"])["source"]
return {"author": author.replace("'", "\\'"), "url": url}
def upload(self,filename, **params):
#x = flickr._prepare_params(params)
#args['api_key'] = self.__api_key
args = params
sig = flickr._get_api_sig(params=params)
args['api_key'] = flickr.API_KEY
args['api_sig'] = sig
args['auth_token'] = flickr.userToken()
f = file(filename, 'rb')
photo_data = f.read()
f.close()
# now make a "files" array to pass to uploader
files = [('photo', filename, photo_data)]
response = post_multipart('api.flickr.com', '/services/upload/', args, files)
# use get data since error checking is handled by it already
data = flickr._get_data(minidom.parseString(response))
photo = flickr.Photo(data.rsp.photoid.text)
return photo
def read_csv():
max_side = 1600
thumb_side = 425
feat_side = 850
quality = 90
if len(sys.argv) < 3:
print "------------------------------------------"
print "ERROR: Script requires 2 variables, an attribute type and a filename."
print "Example: python grab.py cip file.csv"
print "------------------------------------------"
return
attr_type = sys.argv[1]
if attr_type not in attr_map:
print "------------------------------------------"
print "ERROR: Invalid attribute type."
print "Allowed keys: {}".format(", ".join(attr_map.keys()))
print "------------------------------------------"
return
else:
table = attr_map[attr_type]
input_file = csv.DictReader(open(sys.argv[2]))
imgdir = os.path.join(FLICKR_DIR, attr_type)
thumbdir = imgdir.replace("splash", "thumb")
featdir = imgdir.replace("splash", "feature")
badImages = []
smallImages = []
goodImages = []
removedImages = []
# skip = True
if not os.path.exists(imgdir):
os.makedirs(imgdir)
if not os.path.exists(thumbdir):
os.makedirs(thumbdir)
if not os.path.exists(featdir):
os.makedirs(featdir)
for row in input_file:
update = False
uid = row["id"]
# if uid == "31000US12700":
# skip = False
#
# if skip:
# continue
image_only = attr_type == "geo"
if not image_only or (image_only and "image_link" in row
and row["image_link"] != ""):
if "level" in row:
attr = table.query.filter_by(id=uid,
level=row["level"]).first()
else:
attr = table.query.get(uid)
if attr and "image_link" in row:
image = row["image_link"]
# if image: # Use this if statement instead of the next line to force an update on all images.
if image and attr.image_link != image:
if "photolist" in image:
image = image.split("/in/photolist")[0]
pid = image.split("/")[-1]
if "flic.kr" not in image:
image = "http://flic.kr/p/{}".format(short.encode(pid))
photo = flickr.Photo(pid)
try:
photo._load_properties()
except:
removedImages.append(uid)
continue
image = {
"id": uid,
"url": image,
"license": photo._Photo__license
}
if image["license"] in ["0"]:
badImages.append(image)
else:
sizes = [
p for p in photo.getSizes()
if p["width"] >= max_side
]
if len(sizes) == 0:
smallImages.append(image)
else:
download_url = min(
sizes,
key=lambda item: item["width"])["source"]
imgpath = os.path.join(imgdir,
"{}.jpg".format(uid))
thumbpath = os.path.join(thumbdir,
"{}.jpg".format(uid))
featpath = os.path.join(featdir,
"{}.jpg".format(uid))
urllib.urlretrieve(download_url, imgpath)
img = pillow.open(imgpath).convert("RGB")
img.thumbnail((max_side, max_side),
pillow.ANTIALIAS)
img.save(imgpath, "JPEG", quality=quality)
img.thumbnail((feat_side, feat_side),
pillow.ANTIALIAS)
img.save(featpath, "JPEG", quality=quality)
img.thumbnail((thumb_side, thumb_side),
pillow.ANTIALIAS)
img.save(thumbpath, "JPEG", quality=quality)
author = photo._Photo__owner
author = author.realname if author.realname else author.username
image["author"] = author.replace("'", "\\'")
goodImages.append(image)
attr.image_link = image["url"]
attr.image_author = image["author"]
update = True
# set False to True to force thumbnails
elif False and image:
imgpath = os.path.join(imgdir, "{}.jpg".format(uid))
thumbpath = os.path.join(thumbdir, "{}.jpg".format(uid))
featpath = os.path.join(featdir, "{}.jpg".format(uid))
img = pillow.open(imgpath).convert("RGB")
img.thumbnail((feat_side, feat_side), pillow.ANTIALIAS)
img.save(featpath, "JPEG", quality=quality)
img.thumbnail((thumb_side, thumb_side), pillow.ANTIALIAS)
img.save(thumbpath, "JPEG", quality=quality)
if not image_only:
name = row["name"]
if attr and name and attr.name != name:
attr.name = name
update = True
if "image_meta" in row:
meta = row["image_meta"]
if attr and meta and attr.image_meta != meta:
attr.image_meta = meta
update = True
if update:
db.session.add(attr)
db.session.commit()
# break
print "{} new images have been processed.".format(len(goodImages))
if len(badImages) > 0:
print "The following images have bad licenses: {}".format(", ".join(
[i["id"] for i in badImages]))
if len(smallImages) > 0:
print "The following images are too small: {}".format(", ".join(
[i["id"] for i in smallImages]))
if len(removedImages) > 0:
print "The following images have been removed: {}".format(", ".join(
[i for i in removedImages]))
def read_csv():
thumb_side = 425
quality = 90
if len(sys.argv) < 3:
print "------------------------------------------"
print "ERROR: Script requires 2 variables, an attribute type and a filename."
print "Example: python grab.py cip file.csv"
print "------------------------------------------"
return
attr_type = sys.argv[1]
if attr_type not in attr_map:
print "------------------------------------------"
print "ERROR: Invalid attribute type."
print "Allowed keys: {}".format(", ".join(attr_map.keys()))
print "------------------------------------------"
return
else:
table = attr_map[attr_type]
csvFilename = sys.argv[2]
csvReader = csv.DictReader(open(csvFilename))
input_file = list(csvReader)
imgdir = os.path.join(FLICKR_DIR, attr_type)
thumbdir = imgdir.replace("splash", "thumb")
badImages = []
smallImages = []
goodImages = []
removedImages = []
deletedImages = []
# skip = True
if not os.path.exists(imgdir):
os.makedirs(imgdir)
if not os.path.exists(thumbdir):
os.makedirs(thumbdir)
for row in input_file:
update = False
uid = row["id"]
imgpath = os.path.join(imgdir, "{}.jpg".format(uid))
thumbpath = os.path.join(thumbdir, "{}.jpg".format(uid))
image_only = attr_type == "geo"
if "level" in row:
attr = table.query.filter_by(id=uid,level=row["level"]).first()
else:
attr = table.query.get(uid)
if attr and "image_link" in row:
image = row["image_link"]
if not image and attr.image_link:
attr.image_meta = None
attr.image_link = None
attr.image_author = None
update = True
deletedImages.append(uid)
row["error"] = ""
os.remove(imgpath)
os.remove(thumbpath)
elif image and attr.image_link != image:
if "photolist" in image:
image = image.split("/in/photolist")[0]
pid = image.split("/")[-1]
if "flic.kr" not in image:
image = "http://flic.kr/p/{}".format(short.encode(pid))
photo = flickr.Photo(pid)
try:
photo._load_properties()
except:
row["error"] = "removed"
removedImages.append(uid)
continue
image = {"id": uid, "url": image, "license": photo._Photo__license}
if image["license"] not in LICENSES:
badImages.append(image)
row["error"] = "license-{}".format(image["license"])
else:
sizes = [p for p in photo.getSizes() if p["width"] >= MAX_SIDE]
if len(sizes) == 0:
smallImages.append(image)
row["error"] = "resolution"
else:
download_url = min(sizes, key=lambda item: item["width"])["source"]
urllib.urlretrieve(download_url, imgpath)
img = pillow.open(imgpath).convert("RGB")
img.thumbnail((MAX_SIDE, MAX_SIDE), pillow.ANTIALIAS)
img.save(imgpath, "JPEG", quality=quality)
img.thumbnail((thumb_side, thumb_side), pillow.ANTIALIAS)
img.save(thumbpath, "JPEG", quality=quality)
author = photo._Photo__owner
author = author.realname if author.realname else author.username
image["author"] = author.replace("'", "\\'")
goodImages.append(image)
attr.image_link = image["url"]
attr.image_author = image["author"]
update = True
# set False to True to force thumbnails
elif False and image:
imgpath = os.path.join(imgdir, "{}.jpg".format(uid))
thumbpath = os.path.join(thumbdir, "{}.jpg".format(uid))
img = pillow.open(imgpath).convert("RGB")
img.thumbnail((thumb_side, thumb_side), pillow.ANTIALIAS)
img.save(thumbpath, "JPEG", quality=quality)
if not image_only:
name = row["name"]
if attr and name and attr.name != name:
attr.name = name
update = True
if "image_meta" in row:
meta = row["image_meta"]
if attr and meta and attr.image_meta != meta:
attr.image_meta = meta
update = True
if update:
db.session.add(attr)
db.session.commit()
# break
print "\n"
print "Outputing to CSV..."
with open(csvFilename.replace(".csv", "-update.csv"), 'wb') as f:
w = csv.DictWriter(f, None)
w.fieldnames = csvReader.fieldnames
w.writerow(dict((h, h) for h in csvReader.fieldnames))
for row in input_file:
w.writerow(row)
print "\n"
print "{} new images have been processed.".format(len(goodImages))
if len(badImages) > 0:
print "The following images have bad licenses: {}".format(", ".join([i["id"] for i in badImages]))
if len(smallImages) > 0:
print "The following images are too small: {}".format(", ".join([i["id"] for i in smallImages]))
if len(removedImages) > 0:
print "The following images have been removed from Flickr: {}".format(", ".join([i for i in removedImages]))
if len(deletedImages) > 0:
print "The following images have been deleted: {}".format(", ".join([i for i in deletedImages]))
def read_csv():
if len(sys.argv) < 3:
print "------------------------------------------"
print "ERROR: Script requires 2 variables, an attribute type and a filename."
print "Example: python grab.py cip file.csv"
print "------------------------------------------"
return
attr_type = sys.argv[1]
csvReader = csv.DictReader(open(sys.argv[2]))
input_file = list(csvReader)
images = []
print "Analyzing {} images".format(attr_type.upper())
for index, row in enumerate(input_file):
sys.stdout.write("\r{} of {}".format(index, len(input_file)))
sys.stdout.flush()
uid = row["id"]
if "image_link" in row and row["image_link"] != "":
image = row["image_link"]
if "photolist" in image:
image = image.split("/in/photolist")[0]
pid = image.split("/")[-1]
if "flic.kr" not in image:
image = "http://flic.kr/p/{}".format(short.encode(pid))
image = {"id": uid, "url": image, "small": False, "removed": False}
row["error"] = ""
photo = flickr.Photo(pid)
try:
photo._load_properties()
except:
image["removed"] = True
row["error"] = "removed"
if photo._Photo__license:
image["license"] = photo._Photo__license
if image["license"] in LICENSES:
if len([
p for p in photo.getSizes()
if p["width"] >= MAX_SIDE
]) == 0:
image["small"] = True
row["error"] = "resolution"
else:
row["error"] = "license-{}".format(image["license"])
images.append(image)
print "\n"
print "Outputing to CSV..."
with open(sys.argv[2].replace(".csv", "-update.csv"), 'wb') as f:
w = csv.DictWriter(f, None)
w.fieldnames = csvReader.fieldnames
w.writerow(dict((h, h) for h in csvReader.fieldnames))
for row in input_file:
w.writerow(row)
print "\n"
num_images = float(len(images))
print "{} images have been analyzed".format(int(num_images))
bads = sum(1 for image in images
if "license" in image and image["license"] not in LICENSES)
print "{} ({}%) have bad licenses".format(bads,
round((bads / num_images) * 100))
smalls = sum(1 for image in images if image["small"])
print "{} ({}%) are too small".format(smalls,
round((smalls / num_images) * 100))
missing = sum(1 for image in images if image["removed"])
print "{} ({}%) have been removed from Flickr".format(
missing, round((missing / num_images) * 100))